From 60be4fcfa8007f65fbed83e2bf80d6dd7f79ab8f Mon Sep 17 00:00:00 2001 From: Scott Griffin Date: Wed, 13 May 2015 05:29:50 +0000 Subject: [PATCH 1/2] Now lazy-loading s3 connection and s3 buckets. Making these newtork calls proactively in our constructors is not a good practice. Constructors should only configure the object for use and NOT actually instantiate non-config object properties. Perfect case for lazy-loading is testing. Unless the object is heavily mocked these network calls would 1) require a network connection for testing even if the object is only instanted in the test and not used and 2) slow down tests due to network calls. --- muskrat/producer.py | 24 ++++++++++++++++++------ muskrat/s3consumer.py | 17 ++++++++++++++--- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/muskrat/producer.py b/muskrat/producer.py index 5b7fb0b..4270f9f 100644 --- a/muskrat/producer.py +++ b/muskrat/producer.py @@ -1,7 +1,6 @@ """ " Copyright: Loggly " Author: Scott Griffin -" Last Updated: 02/22/2013 " """ try: import simplejson as json @@ -83,12 +82,25 @@ class S3Producer( BaseProducer ): def __init__(self, **kwargs): super( S3Producer, self ).__init__(**kwargs) - self.s3conn = boto.connect_s3( self.config.s3_key, self.config.s3_secret ) - - self.bucket = self.s3conn.lookup( self.config.s3_bucket ) + self._s3conn = None + self._bucket = None + + @property + def s3conn(self): + if self._s3conn is None: + self._s3conn = boto.connect_s3( self.config.s3_key, self.config.s3_secret ) + return self._s3conn + + @property + def bucket(self): + if self._bucket is None: + self._bucket = self.s3conn.lookup( self.config.s3_bucket ) - if not self.bucket: - self.bucket = self.s3conn.create_bucket( self.config.s3_bucket ) + if not self._bucket: + self._bucket = self.s3conn.create_bucket( self.config.s3_bucket ) + + return self._bucket + def send( self, msg, **kwargs ): """ diff --git a/muskrat/s3consumer.py b/muskrat/s3consumer.py index d762597..97551b7 100644 --- a/muskrat/s3consumer.py +++ b/muskrat/s3consumer.py @@ -1,7 +1,6 @@ """ " Copyright: Loggly " Author: Scott Griffin -" Last Updated: 02/22/2013 " " This class provides the ability to register a function as " a consumer to an S3 routing_key. This class also handles tracking @@ -74,8 +73,8 @@ class S3Consumer(object): def __init__(self, routing_key, func, name=None, config='config.py'): self.config = config_loader( config ) - self.s3conn = boto.connect_s3( self.config.s3_key, self.config.s3_secret ) - self.bucket = self.s3conn.get_bucket( self.config.s3_bucket ) + self._s3conn = None + self._bucket = None self.routing_key = routing_key.upper() self.callback = func @@ -90,6 +89,18 @@ def __init__(self, routing_key, func, name=None, config='config.py'): location=self.config.s3_cursor['location'] ) + + @property + def s3conn(self): + if self._s3conn is None: + self._s3conn = boto.connect_s3( self.config.s3_key, self.config.s3_secret ) + return self._s3conn + + @property + def bucket(self): + if self._bucket is None: + self._bucket = self.s3conn.get_bucket( self.config.s3_bucket ) + return self._bucket def _gen_name(self, func): """ Generates a cursor name so that the cursor can be re-attached to """ From d0bd7de7ef82d76bfa9333a2fc3a0ef36b796f28 Mon Sep 17 00:00:00 2001 From: Scott Griffin Date: Wed, 13 May 2015 05:39:38 +0000 Subject: [PATCH 2/2] Made version a property of the actual module. Updated setup.py to use this version property so there is only one source of truth. --- muskrat/__init__.py | 1 + setup.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/muskrat/__init__.py b/muskrat/__init__.py index e69de29..cd7ca49 100644 --- a/muskrat/__init__.py +++ b/muskrat/__init__.py @@ -0,0 +1 @@ +__version__ = '1.0.1' diff --git a/setup.py b/setup.py index 0b4013e..fffe736 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,16 @@ """ " Copyright: Loggly " Author: Scott Griffin -" Last Updated: 01/14/2013 " """ from setuptools import setup +from muskrat import __version__ setup( name='Muskrat', author='Scott Griffin', author_email='scott@loggly.com', - version='0.1dev', + version=__version__, packages=['muskrat', 'muskrat.tests'], long_description=open( 'README.md' ).read(), install_requires=open( 'requirements.txt' ).read().split()