From 3d572a76ab91f8847bec4b7086e58e3cc60554a6 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Mon, 23 Nov 2015 18:43:30 -0800 Subject: [PATCH 1/3] updates to hoover - add retry support to events - change APIs to v2 url --- hoover/session.py | 27 +++++++++++++++++++++++---- setup.py | 2 +- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/hoover/session.py b/hoover/session.py index 48aa4a3..82ff3de 100644 --- a/hoover/session.py +++ b/hoover/session.py @@ -2,11 +2,14 @@ from hoover.exceptions import NotFound, AuthFail from hoover.utils import time_translate import logging + try: from urllib import urlencode except ImportError: from urllib.parse import urlencode import requests +import requests.exceptions + try: from simplejson import loads except ImportError: @@ -52,7 +55,7 @@ def inputs(self): return self._inputs def _inputs_init(self): - inputs = self._api_help('api/inputs') + inputs = self._api_help('apiv2/inputs') self._inputs = [LogglyInput(i, self) for i in inputs] @property @@ -84,14 +87,30 @@ def search(self, q='*', **kwargs): '''Thin wrapper on Loggly's text search API. First parameter is a query string.''' kwargs['q'] = q - return self._api_help('api/search', kwargs) + return self._api_help('apiv2/search', kwargs) + + def events(self, search_result, num_retries=5, **kwargs): + '''Thin wrapper on Loggly's events API. First parameter is the result from search.''' + kwargs['rsid'] = search_result['rsid']['id'] + + # large requests may take some time + retries = 0 + while retries < num_retries: + try: + return self._api_help('apiv2/events', kwargs) + except requests.exceptions.HTTPError as e: + if e.response.status_code == 504: + logging.getLogger('hoover').info('Retrying due to request timeout') + retries += 1 + else: + raise @time_translate def facets(self, q='*', facetby='date', **kwargs): '''Thin wrapper on Loggly's facet search API. facetby can be input, ip, or a json parameter of the form json.foo''' kwargs['q'] = q - return self._api_help('api/facets/%s' % facetby, kwargs) + return self._api_help('apiv2/facets/%s' % facetby, kwargs) def create_input(self, name, service='syslogudp', description='', json=False): @@ -107,7 +126,7 @@ def create_input(self, name, service='syslogudp', description='', format = json and 'json' or 'text' params = {'name': name, 'service': service, 'description': description, 'format': format} - result = self._api_help('api/inputs', params, method='POST') + result = self._api_help('apiv2/inputs', params, method='POST') try: newinput = LogglyInput(result, self) except: diff --git a/setup.py b/setup.py index 19292ce..2887dca 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ from distutils.core import setup setup(name='Hoover', - version='0.6.0', + version='0.6.2', description="Library for logging to Loggly from within Python webapps", author="Mike Blume", author_email="mike@loggly.com", From 85d4555e7f47a4161cfb73316e78c8816f28a2c4 Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 24 Mar 2017 09:12:42 -0700 Subject: [PATCH 2/3] switch to triple quote --- hoover/session.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hoover/session.py b/hoover/session.py index 82ff3de..3a3da78 100644 --- a/hoover/session.py +++ b/hoover/session.py @@ -84,13 +84,13 @@ def config_inputs(self): @time_translate def search(self, q='*', **kwargs): - '''Thin wrapper on Loggly's text search API. First parameter is a query - string.''' + """Thin wrapper on Loggly's text search API. First parameter is a query + string.""" kwargs['q'] = q return self._api_help('apiv2/search', kwargs) def events(self, search_result, num_retries=5, **kwargs): - '''Thin wrapper on Loggly's events API. First parameter is the result from search.''' + """Thin wrapper on Loggly's events API. First parameter is the result from search.""" kwargs['rsid'] = search_result['rsid']['id'] # large requests may take some time From 844aabbf503b8b1bbbbbfdda3ebe5f4c19865aaf Mon Sep 17 00:00:00 2001 From: Alexander Mohr Date: Fri, 24 Mar 2017 12:40:41 -0700 Subject: [PATCH 3/3] add support for events iterator --- hoover/session.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/hoover/session.py b/hoover/session.py index 3a3da78..27852c7 100644 --- a/hoover/session.py +++ b/hoover/session.py @@ -35,19 +35,23 @@ def __init__(self, subdomain, username, password, domain=None, proxy=None, self.proxy = proxy self.protocol = secure and 'https' or 'http' - def _api_help(self, endpoint, params=None, method='GET'): + def _do_api_req(self, method, url, params=None, body=''): s = requests.Session() s.auth = (self.username, self.password) - url = '%s://%s.%s/%s' % (self.protocol, self.subdomain, self.domain, - endpoint) - body = '' - if params and method != 'GET': - body = urlencode(params) - params = None response = s.request(method, url, params=params, data=body, verify=True) response.raise_for_status() return response.json() + + def _api_help(self, endpoint, params=None, method='GET'): + url = '%s://%s.%s/%s' % (self.protocol, self.subdomain, self.domain, + endpoint) + + if params and method != 'GET': + return self._do_api_req(method, url, None, urlencode(params)) + + return self._do_api_req(method, url, params) + @property def inputs(self): if not hasattr(self, '_inputs'): @@ -82,6 +86,21 @@ def config_inputs(self): logger = logging.getLogger(input.name) logger.addHandler(input.get_handler()) + @time_translate + def search_iterator(self, q='*', **kwargs): + """Thin wrapper on Loggly's text search iterator API. First parameter is a query + string.""" + kwargs['q'] = q + response = self._api_help('apiv2/events/iterate', kwargs) + yield response['events'] + + while response.get('next'): + response = self._do_api_req('GET', response.get('next')) + + if response.get('events'): # last one may be empty + yield response['events'] + + @time_translate def search(self, q='*', **kwargs): """Thin wrapper on Loggly's text search API. First parameter is a query