Returns the account ID as a string associated with the acct object.
def acct_id(self, acct):
+ """Returns the account ID as a string associated with the `acct` object."""
+ raise NotImplementedError
def attributes(self)
Returns a dict of all metadata attribute names and values.
def attributes(self):
+ """Returns a dict of all metadata attribute names and values."""
+ raise NotImplementedError
def accounts(self, acct_ids=None, include=None, exclude=None)
@@ -1403,6 +1421,21 @@ Methods
Duplicate account IDs do not result in duplicate account objects. To
filter the list based on account metadata, include and exclude dicts
can be specified identifying the keys and matching values.
+
+
+Expand source code
+
+def accounts(self, acct_ids=None, include=None, exclude=None):
+ """Returns a list of account objects representing accounts and metadata.
+
+ Without any arguments, all accounts are returned. A list of `acct_ids`
+ may be provided to limit the list to accounts matching the account IDs.
+ Duplicate account IDs do not result in duplicate account objects. To
+ filter the list based on account metadata, `include` and `exclude` dicts
+ can be specified identifying the keys and matching values.
+ """
+ raise NotImplementedError
+
@@ -1496,6 +1529,18 @@ Methods
Returns the account ID associated with the acct object.
This is an identity function as the account ID of an account object
loaded by this class is the account object itself, thus acct returned.
+Expand source code
+
+def acct_id(self, acct):
+ """Returns the account ID associated with the `acct` object.
+
+ This is an identity function as the account ID of an account object
+ loaded by this class is the account object itself, thus `acct` returned.
+ """
+ return acct
+
def attributes(self)
@@ -1505,6 +1550,19 @@ Methods
IdentityAccountLoader builds account objects that are simply strings
representing the ID of an account. Because there is no other metadata
associated with these objects, this method returns an empty dict.
+
+
+Expand source code
+
+def attributes(self):
+ """Returns a dict of all metadata attribute names and values.
+
+ `IdentityAccountLoader` builds account objects that are simply strings
+ representing the ID of an account. Because there is no other metadata
+ associated with these objects, this method returns an empty dict.
+ """
+ return {}
+
def accounts(self, acct_ids=None, include=None, exclude=None)
@@ -1517,6 +1575,28 @@ Methods
objects without duplicates. It is an identity function. Because there
are no loaded accounts, use of include and exclude parameters will
raise an AttributeError.
+
+
+Expand source code
+
+def accounts(self, acct_ids=None, include=None, exclude=None):
+ """Returns a list of account objects.
+
+ Without any arguments, no accounts are returned because this class does
+ not load a list of accounts from an external data source. Instead, this
+ method returns the same list of `acct_ids` as the list of account
+ objects without duplicates. It is an identity function. Because there
+ are no loaded accounts, use of `include` and `exclude` parameters will
+ raise an AttributeError.
+ """
+ if include or exclude:
+ raise AttributeError("Cannot use filters as no attributes are defined")
+
+ if acct_ids is None:
+ return []
+
+ return list(set(acct_ids))
+
@@ -1815,7 +1895,7 @@ Methods
)
self.CustomAccount._str_template = ( # type: ignore
self.str_template
- ) # pylint: disable=protected-access
+ )
self.accts, self.attrs = self._parse(accts)
LOG.info(
@@ -2143,6 +2223,20 @@ Methods
instance of this MetaAccountLoader via MetaAccountLoader.accounts().
This method returns the string representing the account ID of the
account object.
+
+
+Expand source code
+
+def acct_id(self, acct):
+ """Returns the account ID associated with the `acct` object.
+
+ The `acct` parameter must be an account object that was created by an
+ instance of this `MetaAccountLoader` via `MetaAccountLoader.accounts`.
+ This method returns the string representing the account ID of the
+ account object.
+ """
+ return getattr(acct, self.id_attr)
+
def attributes(self)
@@ -2165,6 +2259,36 @@ Methods
assert attrs['status'] == {'active', 'suspended'}
assert attrs['id'] == {'100200300400', '200300400100', '300400100200'}
+
+
+Expand source code
+
+def attributes(self):
+ """Returns a dict of all metadata attribute names and values.
+
+ This method returns the metadata associated with the account objects
+ created by this instance of `MetaAccountLoader`. The keys of the
+ returned dict are the attribute names attached to the account objects.
+ The value of each key is a set representing all of the possible values
+ assigned to that attribute. For example:
+
+ accts = [
+ {'id': '100200300400', 'env': 'prod', 'status': 'active'},
+ {'id': '200300400100', 'env': 'prod', 'status': 'suspended'},
+ {'id': '300400100200', 'env': 'dev', 'status': 'active'},
+ ]
+ loader = acctload.MetaAccountLoader(accts)
+ attrs = loader.attributes()
+ assert attrs['env'] == {'prod', 'dev'}
+ assert attrs['status'] == {'active', 'suspended'}
+ assert attrs['id'] == {'100200300400', '200300400100', '300400100200'}
+ """
+ d = defaultdict(set)
+ for acct in self.accts:
+ for attr, value in acct.items():
+ d[attr].add(value)
+ return d
+
def accounts(self, acct_ids=None, include=None, exclude=None)
@@ -2209,6 +2333,84 @@ Methods
to an invalid attribute name, AttributeError is raised. Finally, if
include is not set, then all accounts are matched. Likewise, if
exclude is not set, then no accounts are excluded.
+
+
+Expand source code
+
+def accounts(self, acct_ids=None, include=None, exclude=None):
+ """Returns a list of account objects.
+
+ Without any arguments, account objects for all accounts are returned. A
+ list of `acct_ids` may be provided to limit the list to accounts
+ matching the account IDs. If one or more specified account IDs is
+ missing, `AccountsNotFoundError` is raised.
+
+ The returned list of accounts can be filtered further by providing dicts
+ for `include` and `exclude` parameters that specify attributes and a
+ list of values for those attributes that must match. For example,
+ assuming the following accounts:
+
+ accts = [
+ {'id': '100200300400', 'env': 'prod', 'status': 'active'},
+ {'id': '200300400100', 'env': 'prod', 'status': 'suspended'},
+ {'id': '300400100200', 'env': 'dev', 'status': 'active'},
+ ]
+ loader = acctload.MetaAccountLoader(accts)
+
+ To filter active accounts, use the following:
+
+ include = {'status': ['active']}
+
+ To filter active *and* production accounts:
+
+ include = {'status': ['active'], 'env': ['prod']}
+
+ To filter production accounts, but not suspended:
+
+ include = {'env': ['prod']}
+ exclude = {'status': ['suspended']}
+
+ To filter active *or* suspended accounts, but not dev accounts:
+
+ include = {'status': ['active', 'suspended']}
+ exclude = {'env': ['dev']}
+
+ Both `include` and `exclude` filters are applied after the initial
+ account list has been determined, which is all accounts unless it was
+ first limited by `acct_ids`. Then, the `include` filter is applied,
+ followed by the `exclude` filter. If a filter dict has multiple keys,
+ then *each* key must match. A key matches if at least *one* of the
+ values matches as illustrated in the examples above. If a filter refers
+ to an invalid attribute name, `AttributeError` is raised. Finally, if
+ `include` is not set, then all accounts are matched. Likewise, if
+ `exclude` is not set, then no accounts are excluded.
+ """
+ accts = self.accts
+ acct_ids = [] if acct_ids is None else acct_ids
+ include = {} if include is None else include
+ exclude = {} if exclude is None else exclude
+
+ # Limit our account list to the requested IDs
+ if acct_ids:
+ requested = set(acct_ids)
+ all_ids = (a[self.id_attr] for a in self.accts)
+
+ missing_acct_ids = requested.difference(all_ids)
+ if missing_acct_ids:
+ raise AccountsNotFoundError(list(missing_acct_ids))
+
+ accts = (a for a in self.accts if a[self.id_attr] in requested)
+
+ # Make sure the filters contain valid attribute names
+ for attr in itertools.chain(include.keys(), exclude.keys()):
+ if attr not in self.attrs:
+ raise AttributeError(f"Invalid attribute '{attr}' in filter")
+
+ # Limit our account list by the user-supplied filters
+ return [
+ self.CustomAccount(a) for a in accts if self._filter(a, include, exclude)
+ ]
+
@@ -2945,7 +3147,7 @@ Ancestors
def __init__(
self,
url,
- parser=JSONParser(),
+ parser=_DEFAULT_JSON_PARSER,
auth=None,
max_age=0,
id_attr="id",
@@ -2956,7 +3158,6 @@ Ancestors
no_verify=False,
cache_path=None,
):
-
session = requests.Session()
session.mount("file://", FileAdapter())
@@ -3071,11 +3272,11 @@ Inherited members
return value
def __eq__(self, other):
- return self._attrs == other._attrs # pylint: disable=protected-access
+ return self._attrs == other._attrs
def __repr__(self):
- pairs = (f"{k}={repr(v)}" for k, v in self._attrs.items())
- return f'Account({", ".join(pairs)})'
+ pairs = (f"{k}={v!r}" for k, v in self._attrs.items())
+ return f"Account({', '.join(pairs)})"
def __str__(self):
if not self._str_template:
@@ -3106,7 +3307,7 @@ Inherited members
def __init__(self, missing_acct_ids):
self.missing_acct_ids = missing_acct_ids
- super().__init__(f'Account IDs not found: {", ".join(missing_acct_ids)}')
+ super().__init__(f"Account IDs not found: {', '.join(missing_acct_ids)}")
Ancestors
@@ -3238,7 +3439,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/argparse.html b/docs/argparse.html
index 97c404e..35ff6e5 100644
--- a/docs/argparse.html
+++ b/docs/argparse.html
@@ -3,13 +3,13 @@
-
+
awsrun.argparse API documentation
-
+
@@ -291,6 +291,42 @@ Functions
>>> [f(s) for s in ['yes', 'no', 'true', 'false']]
[True, False, True, False]
+
+
+Expand source code
+
+def from_str_to(type_):
+ """Return a cast function to convert a string to a builtin type.
+
+ The `type` parameter is the name of the type as a string. Returns the
+ builtin Python cast function if `type` is "str", "int", or "float". If
+ `type` is "bool", the returned cast function will return `True` for the
+ values "y", "yes", "true", and "1" (case insensitive), otherwise it will
+ return `False`. For any other `type` specified, the builtin `str`
+ function is returned.
+
+ >>> f = from_str_to("str")
+ >>> f("hello")
+ "hello"
+
+ >>> f = from_str_to("int")
+ >>> f("10")
+ 10
+
+ >>> f = from_str_to("float")
+ >>> f("10")
+ 10.0
+
+ >>> f = from_str_to("bool")
+ >>> [f(s) for s in ['yes', 'no', 'true', 'false']]
+ [True, False, True, False]
+ """
+ if type_ in ("str", "int", "float"):
+ return getattr(builtins, type_)
+ if type_ == "bool":
+ return lambda s: s.lower() in ("y", "yes", "true", "True", "1")
+ return str
+
@@ -397,7 +433,7 @@ Ancestors
class AppendAttributeValuePair
-(option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None)
+(option_strings, dest, nargs=None, const=None, default=None, type=None, choices=None, required=False, help=None, metavar=None, deprecated=False)
-
Argparse action to construct a dict of key/value pairs.
@@ -573,7 +609,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/cache.html b/docs/cache.html
index 5434733..b7c7984 100644
--- a/docs/cache.html
+++ b/docs/cache.html
@@ -3,13 +3,13 @@
-
+
awsrun.cache API documentation
-
+
@@ -97,6 +97,24 @@ Overview
is before the value would have expired, and thus the cached result is returned.
The third value, however, is ten seconds later because by the time the third
invocation of value took place, the original value expired after 10 seconds.
+Dynamic Expiration
+In some cases, the expiration time is only known after the value has been
+obtained (e.g., OAuth2 tokens include expiration in the response). Use
+ValueWithExpiry to wrap the value with a custom expiration. This overrides
+the max_age specified in the ExpiringValue constructor. For example:
+>>> def fetch_token():
+... token = {'access_token': 'EXAMPLE_TOKEN', 'expires_in': 3600}
+... return ValueWithExpiry(token, ttl=token['expires_in'])
+>>> ev = ExpiringValue(refresh_fn=fetch_token, max_age=0)
+>>> ev.value()
+{'access_token': 'EXAMPLE_TOKEN', 'expires_in': 3600}
+
+The ValueWithExpiry wrapper supports both TTL (time-to-live in seconds) and
+absolute timestamps via the expires_at parameter.
+When using dynamic expiry, max_age becomes relevant if the refresh function
+can return a mix of both plain values and ValueWithExpiry instances. When
+a plain value is returned, it is cached for max_age. If only dynamic expiry
+is used, then max_age is not used.
Expand source code
@@ -131,6 +149,28 @@ Overview
is before the value would have expired, and thus the cached result is returned.
The third value, however, is ten seconds later because by the time the third
invocation of `value` took place, the original value expired after 10 seconds.
+
+## Dynamic Expiration
+
+In some cases, the expiration time is only known after the value has been
+obtained (e.g., OAuth2 tokens include expiration in the response). Use
+`ValueWithExpiry` to wrap the value with a custom expiration. This overrides
+the `max_age` specified in the `ExpiringValue` constructor. For example:
+
+ >>> def fetch_token():
+ ... token = {'access_token': 'EXAMPLE_TOKEN', 'expires_in': 3600}
+ ... return ValueWithExpiry(token, ttl=token['expires_in'])
+ >>> ev = ExpiringValue(refresh_fn=fetch_token, max_age=0)
+ >>> ev.value()
+ {'access_token': 'EXAMPLE_TOKEN', 'expires_in': 3600}
+
+The `ValueWithExpiry` wrapper supports both TTL (time-to-live in seconds) and
+absolute timestamps via the `expires_at` parameter.
+
+When using dynamic expiry, `max_age` becomes relevant if the refresh function
+can return a mix of both plain values and `ValueWithExpiry` instances. When
+a plain value is returned, it is cached for `max_age`. If only dynamic expiry
+is used, then `max_age` is not used.
"""
import json
@@ -142,17 +182,53 @@ Overview
LOG = logging.getLogger(__name__)
+class ValueWithExpiry:
+ """Wrapper to return a value with a custom expiration time.
+
+ Use this when your refresh function needs to specify its own expiration
+ rather than using the default max_age. The expiry can be specified as
+ either a TTL in seconds or an absolute timestamp.
+
+ Example usage with TTL::
+
+ def refresh_oauth_token():
+ token = fetch_token() # Returns {'access_token': '...', 'expires_in': 3600}
+ return ValueWithExpiry(token, ttl=token['expires_in'])
+
+ Example usage with absolute timestamp::
+
+ def refresh_oauth_token():
+ token = fetch_token() # Returns {'access_token': '...', 'expires_at': 1702915200}
+ return ValueWithExpiry(token, expires_at=token['expires_at'])
+ """
+
+ def __init__(self, value, *, ttl=None, expires_at=None):
+ if ttl is None and expires_at is None:
+ raise ValueError("Must specify either ttl or expires_at")
+ if ttl is not None and expires_at is not None:
+ raise ValueError("Cannot specify both ttl and expires_at")
+
+ self.value = value
+ self.expires_at = expires_at if expires_at else time.time() + ttl
+
+ def __repr__(self):
+ return f"ValueWithExpiry(value={self.value!r}, expires_at={self.expires_at})"
+
+
class AbstractExpiringValue:
"""Abstract base class to represent a value that expires.
An `AbstractExpiringValue` represents a lazily loaded value that will expire
over time. The constructor takes a `refresh_fn` function of zero arguments,
- which is called to obtain the value to be cached for `max_age` seconds.
+ which is called to obtain the value to be cached. The value is cached for
+ `max_age` seconds unless the `refresh_fn` returns a `ValueWithExpiry` object
+ that specifies a custom expiration time. If so, the default `max_age` is
+ ignored.
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
- method is called. This class is thread-safe. Subclasses must provide
+ method is called. The `value` method is thread-safe. Subclasses must provide
implementations for `is_expired`, `load`, and `save`.
"""
@@ -169,19 +245,35 @@ Overview
invocations of this method will return the cached value until it
expires. If you set `refresh` parameter to `True`, the value will be
refreshed and the expiration will be reset before being returned.
+
+ This method is thread-safe.
"""
with self._lock:
if not refresh and not self.is_expired():
return self.load()
- value = self._refresh_fn()
- self.save(value)
+ result = self._refresh_fn()
+
+ # Unwrap and extract expiry if ValueWithExpiry
+ if isinstance(result, ValueWithExpiry):
+ value = result.value
+ expiry = result.expires_at
+ else:
+ value = result
+ expiry = None
+
+ self.save(value, expiry)
LOG.info("refreshed data and saved in cache")
return value
def is_expired(self):
"""Returns `True` if the value needs to be refreshed, `False` otherwise.
+ A value needs to be refreshed when it has expired. This is determined
+ by the implementation of this method. By default, the value is cached for
+ `max_age` seconds or the dynamic expiry specified by the `refresh_fn`
+ return value (if it is a `ValueWithExpiry` instance).
+
If this returns `True` during the invocation of
`AbstractExpiringValue.value`, the `refresh_fn` will be called, followed
by `save`, to renew the cached value. When this returns `False`, `load`
@@ -198,12 +290,14 @@ Overview
"""
raise NotImplementedError
- def save(self, value):
+ def save(self, value, expiry=None):
"""Saves the value to the cache.
If `is_expired` returns `True` during the invocation of
`AbstractExpiringValue.value`, this method is invoked to save the new
- value to the cache.
+ value to the cache. If `expiry` is provided, it specifies the absolute
+ timestamp when the value should expire. Otherwise, the default `max_age`
+ should be used.
"""
raise NotImplementedError
@@ -219,7 +313,7 @@ Overview
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
- method is called. This class is thread-safe.
+ method is called. The `value` method is thread-safe.
"""
def __init__(self, refresh_fn, max_age):
@@ -234,10 +328,10 @@ Overview
LOG.debug("Loading data from cache")
return self._value
- def save(self, value):
- LOG.debug("Saving value to cache")
+ def save(self, value, expiry=None):
self._value = value
- self._expiry = time.time() + self._max_age
+ self._expiry = expiry if expiry is not None else time.time() + self._max_age
+ LOG.debug("Saving value to cache, will expire at %s", time.ctime(self._expiry))
class PersistentExpiringValue(ExpiringValue):
@@ -252,17 +346,24 @@ Overview
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
- method is called. This class is thread-safe. If the value cannot be
+ method is called. The `value` method is thread-safe. If the value cannot be
persisted as JSON, a TypeError is thrown.
"""
def __init__(self, refresh_fn, path, max_age):
super().__init__(refresh_fn, max_age)
self._path = path if isinstance(path, Path) else Path(path)
+ self._expiry_path = self._path.with_suffix(self._path.suffix + ".expiry")
def is_expired(self):
if not self._path.exists():
return True
+ # Check for explicit expiry file first
+ if self._expiry_path.exists():
+ expiry = float(self._expiry_path.read_text(encoding="utf-8").strip())
+ return time.time() > expiry
+
+ # Otherwise fall back to file modification time + max_age
last_modification = self._path.stat().st_mtime
return time.time() > last_modification + self._max_age
@@ -271,9 +372,9 @@ Overview
with self._path.open("r", encoding="utf-8") as file:
return json.load(file)
- def save(self, value):
- # No need to persist the file if max_age is 0 seconds.
- if self._max_age == 0:
+ def save(self, value, expiry=None):
+ # No need to persist the file if max_age is 0 seconds (and not dynamic).
+ if self._max_age == 0 and expiry is None:
return
LOG.debug("Saving data to cache file %s", self._path)
@@ -282,7 +383,13 @@ Overview
json.dump(value, file)
# Pathlib.replace uses os.replace which is atomic on POSIX systems
- tmp.replace(self._path)
+ tmp.replace(self._path)
+
+ # Write or remove expiry file
+ if expiry is not None:
+ self._expiry_path.write_text(str(expiry), encoding="utf-8")
+ elif self._expiry_path.exists():
+ self._expiry_path.unlink()
@@ -294,6 +401,62 @@ Overview
Classes
+
+class ValueWithExpiry
+(value, *, ttl=None, expires_at=None)
+-
+
Wrapper to return a value with a custom expiration time.
+Use this when your refresh function needs to specify its own expiration
+rather than using the default max_age. The expiry can be specified as
+either a TTL in seconds or an absolute timestamp.
+Example usage with TTL::
+def refresh_oauth_token():
+ token = fetch_token() # Returns {'access_token': '...', 'expires_in': 3600}
+ return ValueWithExpiry(token, ttl=token['expires_in'])
+
+Example usage with absolute timestamp::
+def refresh_oauth_token():
+ token = fetch_token() # Returns {'access_token': '...', 'expires_at': 1702915200}
+ return ValueWithExpiry(token, expires_at=token['expires_at'])
+
+Expand source code
+
+class ValueWithExpiry:
+ """Wrapper to return a value with a custom expiration time.
+
+ Use this when your refresh function needs to specify its own expiration
+ rather than using the default max_age. The expiry can be specified as
+ either a TTL in seconds or an absolute timestamp.
+
+ Example usage with TTL::
+
+ def refresh_oauth_token():
+ token = fetch_token() # Returns {'access_token': '...', 'expires_in': 3600}
+ return ValueWithExpiry(token, ttl=token['expires_in'])
+
+ Example usage with absolute timestamp::
+
+ def refresh_oauth_token():
+ token = fetch_token() # Returns {'access_token': '...', 'expires_at': 1702915200}
+ return ValueWithExpiry(token, expires_at=token['expires_at'])
+ """
+
+ def __init__(self, value, *, ttl=None, expires_at=None):
+ if ttl is None and expires_at is None:
+ raise ValueError("Must specify either ttl or expires_at")
+ if ttl is not None and expires_at is not None:
+ raise ValueError("Cannot specify both ttl and expires_at")
+
+ self.value = value
+ self.expires_at = expires_at if expires_at else time.time() + ttl
+
+ def __repr__(self):
+ return f"ValueWithExpiry(value={self.value!r}, expires_at={self.expires_at})"
+
+
class AbstractExpiringValue
(refresh_fn, max_age)
@@ -302,11 +465,14 @@ Classes
Abstract base class to represent a value that expires.
An AbstractExpiringValue represents a lazily loaded value that will expire
over time. The constructor takes a refresh_fn function of zero arguments,
-which is called to obtain the value to be cached for max_age seconds.
+which is called to obtain the value to be cached. The value is cached for
+max_age seconds unless the refresh_fn returns a ValueWithExpiry object
+that specifies a custom expiration time. If so, the default max_age is
+ignored.
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
-method is called. This class is thread-safe. Subclasses must provide
+method is called. The value method is thread-safe. Subclasses must provide
implementations for is_expired, load, and save.
@@ -317,12 +483,15 @@ Classes
An `AbstractExpiringValue` represents a lazily loaded value that will expire
over time. The constructor takes a `refresh_fn` function of zero arguments,
- which is called to obtain the value to be cached for `max_age` seconds.
+ which is called to obtain the value to be cached. The value is cached for
+ `max_age` seconds unless the `refresh_fn` returns a `ValueWithExpiry` object
+ that specifies a custom expiration time. If so, the default `max_age` is
+ ignored.
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
- method is called. This class is thread-safe. Subclasses must provide
+ method is called. The `value` method is thread-safe. Subclasses must provide
implementations for `is_expired`, `load`, and `save`.
"""
@@ -339,19 +508,35 @@ Classes
invocations of this method will return the cached value until it
expires. If you set `refresh` parameter to `True`, the value will be
refreshed and the expiration will be reset before being returned.
+
+ This method is thread-safe.
"""
with self._lock:
if not refresh and not self.is_expired():
return self.load()
- value = self._refresh_fn()
- self.save(value)
+ result = self._refresh_fn()
+
+ # Unwrap and extract expiry if ValueWithExpiry
+ if isinstance(result, ValueWithExpiry):
+ value = result.value
+ expiry = result.expires_at
+ else:
+ value = result
+ expiry = None
+
+ self.save(value, expiry)
LOG.info("refreshed data and saved in cache")
return value
def is_expired(self):
"""Returns `True` if the value needs to be refreshed, `False` otherwise.
+ A value needs to be refreshed when it has expired. This is determined
+ by the implementation of this method. By default, the value is cached for
+ `max_age` seconds or the dynamic expiry specified by the `refresh_fn`
+ return value (if it is a `ValueWithExpiry` instance).
+
If this returns `True` during the invocation of
`AbstractExpiringValue.value`, the `refresh_fn` will be called, followed
by `save`, to renew the cached value. When this returns `False`, `load`
@@ -368,12 +553,14 @@ Classes
"""
raise NotImplementedError
- def save(self, value):
+ def save(self, value, expiry=None):
"""Saves the value to the cache.
If `is_expired` returns `True` during the invocation of
`AbstractExpiringValue.value`, this method is invoked to save the new
- value to the cache.
+ value to the cache. If `expiry` is provided, it specifies the absolute
+ timestamp when the value should expire. Otherwise, the default `max_age`
+ should be used.
"""
raise NotImplementedError
@@ -392,17 +579,74 @@ Methods
calling the refresh_fn supplied in the constructor. Subsequent
invocations of this method will return the cached value until it
expires. If you set refresh parameter to True, the value will be
-refreshed and the expiration will be reset before being returned.
+refreshed and the expiration will be reset before being returned.
+This method is thread-safe.
+
+
+Expand source code
+
+def value(self, refresh=False):
+ """Returns the value.
+
+ The first time this method is called, the value will be obtained by
+ calling the `refresh_fn` supplied in the constructor. Subsequent
+ invocations of this method will return the cached value until it
+ expires. If you set `refresh` parameter to `True`, the value will be
+ refreshed and the expiration will be reset before being returned.
+
+ This method is thread-safe.
+ """
+ with self._lock:
+ if not refresh and not self.is_expired():
+ return self.load()
+
+ result = self._refresh_fn()
+
+ # Unwrap and extract expiry if ValueWithExpiry
+ if isinstance(result, ValueWithExpiry):
+ value = result.value
+ expiry = result.expires_at
+ else:
+ value = result
+ expiry = None
+
+ self.save(value, expiry)
+ LOG.info("refreshed data and saved in cache")
+ return value
+
def is_expired(self)
Returns True if the value needs to be refreshed, False otherwise.
+A value needs to be refreshed when it has expired. This is determined
+by the implementation of this method. By default, the value is cached for
+max_age seconds or the dynamic expiry specified by the refresh_fn
+return value (if it is a ValueWithExpiry instance).
If this returns True during the invocation of
AbstractExpiringValue.value(), the refresh_fn will be called, followed
by save, to renew the cached value. When this returns False, load
is invoked instead to return the value from the cache.
+Expand source code
+
+def is_expired(self):
+ """Returns `True` if the value needs to be refreshed, `False` otherwise.
+
+ A value needs to be refreshed when it has expired. This is determined
+ by the implementation of this method. By default, the value is cached for
+ `max_age` seconds or the dynamic expiry specified by the `refresh_fn`
+ return value (if it is a `ValueWithExpiry` instance).
+
+ If this returns `True` during the invocation of
+ `AbstractExpiringValue.value`, the `refresh_fn` will be called, followed
+ by `save`, to renew the cached value. When this returns `False`, `load`
+ is invoked instead to return the value from the cache.
+ """
+ raise NotImplementedError
+
def load(self)
@@ -412,15 +656,45 @@ Methods
If is_expired returns False during the invocation of
AbstractExpiringValue.value(), this method is invoked to return the
value from the cache.
+
+
+Expand source code
+
+def load(self):
+ """Returns the value from the cache.
+
+ If `is_expired` returns `False` during the invocation of
+ `AbstractExpiringValue.value`, this method is invoked to return the
+ value from the cache.
+ """
+ raise NotImplementedError
+
-def save(self, value)
+def save(self, value, expiry=None)
Saves the value to the cache.
If is_expired returns True during the invocation of
AbstractExpiringValue.value(), this method is invoked to save the new
-value to the cache.
expiry is provided, it specifies the absolute
+timestamp when the value should expire. Otherwise, the default max_age
+should be used.
+
+
+Expand source code
+
+def save(self, value, expiry=None):
+ """Saves the value to the cache.
+
+ If `is_expired` returns `True` during the invocation of
+ `AbstractExpiringValue.value`, this method is invoked to save the new
+ value to the cache. If `expiry` is provided, it specifies the absolute
+ timestamp when the value should expire. Otherwise, the default `max_age`
+ should be used.
+ """
+ raise NotImplementedError
+
@@ -437,7 +711,7 @@ Methods
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
-method is called. This class is thread-safe.
+method is called. The value method is thread-safe.
Expand source code
@@ -453,7 +727,7 @@ Methods
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
- method is called. This class is thread-safe.
+ method is called. The `value` method is thread-safe.
"""
def __init__(self, refresh_fn, max_age):
@@ -468,10 +742,10 @@ Methods
LOG.debug("Loading data from cache")
return self._value
- def save(self, value):
- LOG.debug("Saving value to cache")
+ def save(self, value, expiry=None):
self._value = value
- self._expiry = time.time() + self._max_age
+ self._expiry = expiry if expiry is not None else time.time() + self._max_age
+ LOG.debug("Saving value to cache, will expire at %s", time.ctime(self._expiry))
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
-method is called. This class is thread-safe. If the value cannot be
+method is called. The value method is thread-safe. If the value cannot be
persisted as JSON, a TypeError is thrown.
ValueWithExpiryAbstractExpiringValuevalue
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/cli.html b/docs/cli.html
index 89b7031..2a358ea 100644
--- a/docs/cli.html
+++ b/docs/cli.html
@@ -3,13 +3,13 @@
-
+
awsrun.cli API documentation
-
+
@@ -100,7 +100,7 @@ CLI User Guide
For the remainder of this guide, we'll be using the general purpose
awsrun.commands.aws.aws command, which is an adapter for the AWS CLI tool.
This allows users to concurrently run the AWS CLI across multiple accounts. When
-combined with other awsrun features such as SAML authentication, cross-account
+combined with other awsrun features such as SAML/OIDC federation, cross-account
access, and metadata account filters, it makes the AWS CLI tool even more
powerful than it already is as you'll see.
Usage
@@ -156,7 +156,8 @@ Usage
default, awsrun will use the same profiles from your ~/.aws/credentials file. If
a profile does not exist for an account, awsrun will fallback to the "default"
profile if you have one configured. As you'll see later, we can use other
-plug-ins to obtain credentials for accounts via SAML and cross-account access.
+plug-ins to obtain credentials for accounts via SAML/OIDC federation and
+cross-account access.
Using awsrun to run a command over two accounts may not seem that interesting
yet, but when running the same AWS CLI command over more than a handful of
accounts, it becomes incredibly powerful especially when coupled with account
@@ -231,15 +232,15 @@
Credential Plug-ins
Up until this point, we have been using the default credential loader called
Profile, which looks for account profiles in your
$HOME/.aws/credentials or $HOME/.aws/config. Let's look at some of the other
-mechanisms included with awsrun such as the SAML and cross-account access
+mechanisms included with awsrun such as the SAML, OIDC, and cross-account access
plug-ins. If you are only seeking reference material, please refer to the
awsrun.plugins.creds page instead.
-Within many enterprises, the use of single sign-on (SSO) is prevalent and allows
-for centralized account management. AWS supports federated users and SSO via the
-use of a SAML-compliant Identity Provider (IdP). Rather than define IAM users in
-each account, an IAM role can be created that allows for federated access via
-your IdP. To use SAML-based access with AWS, you need to obtain temporary tokens
-from the AWS STS service.
+Within many enterprises, the use of federated access is prevalent and allows for
+centralized account management. AWS supports federated users and SSO via the use
+of a SAML- or OIDC-compliant Identity Provider (IdP). Rather than define IAM
+users in each account, an IAM role can be created that allows for federated
+access via your IdP. To use SAML- or OIDC-based access with AWS, you need to
+obtain temporary tokens from the AWS STS service.
The SAML credential plug-in simplifies this process
for you. At a minimum, you'll need the IAM role that has been setup for SAML as
well as the URL to your IdP. Let's assume you are using Microsoft's ADFS server
@@ -732,7 +733,7 @@
Troubleshooting
For the remainder of this guide, we'll be using the general purpose
`awsrun.commands.aws.aws` command, which is an adapter for the AWS CLI tool.
This allows users to concurrently run the AWS CLI across multiple accounts. When
-combined with other awsrun features such as SAML authentication, cross-account
+combined with other awsrun features such as SAML/OIDC federation, cross-account
access, and metadata account filters, it makes the AWS CLI tool even more
powerful than it already is as you'll see.
@@ -796,7 +797,8 @@ Troubleshooting
default, awsrun will use the same profiles from your ~/.aws/credentials file. If
a profile does not exist for an account, awsrun will fallback to the "default"
profile if you have one configured. As you'll see later, we can use other
-plug-ins to obtain credentials for accounts via SAML and cross-account access.
+plug-ins to obtain credentials for accounts via SAML/OIDC federation and
+cross-account access.
Using awsrun to run a command over two accounts may not seem that interesting
yet, but when running the same AWS CLI command over more than a handful of
@@ -885,16 +887,16 @@ Troubleshooting
Up until this point, we have been using the default credential loader called
`awsrun.plugins.creds.aws.Profile`, which looks for account profiles in your
`$HOME/.aws/credentials` or `$HOME/.aws/config`. Let's look at some of the other
-mechanisms included with awsrun such as the SAML and cross-account access
+mechanisms included with awsrun such as the SAML, OIDC, and cross-account access
plug-ins. If you are only seeking reference material, please refer to the
`awsrun.plugins.creds` page instead.
-Within many enterprises, the use of single sign-on (SSO) is prevalent and allows
-for centralized account management. AWS supports federated users and SSO via the
-use of a SAML-compliant Identity Provider (IdP). Rather than define IAM users in
-each account, an IAM role can be created that allows for federated access via
-your IdP. To use SAML-based access with AWS, you need to obtain temporary tokens
-from the AWS STS service.
+Within many enterprises, the use of federated access is prevalent and allows for
+centralized account management. AWS supports federated users and SSO via the use
+of a SAML- or OIDC-compliant Identity Provider (IdP). Rather than define IAM
+users in each account, an IAM role can be created that allows for federated
+access via your IdP. To use SAML- or OIDC-based access with AWS, you need to
+obtain temporary tokens from the AWS STS service.
The `awsrun.plugins.creds.aws.SAML` credential plug-in simplifies this process
for you. At a minimum, you'll need the IAM role that has been setup for SAML as
@@ -1476,7 +1478,7 @@ Troubleshooting
csp = _CSP.from_prog_name(sys.argv[0])
_cli(csp)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# Don't print stack traces by default as it can be overwhelming (scary)
# for those not familiar with Python development.
if os.getenv("AWSRUN_TRACE"):
@@ -1803,8 +1805,8 @@ Troubleshooting
return
print("The following are the available commands:\n", file=out)
- max_cmd_len = max(len(name) for name in commands.keys())
- for name in sorted(commands.keys()):
+ max_cmd_len = max(len(name) for name in commands)
+ for name in sorted(commands):
# By convention, as documented in user documentation, class docstring
# is used when printing a summary of commands.
docstring = commands[name].__doc__ or ""
@@ -1815,7 +1817,7 @@ Troubleshooting
def _print_accounts(accts, out=sys.stdout):
"""Print the list of accounts."""
count = len(accts)
- print(f'{count} account{"s" if count != 1 else ""} selected:\n', file=out)
+ print(f"{count} account{'s' if count != 1 else ''} selected:\n", file=out)
print(", ".join(str(a) for a in accts), file=out, end="\n\n")
@@ -1824,7 +1826,7 @@ Troubleshooting
_print_accounts(accts, out=sys.stderr)
print("Proceed (y/n)? ", flush=True, end="", file=sys.stderr)
answer = input()
- if not answer.lower() in ["y", "yes"]:
+ if answer.lower() not in ["y", "yes"]:
print("Exiting", file=sys.stderr)
sys.exit(0)
@@ -1922,6 +1924,37 @@ Functions
different names. The CLI uses the name of the shell script installed to
determine default path locations for commands as well as the default
session manager.
+
+
+Expand source code
+
+def main():
+ """The main entry point for the `*run` CLI tool installed with this package.
+
+ Runs the CLI tool. Exits with a `0` status code upon success. Upon error,
+ prints the error message to standard error. By default, a stack trace is not
+ included to minimize output. If the trace is desired, set the `AWSRUN_TRACE`
+ environment variable to `1`.
+
+ The CLI tool is installed on the system via the setup.py entry_points key.
+ There can be many instances of this command installed on the system with
+ different names. The CLI uses the name of the shell script installed to
+ determine default path locations for commands as well as the default
+ session manager.
+ """
+ try:
+ csp = _CSP.from_prog_name(sys.argv[0])
+ _cli(csp)
+
+ except Exception as e:
+ # Don't print stack traces by default as it can be overwhelming (scary)
+ # for those not familiar with Python development.
+ if os.getenv("AWSRUN_TRACE"):
+ traceback.print_exc(file=sys.stderr)
+
+ print(e, file=sys.stderr)
+ sys.exit(1)
+
@@ -2006,7 +2039,7 @@ Context
diff --git a/docs/cloudwatch.html b/docs/cloudwatch.html
index 4dbbe1d..48e3706 100644
--- a/docs/cloudwatch.html
+++ b/docs/cloudwatch.html
@@ -3,13 +3,13 @@
-
+
awsrun.cloudwatch API documentation
-
+
@@ -178,7 +178,7 @@ Module awsrun.cloudwatch
self._samples = samples
self._ingestion_interval = ingestion_interval
self._period = self._compute_period()
- self._beg = self._end = datetime.now()
+ self._beg = self._end = datetime.now(timezone.utc)
_LOG.info(
"CWMetrics(client, last=%d, samples=%d, ingestion_interval=%d)",
@@ -453,6 +453,28 @@ Functions
ascending chronological order. The generator will yield a value for each
time interval even if CloudWatch has missing data points. In that case,
the value in the tuple will be a math.nan.
+
+
+Expand source code
+
+def get_metric(client, namespace, name, dimensions, statistic, last=3600, samples=60):
+ """Fetch data for the specified CloudWatch metric immediately.
+
+ This is a convenience method that should only be used if retrieving a
+ single metric. If multiple metrics are desired, it is more efficient to
+ use `CWMetric` instead.
+
+ Returns a generator object that yields a tuple containing a datetime
+ object and a corresponding value at that time. Values are returned in
+ ascending chronological order. The generator will yield a value for each
+ time interval even if CloudWatch has missing data points. In that case,
+ the value in the tuple will be a `math.nan`.
+ """
+ cwm = CWMetrics(client, last, samples)
+ get_values = cwm.add_metric(namespace, name, dimensions, statistic)
+ cwm.bulk_load()
+ return get_values()
+
@@ -492,7 +514,7 @@ Classes
self._samples = samples
self._ingestion_interval = ingestion_interval
self._period = self._compute_period()
- self._beg = self._end = datetime.now()
+ self._beg = self._end = datetime.now(timezone.utc)
_LOG.info(
"CWMetrics(client, last=%d, samples=%d, ingestion_interval=%d)",
@@ -745,6 +767,68 @@ Methods
will yield a value for each time interval even if CloudWatch has
missing data points. In that case, the value in the tuple will be a
math.nan.
+
+
+Expand source code
+
+def add_metric(self, namespace, name, dimensions, statistic):
+ """Queue the specified CloudWatch metric for bulk loading.
+
+ Use this method to register a metric for future retrieval via
+ `CWMetric.bulk_load`. Up to 500 metrics can be added. Once a metric
+ has been added, subsequent calls to `CWMetric.bulk_load` will retrieve
+ it again.
+
+ Returns a function that can be invoked after a bulk load has
+ completed. It will return a generator object that yields a tuple
+ containing a datetime object and a corresponding value at that time.
+ Values are returned in ascending chronological order. The generator
+ will yield a value for each time interval even if CloudWatch has
+ missing data points. In that case, the value in the tuple will be a
+ `math.nan`.
+ """
+ # We need to keep track of how many metrics are being retrieved as AWS
+ # only permits up to 500 in a single get_metric_data call. We also
+ # need a unique ID for each metric added as that is how we will match
+ # the response from the CW API.
+ self._counter += 1
+ if self._counter > _MAX_METRICS:
+ raise ValueError(f"number of metrics exceeded {_MAX_METRICS}")
+
+ # The AWS get_metric_data call requires a unique ID for each metric
+ # being retrieved. This ID is provided with the results, so the caller
+ # is able to match request with response. The ID only needs to be
+ # unique per call to get_metric_data, so we just use a simple counter
+ # as we'll never expose this ID to callers.
+ metric_id = f"id{self._counter}"
+
+ # Convert a dict in form of {"connId": "dxcon-aaa"} to the form
+ # required by AWS: [{"Name": "connId", "Value": "dxcon-aaa"}].
+ dimensions = [{"Name": n, "Value": v} for n, v in dimensions.items()]
+
+ # Create the query dict and append to the list of queries. The query
+ # is not executed until later when the user calls bulk_load().
+ self._queries.append(
+ {
+ "Id": metric_id,
+ "MetricStat": {
+ "Metric": {
+ "Namespace": namespace,
+ "MetricName": name,
+ "Dimensions": dimensions,
+ },
+ "Period": int(self._period),
+ "Stat": statistic,
+ },
+ "ReturnData": True,
+ }
+ )
+
+ # Return a closure to make it easy to retrieve results. After the user
+ # has called bulk_load, this function can be executed to obtain the
+ # results of the API call.
+ return lambda: self._get_metric_generator(metric_id)
+
def bulk_load(self)
@@ -758,6 +842,55 @@ Methods
Returns nothing. Results for each metric can be obtained by invoking
the callable returned from CWMetric.add_metric, which will return
the data associated with the last bulk load.
+
+
+Expand source code
+
+def bulk_load(self):
+ """Retrieve the metrics that have been queued.
+
+ This method will make a single API call to AWS to request all of the
+ requested metrics. This method may be called one or more times. Each
+ call will retrieve the metrics that were requested replacing the
+ results of a prior invocation.
+
+ Returns nothing. Results for each metric can be obtained by invoking
+ the callable returned from `CWMetric.add_metric`, which will return
+ the data associated with the last bulk load.
+ """
+ if not self._queries:
+ return
+
+ self._results = defaultdict(_MetricResult)
+ self._beg, self._end = self._compute_datetime_range()
+
+ for page in self._client.get_paginator("get_metric_data").paginate(
+ MetricDataQueries=self._queries,
+ StartTime=self._beg.isoformat(),
+ EndTime=self._end.isoformat(),
+ ScanBy="TimestampAscending",
+ ):
+ for mdr in page["MetricDataResults"]:
+ count = len(mdr["Values"])
+ _LOG.info(
+ "Results for %s (%s): count=%d status=%s",
+ mdr["Label"],
+ mdr["Id"],
+ count,
+ mdr["StatusCode"],
+ )
+ if count > 1:
+ _LOG.info(" [0]: %s %.2f", mdr["Timestamps"][0], mdr["Values"][0])
+ _LOG.info(" [1]: %s %.2f", mdr["Timestamps"][1], mdr["Values"][1])
+ _LOG.info("[-2]: %s %.2f", mdr["Timestamps"][-2], mdr["Values"][-2])
+ _LOG.info("[-1]: %s %.2f", mdr["Timestamps"][-1], mdr["Values"][-1])
+
+ # We use extend here because results can span multiple pages,
+ # so we just keep extending to the existing lists.
+ result = self._results[mdr["Id"]]
+ result.values.extend(mdr["Values"])
+ result.timestamps.extend(mdr["Timestamps"])
+
@@ -795,7 +928,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/cmdmgr.html b/docs/cmdmgr.html
index 8028e84..eb90889 100644
--- a/docs/cmdmgr.html
+++ b/docs/cmdmgr.html
@@ -3,13 +3,13 @@
-
+
awsrun.cmdmgr API documentation
-
+
@@ -596,6 +596,14 @@ Methods
-
Returns a dict of names and classes of all valid commands.
+Expand source code
+
+def commands(self):
+ """Returns a dict of names and classes of all valid commands."""
+ return self._loader.load_all()
+
def instantiate_command(self, command_name, argv, cfg)
@@ -614,6 +622,52 @@ Methods
CLI options defined by the command. See the documentation on
Config.get() for the parameters of the function.
If command_name is not found, raises CommandNotFoundError.
+
+
+Expand source code
+
+def instantiate_command(self, command_name, argv, cfg):
+ """Returns an instantiated command identified by `command_name`.
+
+ The `argv` parameter should be a a string of command line arguments
+ captured by `argparse.REMAINDER` in the main program that will be passed
+ directly to the command for processing via its static `from_cli` method.
+ If the arguments are not valid, the program will terminate by the arg
+ parser in the command. This is expected as the command's `argparse` will
+ present a user-friendly help message.
+
+ The `cfg` parameter is a function that can lookup key value pairs from a
+ user's configuration file. This is provided to the command author via
+ `Command.from_cli`, so default values can be provided for the any of the
+ CLI options defined by the command. See the documentation on
+ `awsrun.config.Config.get` for the parameters of the function.
+
+ If `command_name` is not found, raises `CommandNotFoundError`.
+ """
+ # Dynamically load the command specified by the user. All commands are
+ # really defined as modules in the 'commands' directory of this package
+ # by default, but this can be overridden by the --cmd-dir flag.
+ cmd_class = self._loader.load(command_name)
+
+ if not issubclass(cmd_class, Command):
+ raise TypeError(
+ f"'{command_name}' must be a subclass of awsrun.runner.Command"
+ )
+
+ # Create an argument parser for the command author, which is populated
+ # with the name and a help string from the command's module docstring.
+ parser = argparse.ArgumentParser(
+ command_name,
+ formatter_class=RawAndDefaultsFormatter,
+ epilog=sys.modules[cmd_class.__module__].__doc__,
+ )
+
+ # We then call the static method on the class to obtain an instance of
+ # the command. The command author is expected to parse whatever command
+ # line args the user passed on the command line. It is expected that the
+ # author terminate the program if incorrect arguments were passed.
+ return cmd_class.from_cli(parser, argv, cfg)
+
@@ -662,6 +716,17 @@ Methods
-
Returns the class object for the command called command_name.
If a valid class cannot be found, raises CommandNotFoundError.
+Expand source code
+
+def load(self, command_name):
+ """Returns the class object for the command called `command_name`.
+
+ If a valid class cannot be found, raises `CommandNotFoundError`.
+ """
+ raise NotImplementedError
+
def load_all(self)
@@ -670,6 +735,18 @@ Methods
Returns a dict of all valid commands found.
The keys of the dict are the command names and the values are the class
objects that have been loaded.
+Expand source code
+
+def load_all(self):
+ """Returns a dict of all valid commands found.
+
+ The keys of the dict are the command names and the values are the class
+ objects that have been loaded.
+ """
+ raise NotImplementedError
+
@@ -745,6 +822,26 @@ Methods
All loaders are searched for the command. If a command is found in
multiple loaders, the first loader containing the command is preferred.
If a valid class cannot be found, raises CommandNotFoundError.
+
+
+Expand source code
+
+def load(self, command_name):
+ """Returns the class object for the command called `command_name`.
+
+ All loaders are searched for the command. If a command is found in
+ multiple loaders, the first loader containing the command is preferred.
+ If a valid class cannot be found, raises `CommandNotFoundError`.
+ """
+ path_errors = {}
+ for loader in self.loaders:
+ try:
+ return loader.load(command_name)
+ except CommandNotFoundError as e:
+ path_errors.update(e.path_errors)
+
+ raise CommandNotFoundError(command_name, path_errors)
+
def load_all(self)
@@ -755,6 +852,26 @@ Methods
objects that have been loaded.
If a command is found in multiple
loaders, the first loader containing the command is preferred.
+
+
+Expand source code
+
+def load_all(self):
+ """Returns a dict of all valid commands found from all loaders.
+
+ The keys of the dict are the command names and the values are the class
+ objects that have been loaded. If a command is found in multiple
+ loaders, the first loader containing the command is preferred.
+ """
+ classes = {}
+ for loader in reversed(self.loaders):
+ # dict.update() replaces existing keys with new values, so reversing
+ # the list ensures that loaders at the beginning of the list take
+ # priority over those that come afterwards.
+ classes.update(loader.load_all())
+
+ return classes
+
@@ -995,7 +1112,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/access_report.html b/docs/commands/aws/access_report.html
index 2c0ec9d..c48d2e9 100644
--- a/docs/commands/aws/access_report.html
+++ b/docs/commands/aws/access_report.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.access_report API documentation
-
+
@@ -210,7 +210,7 @@ Command Options
self.total += 1
try:
result = get_result()
- except Exception: # pylint: disable=broad-except
+ except Exception:
self.no_access.append(acct)
return
@@ -278,7 +278,7 @@ Classes
self.total += 1
try:
result = get_result()
- except Exception: # pylint: disable=broad-except
+ except Exception:
self.no_access.append(acct)
return
@@ -347,7 +347,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/aws.html b/docs/commands/aws/aws.html
index 6368faf..9211275 100644
--- a/docs/commands/aws/aws.html
+++ b/docs/commands/aws/aws.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.aws API documentation
-
+
@@ -921,9 +921,8 @@ Tips on Parsing JSON Output
cmd,
env=env,
check=False,
- universal_newlines=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ text=True,
+ capture_output=True,
)
# Lastly, we return the ProcessCompleted object from the run() method.
@@ -957,7 +956,7 @@ Tips on Parsing JSON Output
# ProcessCompleted object from the subprocess.run() method above ...
result = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# ... unless there was an exception in which case it is raised by
# the call to get_result and we handle it here.
LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
@@ -1132,9 +1131,8 @@ Classes
cmd,
env=env,
check=False,
- universal_newlines=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ text=True,
+ capture_output=True,
)
# Lastly, we return the ProcessCompleted object from the run() method.
@@ -1168,7 +1166,7 @@ Classes
# ProcessCompleted object from the subprocess.run() method above ...
result = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# ... unless there was an exception in which case it is raised by
# the call to get_result and we handle it here.
LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
@@ -1224,12 +1222,136 @@ Methods
-
Invoke an AWS CLI command for an account and region.
+Expand source code
+
+def regional_execute(self, session, acct, region):
+ """Invoke an AWS CLI command for an account and region."""
+
+ # We need to assemble a valid AWS cli command line that can be executed
+ # by the operating system. The instance variable awscli_args contains
+ # all arguments that follow 'aws': awsrun -r us-east-1 aws ... We will
+ # provide the --region argument as we know the region we are processing.
+ # We will also provide --output if the user has asked us to annotate an
+ # output type. This ensures we override any user settings that the AWS
+ # cli tool may pick up from ~/.aws/config.
+ cmd = [self.awscli_path, "--region", region]
+ if self.annotate:
+ cmd += ["--output", self.annotate]
+ elif self.output:
+ cmd += ["--output", self.output]
+ cmd += self.awscli_args
+ LOG.info("%s-%s: AWS CLI command: %s", acct, region, cmd)
+
+ # Before we can execute the AWS cli tool, we need to set a few env vars
+ # with our creds. Normally, the AWS cli expects this file to exist in
+ # the user's home directory at ~/.aws/credentials.
+ creds = session.get_credentials()
+
+ new_vars = {
+ "AWS_ACCESS_KEY_ID": creds.access_key,
+ "AWS_SECRET_ACCESS_KEY": creds.secret_key,
+ "AWS_SESSION_TOKEN": creds.token if creds.token else "",
+ }
+
+ env = ChainMap(new_vars, os.environ)
+
+ # We call run() and capture stdout and stderr from the command's
+ # output. Note: all the output is stored in memory, and then printed
+ # in collect_results. This means that if you run an AWS cli command
+ # that generates huge amounts of data, it'll all be stored in
+ # memory. Why don't we stream tho output from a pipe? We could use
+ # Popen directly, but if we returned from execute() before reading
+ # all of the results, then the worker will start another account, so
+ # in essence, all of the accounts will be "executed" immediately
+ # resulting in potentially many many AWS cli command processes
+ # running waiting for us to read the output.
+
+ result = subprocess.run(
+ cmd,
+ env=env,
+ check=False,
+ text=True,
+ capture_output=True,
+ )
+
+ # Lastly, we return the ProcessCompleted object from the run() method.
+ # Recall, an awsrun command can return anything if you provide your own
+ # collect_results method.
+ return result
+
def regional_collect_results(self, acct, region, get_result)
-
Print the results to the console and files if specified.
+Expand source code
+
+def regional_collect_results(self, acct, region, get_result):
+ """Print the results to the console and files if specified."""
+
+ def annotate_lines(text, file=sys.stdout):
+ for line in filter(None, text.split("\n")):
+ print(f"{acct}/{region}: {line}", file=file, flush=True)
+
+ def annotate_json(text):
+ try:
+ d = {
+ "Account": str(acct),
+ "Region": region,
+ "Results": json.loads(text),
+ }
+ json.dump(d, sys.stdout, indent=4)
+ print()
+ except json.decoder.JSONDecodeError:
+ annotate_lines(
+ "Result of AWS CLI command is not valid JSON", file=sys.stderr
+ )
+
+ try:
+ # Let's get the return value from the execute method, which is the
+ # ProcessCompleted object from the subprocess.run() method above ...
+ result = get_result()
+
+ except Exception as e:
+ # ... unless there was an exception in which case it is raised by
+ # the call to get_result and we handle it here.
+ LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
+ annotate_lines(f"error: {e}", file=sys.stderr)
+ return
+
+ # Print stderr from AWS CLI always annotating the lines
+ annotate_lines(result.stderr, file=sys.stderr)
+
+ # Print stdout from AWS CLI annotating when appropriate
+ if not self.annotate:
+ print(result.stdout, end="", flush=True)
+ elif self.annotate == "json":
+ annotate_json(result.stdout)
+ elif self.annotate in ["text", "table"]:
+ annotate_lines(result.stdout)
+
+ # Save stdout and stderr from AWS CLI to disk if requested
+ if self.output_dir:
+ # Recall, the acct object passed to execute() can be anything. The
+ # str() method should provide us a unique means of identifying the
+ # account, but we need to escape any slashes if we use this as part
+ # of a filename so pathlib doesn't interpret as directories.
+ escaped = re.sub(r"[\\/]", "_", str(acct))
+ name = self.output_dir / f"{escaped}-{region}"
+
+ def save(suffix, text):
+ with name.with_suffix(suffix).open("w") as out:
+ out.write(text)
+
+ save(".stdout.log", result.stdout)
+ if result.stderr:
+ save(".stderr.log", result.stderr)
+
Inherited members
@@ -1292,7 +1414,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/cidr_overlap.html b/docs/commands/aws/cidr_overlap.html
index 499c74d..57790cc 100644
--- a/docs/commands/aws/cidr_overlap.html
+++ b/docs/commands/aws/cidr_overlap.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.cidr_overlap API documentation
-
+
@@ -248,7 +248,7 @@ Command Options
def post_hook(self):
overlap = []
- for i in range(0, len(self.cidrs)): # pylint: disable=consider-using-enumerate
+ for i in range(len(self.cidrs)):
c1 = self.cidrs[i]
for j in range(i + 1, len(self.cidrs)):
c2 = self.cidrs[j]
@@ -319,7 +319,7 @@ Classes
def post_hook(self):
overlap = []
- for i in range(0, len(self.cidrs)): # pylint: disable=consider-using-enumerate
+ for i in range(len(self.cidrs)):
c1 = self.cidrs[i]
for j in range(i + 1, len(self.cidrs)):
c2 = self.cidrs[j]
@@ -384,7 +384,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/console.html b/docs/commands/aws/console.html
index a13fa78..98ed353 100644
--- a/docs/commands/aws/console.html
+++ b/docs/commands/aws/console.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.console API documentation
-
+
@@ -362,7 +362,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/dx_maint.html b/docs/commands/aws/dx_maint.html
index fa8ea44..e2bc748 100644
--- a/docs/commands/aws/dx_maint.html
+++ b/docs/commands/aws/dx_maint.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.dx_maint API documentation
-
+
@@ -79,16 +79,17 @@ Module awsrun.commands.aws.dx_maint
Display Direct Connect maintenance events from AWS Health.
Overview
The dx_maint command displays a summary of Direct Connect maintenance events
-available from the AWS Health service sorted by the time AWS published the
-event, which is not necessarily the start of maintenance.
-Only recent events
-(past 7 days) and upcoming events are displayed by default.
+available from the AWS Health service sorted by the start time of the event.
+Only recent events (past 7 days) and upcoming events are displayed by default.
+The output includes the time the event was published (first date column), the
+start time of the event (second date column), and the end time of the event
+(last date column).
$ awsrun --account 111222333444 --account 222111444333 dx_maint --region us-east-1 --region us-west-2
111222333444 us-east-1 2021-11-19 09:11 CST closed MAINTENANCE_CANCELLED 2021-11-23 02:11 CST -> 2021-11-23 06:11 CST dxcon-aaaaaaaa available
111222333444 us-west-2 2021-12-07 05:12 CST closed MAINTENANCE_COMPLETE 2021-12-07 03:12 CST -> 2021-12-07 07:12 CST dxcon-bbbbbbbb available
111222333444 us-east-1 2022-01-18 02:01 CST closed MAINTENANCE_COMPLETE 2022-01-17 22:01 CST -> 2022-01-18 02:01 CST dxcon-cccccccc available
-222111444333 us-east-1 2022-01-31 21:01 CST upcoming MAINTENANCE_SCHEDULED 2022-02-14 22:02 CST -> 2022-02-15 02:02 CST dxcon-dddddddd available
111222333444 us-east-1 2022-02-01 21:02 CST upcoming MAINTENANCE_SCHEDULED 2022-02-08 22:02 CST -> 2022-02-09 02:02 CST dxcon-eeeeeeee available
+222111444333 us-east-1 2022-01-31 21:01 CST upcoming MAINTENANCE_SCHEDULED 2022-02-14 22:02 CST -> 2022-02-15 02:02 CST dxcon-dddddddd available
For non-emergency maintenance events that occurred in the past, the default
output only shows the COMPLETED or CANCELLED event to minimize on noise. For
@@ -156,16 +157,18 @@
Command Options
## Overview
The dx_maint command displays a summary of Direct Connect maintenance events
-available from the AWS Health service sorted by the time AWS published the
-event, which is not necessarily the start of maintenance. Only recent events
-(past 7 days) and upcoming events are displayed by default.
+available from the AWS Health service sorted by the start time of the event.
+Only recent events (past 7 days) and upcoming events are displayed by default.
+The output includes the time the event was published (first date column), the
+start time of the event (second date column), and the end time of the event
+(last date column).
$ awsrun --account 111222333444 --account 222111444333 dx_maint --region us-east-1 --region us-west-2
111222333444 us-east-1 2021-11-19 09:11 CST closed MAINTENANCE_CANCELLED 2021-11-23 02:11 CST -> 2021-11-23 06:11 CST dxcon-aaaaaaaa available
111222333444 us-west-2 2021-12-07 05:12 CST closed MAINTENANCE_COMPLETE 2021-12-07 03:12 CST -> 2021-12-07 07:12 CST dxcon-bbbbbbbb available
111222333444 us-east-1 2022-01-18 02:01 CST closed MAINTENANCE_COMPLETE 2022-01-17 22:01 CST -> 2022-01-18 02:01 CST dxcon-cccccccc available
- 222111444333 us-east-1 2022-01-31 21:01 CST upcoming MAINTENANCE_SCHEDULED 2022-02-14 22:02 CST -> 2022-02-15 02:02 CST dxcon-dddddddd available
111222333444 us-east-1 2022-02-01 21:02 CST upcoming MAINTENANCE_SCHEDULED 2022-02-08 22:02 CST -> 2022-02-09 02:02 CST dxcon-eeeeeeee available
+ 222111444333 us-east-1 2022-01-31 21:01 CST upcoming MAINTENANCE_SCHEDULED 2022-02-14 22:02 CST -> 2022-02-15 02:02 CST dxcon-dddddddd available
For non-emergency maintenance events that occurred in the past, the default
output only shows the COMPLETED or CANCELLED event to minimize on noise. For
@@ -409,19 +412,22 @@ Command Options
# in this method, but not the execute method above.
self.all_results.extend(get_result())
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s/%s: error: %s", acct, region, e, exc_info=True)
print(f"{acct}/{region}: error: {e}", flush=True, file=sys.stderr)
def post_hook(self):
# The awsrun post hook is called once after all accounts have been
# processed. At this point, we have collected everything we need in the
- # all_results instance variable. We sort this list by the last update
- # time as we want to display the maintenance events in the order they
- # were published by AWS. You might ask why don't we sort by the
- # maintenance start time? Because AWS sometimes does not publish a
- # start time for an event (I have no idea why).
- self.all_results.sort(key=lambda r: r[3]["lastUpdatedTime"])
+ # all_results instance variable. We sort this list by start time when
+ # possible, but the AWS API doesn't always send a startTime, so we
+ # fallback to the lastUpdatedTime, which seems to always be present.
+ self.all_results.sort(
+ key=lambda r: (
+ r[3].get("startTime", r[3]["lastUpdatedTime"]),
+ r[3]["lastUpdatedTime"],
+ )
+ )
for acct, region, dx, event in self.all_results:
for color, field in (
@@ -489,24 +495,64 @@ Functions
>>> list(chunk(l, 2))
[(1, 2), (3, 4), (5, 6), (7, 8), (9,)]
+
+
+Expand source code
+
+def chunk(it, n):
+ """Yield successive n-sized chunks from it.
+
+ >>> l = [1, 2, 3, 4, 5, 6, 7, 8, 9]
+ >>> list(chunk(l, 3))
+ [(1, 2, 3), (4, 5, 6), (7, 8, 9)]
+ >>> list(chunk(l, 2))
+ [(1, 2), (3, 4), (5, 6), (7, 8), (9,)]
+ """
+ it = iter(it)
+ return iter(lambda: tuple(itertools.islice(it, n)), ())
+
def colorize(color, string)
-
Return a string wrapped with ASCII color.
+Expand source code
+
+def colorize(color, string):
+ """Return a string wrapped with ASCII color."""
+ return f"{color}{string}{Style.RESET_ALL}"
+
def date(date_time)
-
Return a human readable string for a datetime object.
+Expand source code
+
+def date(date_time):
+ """Return a human readable string for a datetime object."""
+ return date_time.strftime("%Y-%m-%d %H:%m %Z") if date_time else "not provided"
+
def shorten(event_type)
-
Trim 'AWS_DIRECTCONNECT_' from event_type name.
+Expand source code
+
+def shorten(event_type):
+ """Trim 'AWS_DIRECTCONNECT_' from event_type name."""
+ return re.sub("^AWS_DIRECTCONNECT_", "", event_type)
+
@@ -677,19 +723,22 @@ Classes
# in this method, but not the execute method above.
self.all_results.extend(get_result())
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s/%s: error: %s", acct, region, e, exc_info=True)
print(f"{acct}/{region}: error: {e}", flush=True, file=sys.stderr)
def post_hook(self):
# The awsrun post hook is called once after all accounts have been
# processed. At this point, we have collected everything we need in the
- # all_results instance variable. We sort this list by the last update
- # time as we want to display the maintenance events in the order they
- # were published by AWS. You might ask why don't we sort by the
- # maintenance start time? Because AWS sometimes does not publish a
- # start time for an event (I have no idea why).
- self.all_results.sort(key=lambda r: r[3]["lastUpdatedTime"])
+ # all_results instance variable. We sort this list by start time when
+ # possible, but the AWS API doesn't always send a startTime, so we
+ # fallback to the lastUpdatedTime, which seems to always be present.
+ self.all_results.sort(
+ key=lambda r: (
+ r[3].get("startTime", r[3]["lastUpdatedTime"]),
+ r[3]["lastUpdatedTime"],
+ )
+ )
for acct, region, dx, event in self.all_results:
for color, field in (
@@ -773,7 +822,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/dx_status.html b/docs/commands/aws/dx_status.html
index 83166cb..2cb58dd 100644
--- a/docs/commands/aws/dx_status.html
+++ b/docs/commands/aws/dx_status.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.dx_status API documentation
-
+
@@ -389,6 +389,7 @@ Command Options
"""
+
import io
import re
import sys
@@ -641,14 +642,14 @@ Command Options
Style.RESET_ALL,
Style.BRIGHT,
Fore.YELLOW,
- f"{c_name:22.22} ",
+ f"{c_name:25.25} ",
Style.RESET_ALL,
(Fore.GREEN if c_state == "AVAILABLE" else Fore.RED),
f"{c_state:10} ",
Fore.BLUE,
- f"{c_bandwidth:7} ",
+ f"{c_bandwidth:>9} ",
Fore.MAGENTA,
- f"{len(vifs):3} VIFs ",
+ f"{len(vifs):5} VIFs ",
(Fore.RED + f"({v_down} down)" if v_down > 0 else ""),
Fore.RESET,
]
@@ -722,6 +723,18 @@ Functions
-
Return 1 if n is 0, 0 if n is 1, otherwise n.
+Expand source code
+
+def invert(n):
+ """Return 1 if n is 0, 0 if n is 1, otherwise n."""
+ if n == 0:
+ return 1
+ if n == 1:
+ return 0
+ return n # NaN case
+
def bps(bandwidth)
@@ -731,6 +744,36 @@ Functions
The supported units include: Gbps, Mbps, Kbps, and bps (all case
insenstive). If the bandwidth string does not have a numeric component or
uses an unsuppored unit, a ValueError is raised.
+
+
+Expand source code
+
+def bps(bandwidth):
+ """Convert the bandwidth string to an integer in bits per second.
+
+ The supported units include: Gbps, Mbps, Kbps, and bps (all case
+ insenstive). If the bandwidth string does not have a numeric component or
+ uses an unsuppored unit, a ValueError is raised.
+ """
+ match = re.match(r"(\d+)\s*(\w+)", bandwidth)
+ if not match:
+ raise ValueError("bandwidth not in form of 10Mbps")
+
+ bw, units = match.groups()
+ bw = int(bw)
+ units = units.lower()
+
+ if units == "gbps":
+ return bw * 1000000000
+ if units == "mbps":
+ return bw * 1000000
+ if units == "kbps":
+ return bw * 1000
+ if units == "bps":
+ return bw
+
+ raise ValueError("unsupported unit, must be Gbps, Mbps, Kbps, Bps")
+
@@ -1001,7 +1044,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/index.html b/docs/commands/aws/index.html
index 95b300a..ed3fd9b 100644
--- a/docs/commands/aws/index.html
+++ b/docs/commands/aws/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws API documentation
-
+
@@ -217,7 +217,7 @@ Index
diff --git a/docs/commands/aws/kubectl.html b/docs/commands/aws/kubectl.html
index 5473509..eb7b039 100644
--- a/docs/commands/aws/kubectl.html
+++ b/docs/commands/aws/kubectl.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.kubectl API documentation
-
+
@@ -470,12 +470,15 @@ Module awsrun.commands.aws.kubectl
parser.error("Do not specify --{arg} with awsrun kubectl")
delattr(args, attr)
- if args.awsrun_annotate and args.output:
- if args.awsrun_annotate != "text":
- if args.awsrun_annotate != args.output:
- parser.error(
- "When specifying --awsrun-annotate, you do not need the --output flag"
- )
+ if (
+ args.awsrun_annotate
+ and args.output
+ and args.awsrun_annotate != "text"
+ and args.awsrun_annotate != args.output
+ ):
+ parser.error(
+ "When specifying --awsrun-annotate, you do not need the --output flag"
+ )
return cls(**vars(args))
@@ -576,9 +579,8 @@ Module awsrun.commands.aws.kubectl
cmd,
env=os.environ,
check=False,
- universal_newlines=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ text=True,
+ capture_output=True,
)
results.append(_Result(result.stdout, result.stderr, name, namespace))
@@ -605,7 +607,7 @@ Module awsrun.commands.aws.kubectl
d["Results"] = loader(result.stdout)
dumper(d, sys.stdout, indent=4)
print()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
annotate_lines(result, f"cannot parse output: {e}", file=sys.stderr)
try:
@@ -613,7 +615,7 @@ Module awsrun.commands.aws.kubectl
# ProcessCompleted object from the subprocess.run() method above ...
results = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# ... unless there was an exception in which case it is raised by
# the call to get_result and we handle it here.
LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
@@ -694,7 +696,8 @@ Module awsrun.commands.aws.kubectl
def _save_output(name, text):
- with name.open("w") as out:
+ fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+ with os.fdopen(fd, "w") as out:
out.write(text)
@@ -713,7 +716,7 @@ Module awsrun.commands.aws.kubectl
}
kubedir = Path.home() / Path(".kube")
- kubedir.mkdir(parents=True, exist_ok=True)
+ kubedir.mkdir(mode=0o700, parents=True, exist_ok=True)
filename = kubedir / Path(f"awsrun-{account_id}-{region}-{name}-{namespace}")
_save_output(filename, _KUBECONFIG.format(**substitions))
@@ -721,11 +724,11 @@ Module awsrun.commands.aws.kubectl
def _list_clusters(eks):
- clusters = []
- for page in eks.get_paginator("list_clusters").paginate():
- for name in page["clusters"]:
- clusters.append(name)
- return clusters
+ return [
+ name
+ for page in eks.get_paginator("list_clusters").paginate()
+ for name in page["clusters"]
+ ]
@@ -835,12 +838,15 @@ Classes
parser.error("Do not specify --{arg} with awsrun kubectl")
delattr(args, attr)
- if args.awsrun_annotate and args.output:
- if args.awsrun_annotate != "text":
- if args.awsrun_annotate != args.output:
- parser.error(
- "When specifying --awsrun-annotate, you do not need the --output flag"
- )
+ if (
+ args.awsrun_annotate
+ and args.output
+ and args.awsrun_annotate != "text"
+ and args.awsrun_annotate != args.output
+ ):
+ parser.error(
+ "When specifying --awsrun-annotate, you do not need the --output flag"
+ )
return cls(**vars(args))
@@ -941,9 +947,8 @@ Classes
cmd,
env=os.environ,
check=False,
- universal_newlines=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ text=True,
+ capture_output=True,
)
results.append(_Result(result.stdout, result.stderr, name, namespace))
@@ -970,7 +975,7 @@ Classes
d["Results"] = loader(result.stdout)
dumper(d, sys.stdout, indent=4)
print()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
annotate_lines(result, f"cannot parse output: {e}", file=sys.stderr)
try:
@@ -978,7 +983,7 @@ Classes
# ProcessCompleted object from the subprocess.run() method above ...
results = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# ... unless there was an exception in which case it is raised by
# the call to get_result and we handle it here.
LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
@@ -1027,6 +1032,76 @@ Methods
-
Print the results to the console and files if specified.
+Expand source code
+
+def regional_collect_results(self, acct, region, get_result):
+ """Print the results to the console and files if specified."""
+
+ def annotate_lines(result, text, file=sys.stderr):
+ prefix = f"{acct}/{region}"
+ if result.cluster and result.namespace:
+ prefix += f"/{result.cluster}/{result.namespace}"
+ for line in filter(None, text.split("\n") if text else ""):
+ print(f"{prefix}: {line}", file=file, flush=True)
+
+ def annotate_format(result, loader, dumper):
+ try:
+ d = {}
+ d["Account"] = str(acct)
+ d["Region"] = region
+ if result.cluster and result.namespace:
+ d["Cluster"] = result.cluster
+ d["Namespace"] = result.namespace
+ d["Results"] = loader(result.stdout)
+ dumper(d, sys.stdout, indent=4)
+ print()
+ except Exception as e:
+ annotate_lines(result, f"cannot parse output: {e}", file=sys.stderr)
+
+ try:
+ # Let's get the return value from the execute method, which is the
+ # ProcessCompleted object from the subprocess.run() method above ...
+ results = get_result()
+
+ except Exception as e:
+ # ... unless there was an exception in which case it is raised by
+ # the call to get_result and we handle it here.
+ LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
+ print(f"{acct}/{region}: error: {e}", file=sys.stderr)
+ return
+
+ for result in results:
+ # Print stderr from AWS CLI always annotating the lines
+ annotate_lines(result, result.stderr, file=sys.stderr)
+
+ # Print stdout from AWS CLI annotating when appropriate
+ if not self.annotate:
+ print(result.stdout, end="", flush=True)
+ elif self.annotate == "json":
+ annotate_format(result, json.loads, json.dump)
+ elif self.annotate == "yaml":
+ annotate_format(result, yaml.safe_load, yaml.dump)
+ elif self.annotate == "text":
+ annotate_lines(result, result.stdout)
+
+ # Save stdout and stderr from kubectl to disk if requested
+ if self.output_dir and result.cluster and result.namespace:
+ # Recall, the acct object passed to execute() can be anything. The
+ # str() method should provide us a unique means of identifying the
+ # account, but we need to escape any slashes if we use this as part
+ # of a filename so pathlib doesn't interpret as directories.
+ escaped = re.sub(r"[\\/]", "_", str(acct))
+ name = (
+ self.output_dir
+ / f"{escaped}-{region}-{result.cluster}-{result.namespace}"
+ )
+
+ _save_output(name.with_suffix(".stdout.log"), result.stdout)
+ if result.stderr:
+ _save_output(name.with_suffix(".stderr.log"), result.stderr)
+
Inherited members
@@ -1073,7 +1148,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/last.html b/docs/commands/aws/last.html
index 65feb5c..d3a0e55 100644
--- a/docs/commands/aws/last.html
+++ b/docs/commands/aws/last.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.last API documentation
-
+
@@ -339,7 +339,7 @@ Command Options
from datetime import datetime, timedelta, timezone
from functools import partial
from itertools import chain, cycle
-from typing import Optional
+from typing import ClassVar, Optional
from awsrun.argparse import AppendAttributeValuePair
from awsrun.config import Bool, Dict, Int, List, Str
@@ -1127,7 +1127,7 @@ Command Options
"""
TITLE = "CloudTrail Viewer"
- BINDINGS = [
+ BINDINGS: ClassVar[list] = [
("q", "quit", "Quit"),
("c", "copy", "Copy"),
("e", "export_popup", "Export"),
@@ -1140,7 +1140,7 @@ Command Options
total_count = reactive(0)
filtered_count = reactive(0)
- theme = {
+ theme: ClassVar[dict] = {
"dark": Syntax.get_theme("dracula"),
"light": Syntax.get_theme("friendly"),
}
@@ -1639,30 +1639,72 @@ Methods
def all(self):
+ return self.events
def users(self)
def users(self):
+ return self.events_by_user.keys()
def by_user(self, user)
def by_user(self, user):
+ return self.events_by_user.get(user, [])
def by_id(self, event_id)
def by_id(self, event_id) -> Optional[_UserIdentityType]:
+ return self.events_by_key.get(event_id)
def filter(self, expr)
def filter(self, expr):
+ if not expr: # empty string?
+ events = self.unfiltered_events
+ else:
+ events = _filter_events(self.unfiltered_events, expr)
+
+ self.filter_expr = expr
+ self._load_events(events)
+ return len(self.events)var DEFAULT_CSSThe type of the None singleton.
var Changedvar can_focusThe type of the None singleton.
var can_focus_childrenThe type of the None singleton.
def compose(self) -> ComposeResult:
+ yield Static(Markdown(self.help_md))
+ with Horizontal():
+ yield Input(placeholder=self.prompt)
+ yield Button("Close", variant="primary")
def on_input_submitted(self, event)
def on_input_submitted(self, event: Input.Submitted) -> None:
+ event.stop()
+ self.post_message(self.Changed(event.value))
def on_button_pressed(self, event)
def on_button_pressed(self, event: Button.Pressed) -> None:
+ event.stop()
+ self.post_message(self.Closed())textual.dom.DOMNode textual.message_pump.MessagePump var can_focusvar can_focus_childrenPopup:
- textual.dom.DOMNode
- textual.message_pump.MessagePump
var can_focusvar can_focus_childrenPopup:
var DEFAULT_CSSThe type of the None singleton.
var SelectionChangedvar can_focusThe type of the None singleton.
var can_focus_childrenThe type of the None singleton.
def on_data_table_row_highlighted(self, event: DataTable.RowHighlighted) -> None:
+ event.stop()
+ event.control.border_subtitle = (
+ f"{event.cursor_row + 1} of {len(event.control.rows)}"
+ )
+ self.post_message(self.SelectionChanged(event.row_key))
def compose(self)
@@ -2123,6 +2186,13 @@ Example
)
yield Footer()
+def compose(self) -> ComposeResult:
+ yield DataTable()
def focus(self)
@@ -2136,12 +2206,28 @@ Args
Returns
The Widget instance.
+
+
+Expand source code
+
+def focus(self):
+ self.query_one(DataTable).focus()
+
def on_mount(self)
-
+
+
+Expand source code
+
+def on_mount(self):
+ dt = self.query_one(DataTable)
+ dt.cursor_type = "row"
+ dt.add_columns(*self.col_names)
+
@@ -2203,21 +2289,6 @@ Ancestors
- textual.dom.DOMNode
- textual.message_pump.MessagePump
var DEFAULT_CSSvar can_focusvar can_focus_children
@@ -2225,19 +2296,43 @@ Methods
def on_mount(self):
+ dt = self.query_one(DataTable)
+ dt.show_header = False
+ dt.border_title = "Users"
def watch_contents(self, users)
def watch_contents(self, users):
+ dt = self.query_one(DataTable)
+ dt.clear()
+ for user in users:
+ dt.add_row(user, key=user)
+ dt.scroll_home()
+ dt.border_subtitle = f"0 of {len(users)}"RowTable:
DEFAULT_CSSSelectionChangedcan_focuscan_focus_childrencomposecontentsfocus- textual.dom.DOMNode
- textual.message_pump.MessagePump
var DEFAULT_CSSvar can_focusvar can_focus_children
@@ -2327,19 +2407,45 @@ Methods
def on_mount(self):
+ self.query_one(DataTable).border_title = "Events"
def watch_contents(self, events)
def watch_contents(self, events):
+ dt = self.query_one(DataTable)
+ dt.clear()
+ for event in events:
+ row = event.to_row()
+ if event.has_error():
+ error = dt.app.get_css_variables()["error"]
+ row = [f"[{error}]{c}[/]" for c in row]
+ dt.add_row(*row, key=event.event_id())
+ dt.scroll_home()
+ dt.border_subtitle = f"0 of {len(events)}"RowTable:
DEFAULT_CSSSelectionChangedcan_focuscan_focus_childrencomposecontentsfocusvar CSSThe type of the None singleton.
var TITLEThe type of the None singleton.
var BINDINGSThe type of the None singleton.
var themeThe type of the None singleton.
Yield child widgets for a container.
This method should be implemented in a subclass.
def compose(self) -> ComposeResult:
+ yield FilterPopup() # Hidden off-screen initially
+ yield ExportPopup() # Hidden off-screen initially
+ yield Header()
+ yield Horizontal(
+ Vertical(UserTable(), EventTable(), id="left"),
+ EventDetail(),
+ id="main",
+ )
+ yield Footer()
def on_mount(self)
def on_mount(self):
+ self.query_one(Header).tall = True
+ self.query_one(FilterPopup).disabled = True
+ self.query_one(ExportPopup).disabled = True
+ self.total_count = self.filtered_count = len(self.events.all())
+ self.populate_data_tables()
def populate_data_tables(self, focus=True)
def populate_data_tables(self, focus=True):
+ ut = self.query_one(UserTable)
+ et = self.query_one(EventTable)
+ ed = self.query_one(EventDetail)
+
+ if not self.events.all():
+ ut.contents = et.contents = []
+ ed.event = None
+
+ elif not ut.has_class("hidden"):
+ ut.contents = sorted(self.events.users(), key=str.lower)
+ if focus:
+ ut.focus()
+
+ else:
+ et.contents = self.events.all()
+ if focus:
+ et.focus()
def action_toggle_dark(self)
An action to toggle dark mode.
def action_toggle_dark(self):
+ self.dark = not self.dark
+
+ # In EventDetail we define "event" as reactive with always_update,
+ # so this will force watch_event to be called which redraws the JSON
+ # with the correct theme. We need to do this as the content of the
+ # Static widget with our JSON is colored by rich which does not know
+ # about dark mode. So when user toggles, we need to syntax highlight
+ # the JSON with an appropriate theme.
+ ed = self.query_one(EventDetail)
+ ed.event = ed.event
def action_toggle_layout(self)
def action_toggle_layout(self):
+ self.toggle_class("vertical")
def action_copy(self)
def action_copy(self):
+ ed = self.query_one(EventDetail)
+ if ed.event:
+ pyperclip.copy(ed.event.to_json())
def action_filter_popup(self)
def action_filter_popup(self):
+ self.show_popup(FilterPopup)
def action_export_popup(self)
def action_export_popup(self):
+ self.show_popup(ExportPopup)
def action_toggle_users(self)
def action_toggle_users(self):
+ ut = self.query_one(UserTable)
+ ut.toggle_class("hidden")
+ self.populate_data_tables()
def dismiss_popup(self, name)
def dismiss_popup(self, name):
+ p = self.query_one(name)
+ p.disabled = True
+ p.add_class("offscreen")
+ self.set_focus(self.original_focus)
+ self.refresh(repaint=True, layout=True)
def show_popup(self, name)
def show_popup(self, name):
+ self.original_focus = self.query_one("*:focus")
+ p = self.query_one(name)
+ p.disabled = False
+ p.remove_class("offscreen")
+ p.query_one(Input).focus()
def on_filter_popup_closed(self, _)
def on_filter_popup_closed(self, _) -> None:
+ self.dismiss_popup(FilterPopup)
def on_filter_popup_changed(self, event)
def on_filter_popup_changed(self, event: FilterPopup.Changed) -> None:
+ # Only update the UI if the expression is different
+ if event.value != self.events.filter_expr:
+ self.filtered_count = self.events.filter(event.value)
+ self.query_one(EventDetail).filter_expr = event.value
+ self.populate_data_tables(focus=False)
+ self.dismiss_popup(FilterPopup)
def on_export_popup_closed(self, _)
def on_export_popup_closed(self, _) -> None:
+ self.dismiss_popup(ExportPopup)
def on_export_popup_changed(self, event)
def on_export_popup_changed(self, event: ExportPopup.Changed) -> None:
+ try:
+ with open(event.value, "w") as out:
+ json.dump(
+ [e.event for e in self.events.all()],
+ default=str,
+ indent=4,
+ fp=out,
+ )
+ self.dismiss_popup(ExportPopup)
+ except Exception:
+ self.bell()
def on_user_table_selection_changed(self, message)
def on_user_table_selection_changed(self, message: UserTable.SelectionChanged):
+ et = self.query_one(EventTable)
+ user = message.row_key
+ if user:
+ et.contents = self.events.by_user(user)
def on_event_table_selection_changed(self, message)
def on_event_table_selection_changed(self, message: EventTable.SelectionChanged):
+ ed = self.query_one(EventDetail)
+ event_key = message.row_key
+ if event_key:
+ event = self.events.by_id(event_key)
+ ed.event = event
def watch_total_count(self)
def watch_total_count(self):
+ self.update_sub_title()
def watch_filtered_count(self)
def watch_filtered_count(self):
+ self.update_sub_title()
def update_sub_title(self)
def update_sub_title(self):
+ s = f"{self.total_count} events loaded"
+ if self.filtered_count != self.total_count:
+ s += f", {self.filtered_count} matched"
+ self.sub_title = svar DEFAULT_CSSThe type of the None singleton.
var can_focusThe type of the None singleton.
var can_focus_childrenThe type of the None singleton.
def on_mount(self):
+ self.border_title = "Event Detail"
def compose(self)
@@ -2953,12 +3280,37 @@ Example
)
yield Footer()
+def compose(self) -> ComposeResult:
+ yield Static()
def watch_event(self, event)
def watch_event(self, event):
+ content = event.to_json() if event else ""
+ matching_lines = self._find_matching_lines(content)
+
+ self.query_one(Static).update(
+ Syntax(
+ content,
+ "json",
+ theme=EventViewer.theme["dark" if self.app.dark else "light"],
+ highlight_lines=matching_lines,
+ line_numbers=True,
+ )
+ )FilterPopupExportPopupRowTable
on_mountwatch_contentsDEFAULT_CSScan_focuscan_focus_children
on_mountwatch_contentsDEFAULT_CSScan_focuscan_focus_children
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_hosted_zones.html b/docs/commands/aws/list_hosted_zones.html
index 7363b05..5e4abed 100644
--- a/docs/commands/aws/list_hosted_zones.html
+++ b/docs/commands/aws/list_hosted_zones.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_hosted_zones API documentation
-
+
@@ -273,7 +273,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_iam_policies.html b/docs/commands/aws/list_iam_policies.html
index 29b596d..8bfb777 100644
--- a/docs/commands/aws/list_iam_policies.html
+++ b/docs/commands/aws/list_iam_policies.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_iam_policies API documentation
-
+
@@ -732,8 +732,8 @@ Command Options
if self.include_roles:
identities["role"] = get_identities(iam.roles, iam.Role, self.search_roles)
- for i_type in identities: # pylint: disable=consider-using-dict-items
- for identity in identities[i_type]:
+ for i_type, identity_list in identities.items():
+ for identity in identity_list:
ip = IdentityPrinter(out, f"{acct}: identity={i_type}:{identity.name}")
self.show_inline_policies(identity, ip)
self.show_attached_policies(identity, ip)
@@ -746,12 +746,9 @@ Command Options
return
for inline in identity.policies.all():
- # pylint: disable=cell-var-from-loop
# We wrap the policy_document in a lambda so boto3 resource is not
- # fetched unless it is really needed. Although pylint complains
- # about wrapping the looping var in a lambda, we use the lambda
- # immediately if needed.
- if self.should_skip(inline.policy_name, lambda: inline.policy_document):
+ # fetched unless it is really needed.
+ if self.should_skip(inline.policy_name, lambda inline=inline: inline.policy_document):
continue
ip.print(f"policy=inline:{inline.policy_name}")
@@ -764,11 +761,10 @@ Command Options
return
for attached in identity.attached_policies.all():
- # pylint: disable=cell-var-from-loop
# We wrap the default_version.document in a lambda so boto3 resource
# is not fetched unless it is really needed.
if self.should_skip(
- attached.policy_name, lambda: attached.default_version.document
+ attached.policy_name, lambda attached=attached: attached.default_version.document
):
continue
@@ -900,7 +896,7 @@ Command Options
except ClientError as e:
if e.response["Error"]["Code"] == "NoSuchEntity":
return False
- raise e
+ raise
def has_actions(policy_doc, search_actions):
+ """Search a policy document, specifically the Action and NotAction blocks
+ for any IAM actions that match the search_actions. Matching does take into
+ account wildcards specified in the policy document. Returns true if the
+ document contains any of the actions in search_actions. Note: this does not
+ take into account whether an action is allowed or denied."""
+
+ for statement in make_list(policy_doc["Statement"]):
+ if "Action" in statement:
+ action_block = statement["Action"]
+ else:
+ action_block = statement["NotAction"]
+
+ for action in make_list(action_block):
+ pattern = re.compile("^" + re.escape(action).replace("\\*", ".*") + "$")
+ for search_action in search_actions:
+ if pattern.search(search_action):
+ return True
+ return False
def make_list(obj)
@@ -926,6 +946,19 @@ Functions
Returns obj if it is a list, otherwise returns a list of one element
containing obj. This is due to AWS's inconsistent use of JSON arrays.
+Expand source code
+
+def make_list(obj):
+ """Returns obj if it is a list, otherwise returns a list of one element
+ containing obj. This is due to AWS's inconsistent use of JSON arrays."""
+
+ if isinstance(obj, list):
+ return obj
+
+ return [obj]
+
def get_identities(collection, subresource, search_names)
@@ -935,6 +968,26 @@ Functions
identities from collection are returned. If search_names contains a list of
names, a resource is created by calling subresource. The resource is then
loaded to ensure it exists. All valid resources are returned.
+
+
+Expand source code
+
+def get_identities(collection, subresource, search_names):
+ """Return an iterable of IAM identities. If search_names is empty, then all
+ identities from collection are returned. If search_names contains a list of
+ names, a resource is created by calling subresource. The resource is then
+ loaded to ensure it exists. All valid resources are returned."""
+
+ if not search_names:
+ return collection.all()
+
+ # For each name being searched, create a resource object
+ identities = [subresource(name) for name in search_names]
+
+ # Resource objects load lazily, so we don't know if the identities
+ # above are valid or not. Let's filter out only the valid ones.
+ return filter(identity_exists, identities)
+
def identity_exists(identity)
@@ -942,6 +995,22 @@ Functions
Returns True if the identity exists, otherwise false. As a side
effect, the identity's resources are loaded.
+Expand source code
+
+def identity_exists(identity):
+ """Returns True if the identity exists, otherwise false. As a side
+ effect, the identity's resources are loaded."""
+
+ try:
+ identity.load()
+ return True
+ except ClientError as e:
+ if e.response["Error"]["Code"] == "NoSuchEntity":
+ return False
+ raise
+
@@ -1138,8 +1207,8 @@ Classes
if self.include_roles:
identities["role"] = get_identities(iam.roles, iam.Role, self.search_roles)
- for i_type in identities: # pylint: disable=consider-using-dict-items
- for identity in identities[i_type]:
+ for i_type, identity_list in identities.items():
+ for identity in identity_list:
ip = IdentityPrinter(out, f"{acct}: identity={i_type}:{identity.name}")
self.show_inline_policies(identity, ip)
self.show_attached_policies(identity, ip)
@@ -1152,12 +1221,9 @@ Classes
return
for inline in identity.policies.all():
- # pylint: disable=cell-var-from-loop
# We wrap the policy_document in a lambda so boto3 resource is not
- # fetched unless it is really needed. Although pylint complains
- # about wrapping the looping var in a lambda, we use the lambda
- # immediately if needed.
- if self.should_skip(inline.policy_name, lambda: inline.policy_document):
+ # fetched unless it is really needed.
+ if self.should_skip(inline.policy_name, lambda inline=inline: inline.policy_document):
continue
ip.print(f"policy=inline:{inline.policy_name}")
@@ -1170,11 +1236,10 @@ Classes
return
for attached in identity.attached_policies.all():
- # pylint: disable=cell-var-from-loop
# We wrap the default_version.document in a lambda so boto3 resource
# is not fetched unless it is really needed.
if self.should_skip(
- attached.policy_name, lambda: attached.default_version.document
+ attached.policy_name, lambda attached=attached: attached.default_version.document
):
continue
@@ -1227,12 +1292,55 @@ Methods
Prints the inline policies associated with identity.
+Expand source code
+
+def show_inline_policies(self, identity, ip):
+ """Prints the inline policies associated with identity."""
+ if not self.include_inline:
+ return
+
+ for inline in identity.policies.all():
+ # We wrap the policy_document in a lambda so boto3 resource is not
+ # fetched unless it is really needed.
+ if self.should_skip(inline.policy_name, lambda inline=inline: inline.policy_document):
+ continue
+
+ ip.print(f"policy=inline:{inline.policy_name}")
+ if self.verbose:
+ ip.print(json.dumps(inline.policy_document, indent=4), prefix=False)
+
def show_attached_policies(self, identity, ip)
Prints the attached policies associated with identity.
+Expand source code
+
+def show_attached_policies(self, identity, ip):
+ """Prints the attached policies associated with identity."""
+ if not self.include_attached:
+ return
+
+ for attached in identity.attached_policies.all():
+ # We wrap the default_version.document in a lambda so boto3 resource
+ # is not fetched unless it is really needed.
+ if self.should_skip(
+ attached.policy_name, lambda attached=attached: attached.default_version.document
+ ):
+ continue
+
+ ip.print(f"policy=attached:{attached.policy_name}")
+ if self.verbose:
+ ip.print(
+ json.dumps(attached.default_version.document, indent=4),
+ prefix=False,
+ )
+
def should_skip(self, name, get_doc)
@@ -1242,6 +1350,41 @@ Methods
skipped.
For efficiency, the get_doc argument should be a function
that returns the policy document, so it is only called if needed.
+
+
+Expand source code
+
+def should_skip(self, name, get_doc):
+ """Returns false if the policy with name and policy document should be
+ skipped. For efficiency, the get_doc argument should be a function
+ that returns the policy document, so it is only called if needed."""
+
+ if self.search_policies and not any(
+ name.startswith(n) for n in self.search_policies
+ ):
+ return True
+ if self.not_search_policies and any(
+ name.startswith(n) for n in self.not_search_policies
+ ):
+ return True
+
+ # Short-circuit us out of here if we don't need to search for actions,
+ # which would require downloading the policy document. Recall, boto
+ # loads these things lazily, so if we don't need to access it, then
+ # don't load it.
+ if not self.search_actions and not self.not_search_actions:
+ return False
+
+ # Since we now need to search through the actual policy for action
+ # statements, invoke the function passed to actually get the policy.
+ doc = get_doc()
+ if self.search_actions and not has_actions(doc, self.search_actions):
+ return True
+ if self.not_search_actions and has_actions(doc, self.not_search_actions):
+ return True
+
+ return False
+
Inherited members
@@ -1289,6 +1432,17 @@ Methods
Print msg to buffer, if prefix is True, prepend the prefix.
+Expand source code
+
+def print(self, msg, prefix=True):
+ """Print msg to buffer, if prefix is True, prepend the prefix."""
+ if prefix:
+ print(f"{self.prefix} {msg}", file=self.out)
+ else:
+ print(msg, file=self.out)
+
@@ -1344,7 +1498,7 @@
diff --git a/docs/commands/aws/list_iam_roles.html b/docs/commands/aws/list_iam_roles.html
index e90046d..759bcde 100644
--- a/docs/commands/aws/list_iam_roles.html
+++ b/docs/commands/aws/list_iam_roles.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_iam_roles API documentation
-
+
@@ -252,7 +252,7 @@ Command Options
arns = []
for statement in policy.get("Statement", []):
- if not statement.get("Effect", "") == "Allow":
+ if statement.get("Effect", "") != "Allow":
continue
for arn in statement.get("Principal", {}).values():
if isinstance(arn, list):
@@ -376,7 +376,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_igws.html b/docs/commands/aws/list_igws.html
index 26bcbae..895befd 100644
--- a/docs/commands/aws/list_igws.html
+++ b/docs/commands/aws/list_igws.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_igws API documentation
-
+
@@ -179,7 +179,7 @@ Command Options
file=out,
)
if attachments:
- print(f' vpcs={", ".join(attachments)}', end="", file=out)
+ print(f" vpcs={', '.join(attachments)}", end="", file=out)
print(file=out)
@@ -224,7 +224,7 @@ Classes
file=out,
)
if attachments:
- print(f' vpcs={", ".join(attachments)}', end="", file=out)
+ print(f" vpcs={', '.join(attachments)}", end="", file=out)
print(file=out)
@@ -285,7 +285,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_lambdas.html b/docs/commands/aws/list_lambdas.html
index 45d0f67..41d78fd 100644
--- a/docs/commands/aws/list_lambdas.html
+++ b/docs/commands/aws/list_lambdas.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_lambdas API documentation
-
+
@@ -213,7 +213,7 @@ Command Options
by_role[fn["Role"]].append(fn)
continue
print(
- f'{acct}/{region}: name={fn["FunctionName"]} runtime={fn["Runtime"]} role={fn["Role"]} public={_is_public(fn)}',
+ f"{acct}/{region}: name={fn['FunctionName']} runtime={fn['Runtime']} role={fn['Role']} public={_is_public(fn)}",
file=out,
)
@@ -287,7 +287,7 @@ Classes
by_role[fn["Role"]].append(fn)
continue
print(
- f'{acct}/{region}: name={fn["FunctionName"]} runtime={fn["Runtime"]} role={fn["Role"]} public={_is_public(fn)}',
+ f"{acct}/{region}: name={fn['FunctionName']} runtime={fn['Runtime']} role={fn['Role']} public={_is_public(fn)}",
file=out,
)
@@ -357,7 +357,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_public_ips.html b/docs/commands/aws/list_public_ips.html
index 1411357..8ccfbf8 100644
--- a/docs/commands/aws/list_public_ips.html
+++ b/docs/commands/aws/list_public_ips.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_public_ips API documentation
-
+
@@ -187,7 +187,7 @@ Command Options
# are processing.
for (vpc_id, owner_id), ips in public_ips.items():
print(
- f'{acct}/{region}: id={vpc_id} owner={owner_id} ips={", ".join(ips)}',
+ f"{acct}/{region}: id={vpc_id} owner={owner_id} ips={', '.join(ips)}",
file=out,
)
@@ -242,7 +242,7 @@ Classes
# are processing.
for (vpc_id, owner_id), ips in public_ips.items():
print(
- f'{acct}/{region}: id={vpc_id} owner={owner_id} ips={", ".join(ips)}',
+ f"{acct}/{region}: id={vpc_id} owner={owner_id} ips={', '.join(ips)}",
file=out,
)
@@ -303,7 +303,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_vpc_attribute.html b/docs/commands/aws/list_vpc_attribute.html
index 2322bbb..0099899 100644
--- a/docs/commands/aws/list_vpc_attribute.html
+++ b/docs/commands/aws/list_vpc_attribute.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_vpc_attribute API documentation
-
+
@@ -350,7 +350,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/aws/list_vpcs.html b/docs/commands/aws/list_vpcs.html
index 983e9a6..22e5010 100644
--- a/docs/commands/aws/list_vpcs.html
+++ b/docs/commands/aws/list_vpcs.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.aws.list_vpcs API documentation
-
+
@@ -264,7 +264,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/azure/az.html b/docs/commands/azure/az.html
index 1971ea2..50e94b7 100644
--- a/docs/commands/azure/az.html
+++ b/docs/commands/azure/az.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure.az API documentation
-
+
@@ -623,9 +623,8 @@ Note about Option Names
result = subprocess.run(
cmd,
check=False,
- universal_newlines=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ text=True,
+ capture_output=True,
)
# Lastly, we return the ProcessCompleted object from the run() method.
@@ -639,7 +638,7 @@ Note about Option Names
def annotate_lines(text, delimiter=": ", file=sys.stdout, separator=False):
for line in filter(None, text.split("\n")):
print(f"{acct}{delimiter}{line}", file=file, flush=True)
- if separator and not text == "\n":
+ if separator and text != "\n":
print()
def annotate_json(text):
@@ -673,7 +672,7 @@ Note about Option Names
# ProcessCompleted object from the subprocess.run() method above ...
result = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# ... unless there was an exception in which case it is raised by
# the call to get_result and we handle it here.
LOG.info("%s: error: %s", acct, e, exc_info=True)
@@ -844,9 +843,8 @@ Classes
result = subprocess.run(
cmd,
check=False,
- universal_newlines=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ text=True,
+ capture_output=True,
)
# Lastly, we return the ProcessCompleted object from the run() method.
@@ -860,7 +858,7 @@ Classes
def annotate_lines(text, delimiter=": ", file=sys.stdout, separator=False):
for line in filter(None, text.split("\n")):
print(f"{acct}{delimiter}{line}", file=file, flush=True)
- if separator and not text == "\n":
+ if separator and text != "\n":
print()
def annotate_json(text):
@@ -894,7 +892,7 @@ Classes
# ProcessCompleted object from the subprocess.run() method above ...
result = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# ... unless there was an exception in which case it is raised by
# the call to get_result and we handle it here.
LOG.info("%s: error: %s", acct, e, exc_info=True)
@@ -958,12 +956,150 @@ Methods
Invoke an Azure CLI command for an account.
+Expand source code
+
+def execute(self, session, acct):
+ """Invoke an Azure CLI command for an account."""
+
+ # We need to assemble a valid Azure CLI command line that can be
+ # executed by the operating system. The instance variable azureCLI_args
+ # contains all arguments that follow 'az': azurerun az ... We will
+ # provide --output if the user has asked us to annotate an output type.
+ # This ensures we override any user settings that the Azure CLI tool may
+ # pick up from ~/.azure directory.
+ cmd = [self.azurecli_path]
+ cmd += self.azurecli_args
+ cmd += ["--subscription", str(acct)]
+ if self.annotate:
+ cmd += ["--output", self.annotate]
+ elif self.output:
+ cmd += ["--output", self.output]
+ LOG.info("%s: Azure CLI command: %s", acct, cmd)
+
+ # Although the execute method receives a valid credential in the
+ # `session` argument, I've not found a way to pass that to the az CLI
+ # command. It doesn't matter though as az CLI users will simply use `az
+ # login` before running this azurerun wrapper.
+
+ # We call run() and capture stdout and stderr from the command's output.
+ # Note: all the output is stored in memory, and then printed in
+ # collect_results. This means that if you run an az CLI command that
+ # generates huge amounts of data, it'll all be stored in memory. Why
+ # don't we stream tho output from a pipe? We could use Popen directly,
+ # but if we returned from execute() before reading all of the results,
+ # then the worker will start another account, so in essence, all of the
+ # accounts will be "executed" immediately resulting in potentially many
+ # many Azure CLI command processes running waiting for us to read the
+ # output.
+
+ result = subprocess.run(
+ cmd,
+ check=False,
+ text=True,
+ capture_output=True,
+ )
+
+ # Lastly, we return the ProcessCompleted object from the run() method.
+ # Recall, an azurerun command can return anything if you provide your
+ # own collect_results method.
+ return result
+
def collect_results(self, acct, get_result)
Print the results to the console and files if specified.
+Expand source code
+
+def collect_results(self, acct, get_result):
+ """Print the results to the console and files if specified."""
+
+ def annotate_lines(text, delimiter=": ", file=sys.stdout, separator=False):
+ for line in filter(None, text.split("\n")):
+ print(f"{acct}{delimiter}{line}", file=file, flush=True)
+ if separator and text != "\n":
+ print()
+
+ def annotate_json(text):
+ try:
+ d = {
+ "Subscription": str(acct),
+ "Results": json.loads(text),
+ }
+ json.dump(d, sys.stdout, indent=4)
+ print()
+ except json.decoder.JSONDecodeError:
+ annotate_lines(
+ "Result of Azure CLI command is not valid JSON", file=sys.stderr
+ )
+
+ def annotate_yaml(text):
+ try:
+ d = {
+ "Subscription": str(acct),
+ "Results": yaml.safe_load(text),
+ }
+ yaml.safe_dump(d, sys.stdout, indent=4)
+ print("...") # end of yaml document separator
+ except yaml.representer.RepresenterError:
+ annotate_lines(
+ "Result of Azure CLI command is not valid JSON", file=sys.stderr
+ )
+
+ try:
+ # Let's get the return value from the execute method, which is the
+ # ProcessCompleted object from the subprocess.run() method above ...
+ result = get_result()
+
+ except Exception as e:
+ # ... unless there was an exception in which case it is raised by
+ # the call to get_result and we handle it here.
+ LOG.info("%s: error: %s", acct, e, exc_info=True)
+ annotate_lines(f"error: {e}", file=sys.stderr)
+ return
+
+ # Print stderr from Azure CLI always annotating the lines
+ annotate_lines(result.stderr, file=sys.stderr)
+
+ # Print stdout from Azure CLI annotating when appropriate
+ if not self.annotate:
+ if result.stdout not in ["", "\n"]: # skip blank output
+ print(
+ result.stdout,
+ end="\n" if self.output == "table" else "",
+ flush=True,
+ )
+ elif self.annotate == "json":
+ annotate_json(result.stdout)
+ elif self.annotate == "yaml":
+ annotate_yaml(result.stdout)
+ elif self.annotate == "table":
+ annotate_lines(result.stdout, separator=True)
+ elif self.annotate == "tsv":
+ annotate_lines(result.stdout, delimiter="\t")
+
+ # Save stdout and stderr from Azure CLI to disk if requested
+ if self.output_dir:
+ # Recall, the acct object passed to execute() can be anything. The
+ # str() method should provide us a unique means of identifying the
+ # account, but we need to escape any slashes if we use this as part
+ # of a filename so pathlib doesn't interpret as directories.
+ escaped = re.sub(r"[\\/]", "_", str(acct))
+ name = self.output_dir / f"{escaped}"
+
+ def save(suffix, text):
+ with name.with_suffix(suffix).open("w") as out:
+ out.write(text)
+
+ save(".stdout.log", result.stdout)
+ if result.stderr:
+ save(".stderr.log", result.stderr)
+
Inherited members
@@ -1022,7 +1158,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/azure/cidr_overlap.html b/docs/commands/azure/cidr_overlap.html
index dd8cbcc..c5a0e4b 100644
--- a/docs/commands/azure/cidr_overlap.html
+++ b/docs/commands/azure/cidr_overlap.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure.cidr_overlap API documentation
-
+
@@ -239,7 +239,7 @@ Command Options
def post_hook(self):
overlap = []
- for i in range(0, len(self.cidrs)): # pylint: disable=consider-using-enumerate
+ for i in range(len(self.cidrs)):
c1 = self.cidrs[i]
for j in range(i + 1, len(self.cidrs)):
c2 = self.cidrs[j]
@@ -310,7 +310,7 @@ Classes
def post_hook(self):
overlap = []
- for i in range(0, len(self.cidrs)): # pylint: disable=consider-using-enumerate
+ for i in range(len(self.cidrs)):
c1 = self.cidrs[i]
for j in range(i + 1, len(self.cidrs)):
c2 = self.cidrs[j]
@@ -371,7 +371,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/azure/index.html b/docs/commands/azure/index.html
index e8dcbab..14f9c82 100644
--- a/docs/commands/azure/index.html
+++ b/docs/commands/azure/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure API documentation
-
+
@@ -157,7 +157,7 @@ Index
diff --git a/docs/commands/azure/list_udrs.html b/docs/commands/azure/list_udrs.html
index a7c7191..a9f274b 100644
--- a/docs/commands/azure/list_udrs.html
+++ b/docs/commands/azure/list_udrs.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure.list_udrs API documentation
-
+
@@ -235,7 +235,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/azure/list_vnets.html b/docs/commands/azure/list_vnets.html
index 4db6d5b..d74dc47 100644
--- a/docs/commands/azure/list_vnets.html
+++ b/docs/commands/azure/list_vnets.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands.azure.list_vnets API documentation
-
+
@@ -216,7 +216,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/commands/index.html b/docs/commands/index.html
index a65881a..83f7283 100644
--- a/docs/commands/index.html
+++ b/docs/commands/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.commands API documentation
-
+
@@ -1603,7 +1603,7 @@ Index
diff --git a/docs/config.html b/docs/config.html
index f8b44d2..ae1541f 100644
--- a/docs/config.html
+++ b/docs/config.html
@@ -3,13 +3,13 @@
-
+
awsrun.config API documentation
-
+
@@ -211,13 +211,12 @@ Reading Values
import re
from functools import reduce
from pathlib import Path
+from typing import ClassVar
import yaml
LOG = logging.getLogger(__name__)
-# pylint: disable=unidiomatic-typecheck
-#
# Because isinstance(True, int) is true, we do not rely on isinstance for our
# type checking in this module as we want to match exact types. We don't want to
# consider subclasses and True should not type check successfully against an
@@ -235,7 +234,7 @@ Reading Values
load configs from files.
"""
- _filetypes = {}
+ _filetypes: ClassVar[dict] = {}
@classmethod
def register_filetype(cls, config_class, *extensions):
@@ -302,8 +301,6 @@ Reading Values
c.get('path', 'to', 'value', type=And(StrMatch(r'\\d+'), StrMatch(r'[A-Z]')))
c.get('path', 'to', 'value', type=Not(Or(Int, Float)))
"""
- # pylint: disable=redefined-builtin
-
# This one-liner will recursively follow a list of keys into a
# dictionary and return the value. If a key does not exist, return an
# empty dict.
@@ -338,7 +335,7 @@ Reading Values
# Finally, all other cases indicate a type error.
raise TypeError(
- f"Error in config: {'->'.join(keys)}: not a {type}: {repr(value)}"
+ f"Error in config: {'->'.join(keys)}: not a {type}: {value!r}"
)
@@ -447,8 +444,7 @@ Reading Values
# we did not check types, then this would report incorrect results.
# Likewise, we cannot use isinstance here either as a bool is a subclass
# of int, so it would also report incorrect results.
- if type(obj) != type(self.const): # noqa: E721
- return False
+ if type(obj) != type(self.const): return False
return obj == self.const
def __str__(self):
@@ -476,8 +472,7 @@ Reading Values
self.type = type_
def type_check(self, obj):
- return type(obj) == self.type # noqa: E721
-
+ return type(obj) == self.type
def __str__(self):
return self.type.__name__
@@ -492,8 +487,7 @@ Reading Values
self.pattern = pattern
def type_check(self, obj):
- if type(obj) != str: # noqa: E721
- return False
+ if type(obj) != str: return False
return bool(re.search(self.pattern, obj))
def __str__(self):
@@ -504,8 +498,7 @@ Reading Values
"""Represents a string matching an IP address (v4 or v6)."""
def type_check(self, obj):
- if type(obj) != str: # noqa: E721
- return False
+ if type(obj) != str: return False
try:
ipaddress.ip_address(obj)
return True
@@ -520,8 +513,7 @@ Reading Values
"""Represents a string matching an IP network (v4 or v6)."""
def type_check(self, obj):
- if type(obj) != str: # noqa: E721
- return False
+ if type(obj) != str: return False
try:
ipaddress.ip_network(obj)
return True
@@ -536,8 +528,7 @@ Reading Values
"""Represents a string pointing to an existing file."""
def type_check(self, obj):
- if type(obj) != str: # noqa: E721
- return False
+ if type(obj) != str: return False
return Path(obj).exists()
def __str__(self):
@@ -600,8 +591,7 @@ Reading Values
self.element_type = element_type
def type_check(self, obj):
- if type(obj) != list: # noqa: E721
- return False
+ if type(obj) != list: return False
return all(self.element_type.type_check(e) for e in obj)
def __str__(self):
@@ -623,9 +613,8 @@ Reading Values
self.value_type = value_type
def type_check(self, obj):
- if type(obj) != dict: # noqa: E721
- return False
- return all(self.key_type.type_check(k) for k in obj.keys()) and all(
+ if type(obj) != dict: return False
+ return all(self.key_type.type_check(k) for k in obj) and all(
self.value_type.type_check(v) for v in obj.values()
)
@@ -716,7 +705,7 @@ Classes
load configs from files.
"""
- _filetypes = {}
+ _filetypes: ClassVar[dict] = {}
@classmethod
def register_filetype(cls, config_class, *extensions):
@@ -783,8 +772,6 @@ Classes
c.get('path', 'to', 'value', type=And(StrMatch(r'\\d+'), StrMatch(r'[A-Z]')))
c.get('path', 'to', 'value', type=Not(Or(Int, Float)))
"""
- # pylint: disable=redefined-builtin
-
# This one-liner will recursively follow a list of keys into a
# dictionary and return the value. If a key does not exist, return an
# empty dict.
@@ -819,7 +806,7 @@ Classes
# Finally, all other cases indicate a type error.
raise TypeError(
- f"Error in config: {'->'.join(keys)}: not a {type}: {repr(value)}"
+ f"Error in config: {'->'.join(keys)}: not a {type}: {value!r}"
)
Subclasses
@@ -880,6 +867,75 @@ Methods
c.get('path', 'to', 'value', type=And(StrMatch(r'\d+'), StrMatch(r'[A-Z]')))
c.get('path', 'to', 'value', type=Not(Or(Int, Float)))
+
+
+Expand source code
+
+def get(self, *keys, default=None, type=None, must_exist=False):
+ """Return the specified value from the `Config`.
+
+ Specify the value to read by providing the keys required to reach the
+ value in the configuration. If the value is not found at the specified
+ key path, `None` or the `default` value is returned unless the
+ `must_exist` flag is `True`, in which case a `ValueError` is raised.
+
+ Values can be optionally type-checked to ensure it matches the specified
+ type. If the `type` matches the value in the configuration, the value is
+ returned, otherwise a `TypeError` is raised. Types are specified by
+ passing a `Type` object. There are numerous type objects defined in this
+ module. For example:
+
+ c.get('path', 'to', 'value', type=Int)
+ c.get('path', 'to', 'value', type=Bool)
+ c.get('path', 'to', 'value', type=Float)
+ c.get('path', 'to', 'value', type=Str)
+ c.get('path', 'to', 'value', type=StrMatch(r'^\\d+-\\d+$'))
+ c.get('path', 'to', 'value', type=IP)
+ c.get('path', 'to', 'value', type=List(IP))
+ c.get('path', 'to', 'value', type=List(Str))
+ c.get('path', 'to', 'value', type=List(Dict(Int, Str)))
+ c.get('path', 'to', 'value', type=Dict(Str, Int))
+ c.get('path', 'to', 'value', type=Or(Int, Float))
+ c.get('path', 'to', 'value', type=And(StrMatch(r'\\d+'), StrMatch(r'[A-Z]')))
+ c.get('path', 'to', 'value', type=Not(Or(Int, Float)))
+ """
+ # This one-liner will recursively follow a list of keys into a
+ # dictionary and return the value. If a key does not exist, return an
+ # empty dict.
+ try:
+ value = reduce(lambda a, p: a.get(p, {}), keys, self.conf)
+ except AttributeError as e:
+ raise ValueError(
+ f"Error in config: {'->'.join(keys[:-1])}: not a dictionary"
+ ) from e
+
+ # If value is {} that means the key doesn't exist. If the must_exist
+ # flag was passed, then we raise a descriptive ValueError, otherwise we
+ # set it to the default.
+ if value == {}:
+ if must_exist:
+ raise ValueError(f"Error in config: {'->'.join(keys)}: must be set")
+ value = default
+
+ # If no value has been set in the config and none has been provided as a
+ # default, then return None.
+ if value is None:
+ return value
+
+ # If no type has been specified, then return the value in the config or
+ # the default without doing any type checking.
+ if not type:
+ return value
+
+ # Only return the value if it type checks correctly.
+ if type.type_check(value):
+ return value
+
+ # Finally, all other cases indicate a type error.
+ raise TypeError(
+ f"Error in config: {'->'.join(keys)}: not a {type}: {value!r}"
+ )
+
@@ -988,6 +1044,14 @@ Methods
Returns true if obj is a type matching this Type.
def type_check(self, obj):
+ """Returns true if obj is a type matching this `Type`."""
+ raise NotImplementedErrorDict
diff --git a/docs/index.html b/docs/index.html
index f0b96c0..0ca6d5a 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -3,13 +3,13 @@
-
+
awsrun API documentation
-
+
@@ -316,7 +316,7 @@ Roadmap
"""
name = "awsrun"
-__version__ = "3.2.1"
+__version__ = "3.3.0"
awsrun.plugins.accts.azure
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/plugins/accts/index.html b/docs/plugins/accts/index.html
index 46dc4fa..854b341 100644
--- a/docs/plugins/accts/index.html
+++ b/docs/plugins/accts/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins.accts API documentation
-
+
@@ -1010,7 +1010,6 @@ Module awsrun.plugins.accts
# Check and set auth options if using authentication.
if args.loader_auth != "none":
-
# Command line flags take priority
if args.loader_username:
auth_options["username"] = args.loader_username
@@ -2563,7 +2562,6 @@ Plug-in Options
# Check and set auth options if using authentication.
if args.loader_auth != "none":
-
# Command line flags take priority
if args.loader_username:
auth_options["username"] = args.loader_username
@@ -2680,7 +2678,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/plugins/creds/aws.html b/docs/plugins/creds/aws.html
index 85947a4..6121523 100644
--- a/docs/plugins/creds/aws.html
+++ b/docs/plugins/creds/aws.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins.creds.aws API documentation
-
+
@@ -156,12 +156,17 @@ Module awsrun.plugins.creds.aws
from awsrun.config import URL, Bool, Choice, Dict, Int, Str
from awsrun.plugmgr import Plugin
-from awsrun.session.aws import CredsViaCrossAccount, CredsViaProfile, CredsViaSAML
+from awsrun.session.aws import (
+ CredsViaCrossAccount,
+ CredsViaOAuth2ROPC,
+ CredsViaProfile,
+ CredsViaSAML,
+)
# This is only used to prevent pdoc (the doc generator) from exposing
# AbstractCrossAccount in the module's documentation, which is intended for CLI
# users and not programmers as this module deals with plugin configurations.
-__all__ = ["Profile", "SAML", "ProfileCrossAccount", "SAMLCrossAccount"]
+__all__ = ["SAML", "Profile", "ProfileCrossAccount", "SAMLCrossAccount"]
_AUTH_CLASSES = {"basic": HTTPBasicAuth, "digest": HTTPDigestAuth, "ntlm": HttpNtlmAuth}
@@ -407,6 +412,172 @@ Module awsrun.plugins.creds.aws
return session_provider
+class OAuth2(Plugin):
+ """CLI plug-in that uses federated authentication via OAuth2 for credentials.
+
+ AWS supports federated access via OpenID Connect (OIDC). With this
+ plug-in, the user authenticates to an OAuth2 server using the Resource
+ Owner Password Credential (ROPC) flow, such as Microsoft Entra, to obtain
+ an access token that is used to assume an AWS role within an account to
+ obtain STS credentials. Unlike the `Profile` plug-in, this plug-in does
+ not rely on the standard AWS credentials or configuration files, and thus
+ does not require defining all accounts ahead of time. This plug-in does,
+ however, require that the OAuth2 server is configured to grant a user
+ access to authorized AWS accounts.
+
+ Please refer to AWS [OIDC federation guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html)
+ for additional details on the use of federated access with OIDC.
+
+ ## Configuration
+
+ Options with an asterisk are mandatory and must be provided:
+
+ Credentials:
+ plugin: awsrun.plugins.creds.aws.OAuth2
+ options:
+ username: STRING
+ password: STRING
+ role: STRING*
+ url: STRING*
+ http_headers:
+ STRING: STRING
+ oauth2_params:
+ STRING: STRING
+ no_verify: BOOLEAN
+ duration: INTEGER
+
+ ## Plug-in Options
+
+ Some options can be overridden on the awsrun CLI via command line flags.
+ In those cases, the CLI flags are specified next to the option name below.
+
+ `username`, `--oauth2-username`
+ : The username to use when authenticating with the OAuth2 server for the base
+ account. If this value is not provided, the following environment variables
+ are checked in order: LOGNAME, USER, LNAME and USERNAME.
+
+ `password`, `--oauth2-password`
+ : The password to use when authenticating with the OAuth2 server for the base
+ account. The default is the value, if any, of the PASSWORD environment
+ variable. If none of these are set, the user will be prompted via the
+ console when awsrun is invoked.
+
+ `role`, `--oauth2-role`
+ : The AWS role (not ARN) to assume in the base or source account when
+ federating via OIDC. This value **must** be provided via the user
+ configuration or as an awsrun command line argument. Note: this is not the
+ IAM role name used when performing the cross-account assume role (see
+ `x_acct: role` below).
+
+ `url`
+ : The URL to the OAuth2 server that will provide an access token upon
+ successful authentication for the base account. This value **must** be
+ provided via the user configuration file.
+
+ `http_headers`
+ : Additional HTTP headers to send in the request to the OAuth2 server. If
+ specified, it must be a dictionary of `key: value` pairs, where keys and
+ values are strings.
+
+ `oauth2_params`
+ : Additional OAuth2 parameters to send in the request to the OAuth2 server. If
+ specified, it must be a dictionary of `key: value` pairs, where keys and
+ values are strings. For example, this can be used to specify `client_id`
+ and `scope` parameters if required by the OAuth2 server.
+
+ `no_verify`, `--oauth2-no-verify`
+ : Disable HTTP certificate verification. This is not advisable and user will
+ be warned on the command line if verification has been disabled. The default
+ value is `false`.
+
+ `duration`, `--oauth2-duration`
+ : The amount of time, in seconds, that the AWS credentials for the role /
+ base account combination are cached in memory. The default value is `3600`
+ seconds (1 hour). Caching can be disabled by specifying `0` seconds. If AWS
+ credentials are expired sooner by a local IAM policy, then lower the value.
+ """
+
+ def __init__(self, parser, cfg):
+ super().__init__(parser, cfg)
+
+ # Define the arguments that we want to allow a user to override via the
+ # main CLI. Any CLI args added via add_argument will be commingled with
+ # the main awsrun args, so they are prefixed with '--oauth2-' to lessen
+ # chance of a name collision.
+ group = parser.add_argument_group("OAuth2 options")
+ group.add_argument(
+ "--oauth2-username",
+ metavar="USER",
+ default=self.cfg("username", type=Str, default=getpass.getuser()),
+ help="username for OAuth2 authentication",
+ )
+
+ group.add_argument(
+ "--oauth2-password",
+ metavar="PASS",
+ default=self.cfg(
+ "password", type=Str, default=os.environ.get("PASSWORD", None)
+ ),
+ help="password for OAuth2 authentication",
+ )
+
+ group.add_argument(
+ "--oauth2-role",
+ metavar="ROLE",
+ default=self.cfg("role", type=Str, must_exist=True),
+ help="base role name (not ARN) to assume via OAuth2",
+ )
+
+ group.add_argument(
+ "--oauth2-duration",
+ metavar="SECS",
+ type=int,
+ default=cfg("duration", type=Int, default=3600),
+ help="duration when requesting aws credentials in assume_role*",
+ )
+
+ group.add_argument(
+ "--oauth2-no-verify",
+ action="store_true",
+ default=cfg("no_verify", type=Bool, default=False),
+ help="disable cert verification for HTTP requests",
+ )
+
+ def instantiate(self, args):
+ cfg = self.cfg
+
+ # We don't prompt for this password above when checking the PASSWORD
+ # environment as we wouldn't have access to the username. which should
+ # be included in the prompt to the user to remind them of the username
+ # being used.
+ args.oauth2_password = args.oauth2_password or getpass.getpass(
+ f"Password for {args.oauth2_username}? "
+ )
+
+ # Build a session provider using the combination of options that have
+ # been specified in the user's configuration file or have been
+ # overridden on the command line.
+ session_provider = CredsViaOAuth2ROPC(
+ role=args.oauth2_role,
+ url=cfg("url", type=URL, must_exist=True),
+ username=args.oauth2_username,
+ password=args.oauth2_password,
+ headers=cfg("http_headers", type=Dict(Str, Str), default={}),
+ oauth2_params=cfg("oauth2_params", type=Dict(Str, Str), default={}),
+ duration=args.oauth2_duration,
+ no_verify=args.oauth2_no_verify,
+ )
+
+ # Test if password provided is correct. If the wrong password is used and
+ # one cannot be authenticated, then AccountRunner might inadvertently lock
+ # the user out as it will repeatedly attempt to get an OAuth2 token for
+ # each account. If this fails, an exception will be thrown and caught in
+ # the main CLI.
+ session_provider.token()
+
+ return session_provider
+
+
class AbstractCrossAccount(Plugin):
"""Abstract base class for cross-account access plug-ins.
@@ -680,122 +851,137 @@ Module awsrun.plugins.creds.aws
super().__init__(parser, cfg)
def _get_base_auth(self):
- return self._base_auth
-
-
-class Profile
-(parser, cfg)
-CLI plug-in that uses standard AWS configuration files for credentials.
-The AWS CLI allows users to define "profiles" in their AWS credentials file -(~/.aws/credentials) and their AWS configuration file (~/.aws/config). For -example, if ~/.aws/credentials file defines the following two profiles, then -awsrun can be used to access accounts 111222333444 and 222333444111:
-[111222333444]
-aws_access_key_id=AKIAIOSFODNN7EXAMPLE
-aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+ return self._base_auth
-[222333444111]
-aws_access_key_id=BEOQODRDNEUDFEXAMPLE
-aws_secret_access_key=8weuou7etAUGZehZHoZeQHVBxogGRLEXAMPLEKEY
-If a profile is not explicitly defined for an account, this plug-in will -fallback to the "default" profile for account access. For example:
-[default]
-aws_access_key_id=BEOQODRDNEUDFEXAMPLE
-aws_secret_access_key=8weuou7etAUGZehZHoZeQHVBxogGRLEXAMPLEKEY
-By leveraging the standard AWS CLI credential and configuration files, users -can specify credentials in a variety of ways including standard access and -secret keys, pre-defined cross-account access, or even use an external -process to obtain credentials.
-Please refer to the AWS CLI Configuration and Credential -Files -for additional details on the use of the standard AWS files and how to -configure them.
-Credentials:
- plugin: awsrun.plugins.creds.aws.Profile
-There are no options for this plug-in.
class Profile(Plugin):
- """CLI plug-in that uses standard AWS configuration files for credentials.
- ## Overview
+class OAuth2CrossAccount(AbstractCrossAccount):
+ """CLI plug-in that uses a OAuth2 base account for cross-account access.
- The AWS CLI allows users to define "profiles" in their AWS credentials file
- (~/.aws/credentials) and their AWS configuration file (~/.aws/config). For
- example, if ~/.aws/credentials file defines the following two profiles, then
- awsrun can be used to access accounts 111222333444 and 222333444111:
+ AWS supports cross-account access from one "base" or "source" account to
+ another account. This plug-in obtains credentials for the base account via
+ the `OAuth2` plug-in, and then uses those credentials to assume role to the
+ other account provided the proper IAM permissions have been setup. Cross
+ account access is typically used in large enterprises with a significant
+ number of AWS accounts to simply management of credentials. The benefit is
+ that a user does not need direct access to every account. Only access to the
+ base account is required from which the user can "hop off" to other
+ accounts.
- [111222333444]
- aws_access_key_id=AKIAIOSFODNN7EXAMPLE
- aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+ Please refer to [Providing Access to an IAM User in Another AWS Account That
+ You
+ Own](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_aws-accounts.html)
+ for additional details on cross-account access.
- [222333444111]
- aws_access_key_id=BEOQODRDNEUDFEXAMPLE
- aws_secret_access_key=8weuou7etAUGZehZHoZeQHVBxogGRLEXAMPLEKEY
+ ## Configuration
- If a profile is not explicitly defined for an account, this plug-in will
- fallback to the "default" profile for account access. For example:
+ Options with an asterisk are mandatory and must be provided:
- [default]
- aws_access_key_id=BEOQODRDNEUDFEXAMPLE
- aws_secret_access_key=8weuou7etAUGZehZHoZeQHVBxogGRLEXAMPLEKEY
+ Credentials:
+ plugin: awsrun.plugins.creds.aws.OAuth2CrossAccount
+ options:
+ username: STRING
+ password: STRING
+ role_arn: STRING*
+ url: STRING*
+ http_headers:
+ STRING: STRING
+ oauth2_params:
+ STRING: STRING
+ no_verify: BOOLEAN
+ duration: INTEGER
+ x_acct:
+ base: STRING*
+ role: STRING*
+ duration: INTEGER
- By leveraging the standard AWS CLI credential and configuration files, users
- can specify credentials in a variety of ways including standard access and
- secret keys, pre-defined cross-account access, or even use an external
- process to obtain credentials.
+ ## Plug-in Options
- Please refer to the [AWS CLI Configuration and Credential
- Files](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html)
- for additional details on the use of the standard AWS files and how to
- configure them.
+ Some options can be overridden on the awsrun CLI via command line flags.
+ In those cases, the CLI flags are specified next to the option name below.
- ## Configuration
+ `username`, `--oauth2-username`
+ : The username to use when authenticating with the OAuth2 server for the base
+ account. If this value is not provided, the following environment variables
+ are checked in order: LOGNAME, USER, LNAME and USERNAME.
- Credentials:
- plugin: awsrun.plugins.creds.aws.Profile
+ `password`, `--oauth2-password`
+ : The password to use when authenticating with the OAuth2 server for the base
+ account. The default is the value, if any, of the PASSWORD environment
+ variable. If none of these are set, the user will be prompted via the
+ console when awsrun is invoked.
- ## Plug-in Options
+ `role_arn`, `--oauth2-role-arn`
+ : The AWS role ARN to assume in the base or source account when
+ federating via OIDC. This value **must** be provided via the user
+ configuration or as an awsrun command line argument. Note: this is not the
+ IAM role name used when performing the cross-account assume role (see
+ `x_acct: role` below).
- There are no options for this plug-in.
+ `url`
+ : The URL to the OAuth2 server that will provide an access token upon
+ successful authentication for the base account. This value **must** be
+ provided via the user configuration file.
+
+ `http_headers`
+ : Additional HTTP headers to send in the request to the OAuth2 server. If
+ specified, it must be a dictionary of `key: value` pairs, where keys and
+ values are strings.
+
+ `oauth2_params`
+ : Additional OAuth2 parameters to send in the request to the OAuth2 server. If
+ specified, it must be a dictionary of `key: value` pairs, where keys and
+ values are strings. For example, this can be used to specify `client_id`
+ and `scope` parameters if required by the OAuth2 server.
+
+ `no_verify`, `--oauth2-no-verify`
+ : Disable HTTP certificate verification. This is not advisable and user will
+ be warned on the command line if verification has been disabled. The default
+ value is `false`.
+
+ `duration`, `--oauth2-duration`
+ : The amount of time, in seconds, that the AWS credentials for the role /
+ base account combination are cached in memory. The default value is `3600`
+ seconds (1 hour). Caching can be disabled by specifying `0` seconds. If AWS
+ credentials are expired sooner by a local IAM policy, then lower the value.
+
+ `x_acct: base`, `--x-acct-base`
+ : The base or source account from which to "hop off" to other accounts. This
+ value **must** be provided via the user configuration or as an awsrun
+ command line argument. OAuth2-based authentication will be used to obtain
+ credentials for this base account.
+
+ `x_acct: role`, `--x-acct-role`
+ : The IAM role name, not ARN, to assume from the base account to obtain
+ credentials for other accounts. It is not an ARN as the ARN is dynamically
+ created as each account is processed. This value **must** be provided via
+ the user configuration or as an awsrun command line argument. Note: this
+ value is not the role used for the OAuth2 authentication to the base
+ account (see the `role_arn` option above).
+
+ `x_acct: duration`, `--x-acct-duration`
+ : The amount of time, in seconds, that the AWS credentials for the role /
+ cross-account combination are cached in memory. The default value is `3600`
+ seconds (1 hour). Caching can be disabled by specifying `0` seconds. If AWS
+ credentials are expired sooner by a local IAM policy, then lower the value.
"""
- def instantiate(self, args):
- return CredsViaProfile()- Plugin
Plugin:
-instantiate
class SAML
(parser, cfg)
@@ -1083,6 +1269,110 @@ Inherited members
+class Profile
+(parser, cfg)
+CLI plug-in that uses standard AWS configuration files for credentials.
+The AWS CLI allows users to define "profiles" in their AWS credentials file +(~/.aws/credentials) and their AWS configuration file (~/.aws/config). For +example, if ~/.aws/credentials file defines the following two profiles, then +awsrun can be used to access accounts 111222333444 and 222333444111:
+[111222333444]
+aws_access_key_id=AKIAIOSFODNN7EXAMPLE
+aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
+[222333444111]
+aws_access_key_id=BEOQODRDNEUDFEXAMPLE
+aws_secret_access_key=8weuou7etAUGZehZHoZeQHVBxogGRLEXAMPLEKEY
+If a profile is not explicitly defined for an account, this plug-in will +fallback to the "default" profile for account access. For example:
+[default]
+aws_access_key_id=BEOQODRDNEUDFEXAMPLE
+aws_secret_access_key=8weuou7etAUGZehZHoZeQHVBxogGRLEXAMPLEKEY
+By leveraging the standard AWS CLI credential and configuration files, users +can specify credentials in a variety of ways including standard access and +secret keys, pre-defined cross-account access, or even use an external +process to obtain credentials.
+Please refer to the AWS CLI Configuration and Credential +Files +for additional details on the use of the standard AWS files and how to +configure them.
+Credentials:
+ plugin: awsrun.plugins.creds.aws.Profile
+There are no options for this plug-in.
class Profile(Plugin):
+ """CLI plug-in that uses standard AWS configuration files for credentials.
+
+ ## Overview
+
+ The AWS CLI allows users to define "profiles" in their AWS credentials file
+ (~/.aws/credentials) and their AWS configuration file (~/.aws/config). For
+ example, if ~/.aws/credentials file defines the following two profiles, then
+ awsrun can be used to access accounts 111222333444 and 222333444111:
+
+ [111222333444]
+ aws_access_key_id=AKIAIOSFODNN7EXAMPLE
+ aws_secret_access_key=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+
+ [222333444111]
+ aws_access_key_id=BEOQODRDNEUDFEXAMPLE
+ aws_secret_access_key=8weuou7etAUGZehZHoZeQHVBxogGRLEXAMPLEKEY
+
+ If a profile is not explicitly defined for an account, this plug-in will
+ fallback to the "default" profile for account access. For example:
+
+ [default]
+ aws_access_key_id=BEOQODRDNEUDFEXAMPLE
+ aws_secret_access_key=8weuou7etAUGZehZHoZeQHVBxogGRLEXAMPLEKEY
+
+ By leveraging the standard AWS CLI credential and configuration files, users
+ can specify credentials in a variety of ways including standard access and
+ secret keys, pre-defined cross-account access, or even use an external
+ process to obtain credentials.
+
+ Please refer to the [AWS CLI Configuration and Credential
+ Files](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html)
+ for additional details on the use of the standard AWS files and how to
+ configure them.
+
+ ## Configuration
+
+ Credentials:
+ plugin: awsrun.plugins.creds.aws.Profile
+
+ ## Plug-in Options
+
+ There are no options for this plug-in.
+ """
+
+ def instantiate(self, args):
+ return CredsViaProfile()- Plugin
Plugin:
+instantiate
class ProfileCrossAccount
(parser, cfg)
@@ -1507,10 +1797,10 @@ Index
Classes
-
-
Profile
+SAML
-
-
SAML
+Profile
-
ProfileCrossAccount
@@ -1524,7 +1814,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/plugins/creds/azure.html b/docs/plugins/creds/azure.html
index b7db8e9..498a005 100644
--- a/docs/plugins/creds/azure.html
+++ b/docs/plugins/creds/azure.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins.creds.azure API documentation
-
+
@@ -595,7 +595,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/plugins/creds/index.html b/docs/plugins/creds/index.html
index f7cc363..83515d8 100644
--- a/docs/plugins/creds/index.html
+++ b/docs/plugins/creds/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins.creds API documentation
-
+
@@ -163,7 +163,7 @@ Index
diff --git a/docs/plugins/index.html b/docs/plugins/index.html
index cccfab8..c6d958d 100644
--- a/docs/plugins/index.html
+++ b/docs/plugins/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugins API documentation
-
+
@@ -189,7 +189,7 @@ Index
diff --git a/docs/plugmgr.html b/docs/plugmgr.html
index 0aec0bb..277b317 100644
--- a/docs/plugmgr.html
+++ b/docs/plugmgr.html
@@ -3,13 +3,13 @@
-
+
awsrun.plugmgr API documentation
-
+
@@ -204,6 +204,7 @@ Overview
`PluginManager.parse_args` as it will be called by `PluginManager.instantiate`
if it was not already.
"""
+
import importlib
import logging
from contextlib import suppress
@@ -529,6 +530,45 @@ Functions
the module to return. For example, some.module.MyClass will return the
class object MyClass from the Python module some.module. If the object
cannot be loaded, ImportError is raised.
+
+
+Expand source code
+
+def load_dotted_object(dotted_name):
+ """Returns the Python object found at the `dotted_name`.
+
+ `dotted_name` should include both the Python module as well as the object in
+ the module to return. For example, `some.module.MyClass` will return the
+ class object `MyClass` from the Python module `some.module`. If the object
+ cannot be loaded, `ImportError` is raised.
+ """
+
+ def doit(mod_name, attributes=None):
+ attributes = [] if attributes is None else attributes
+
+ if not mod_name:
+ raise ImportError(f"cannot import '{dotted_name}'")
+
+ mod = None
+ with suppress(ModuleNotFoundError):
+ mod = importlib.import_module(mod_name)
+
+ if not mod:
+ mod_name, _, attr = mod_name.rpartition(".")
+ attributes.append(attr)
+ return doit(mod_name, attributes)
+
+ attributes.reverse()
+ obj = reduce(lambda a, p: getattr(a, p, {}), attributes, mod)
+ if not obj:
+ raise ImportError(
+ f"module '{mod_name}' does not contain '{'.'.join(attributes)}'"
+ )
+
+ return obj
+
+ return doit(dotted_name)
+
@@ -700,6 +740,7 @@ Subclasses
- awsrun.plugins.accts._CachingLoaderPlugin
- AzureCLI
- awsrun.plugins.creds.aws.AbstractCrossAccount
- awsrun.plugins.creds.aws.OAuth2
- Profile
- SAML
- Default
Methods
It is perfectly acceptable to terminate the main program from
within this method. Alternatively, one can raise an exception which will
also terminate the program.
+
+
+Expand source code
+
+def instantiate(self, args):
+ """Returns an object created with options and arguments defined by the plug-in.
+
+ The `PluginManager` will invoke this method after it has completed
+ parsing the command line arguments that the plug-in defined in the
+ constructor. The `args` argument is a populated `argparse.Namespace`
+ object that contains the values of any command line arguments provided
+ by the user on the CLI.
+
+ This method also has access to the `self.parser` and `self.cfg` objects
+ that were provided in the constructor. The `self.cfg` object is useful
+ for cases where one does not want to provide a CLI flag for an option
+ defined in the configuration file. In this case, when instantiating the
+ object, you can pull values for the configuration file.
+
+ The `self.parser` object is useful if one wishes to abort the
+ instantiation of the plug-in. `argparse.ArgumentParser.error()` can be
+ used to provide an error message to the CLI user and terminate the
+ program. It is perfectly acceptable to terminate the main program from
+ within this method. Alternatively, one can raise an exception which will
+ also terminate the program.
+ """
+ raise NotImplementedError
+
@@ -996,6 +1065,61 @@ Methods
If the Accounts key does not exist in the configuration, then the
Identity plug-in will be used instead.
+
+
+Expand source code
+
+def parse_args(self, *keys, default=None):
+ """Load the plug-in and parse command line arguments passed via the CLI.
+
+ This method does not return anything, nor does it instantiate the
+ plug-in. It only loads the plug-in class and performs command line
+ argument processing for the `Plugin`. It is provided to allow one to
+ separate the parsing of all line arguments from the instantiation of
+ plug-ins. If this method is not explicitly called by the user, then it
+ will be implicitly called when `PluginManager.instantiate` is called.
+ See the `awsrun.plugmgr` documentation for the rationale.
+
+ The `keys` varargs specifies the path to the plug-in specification
+ contained with the configuration. See the `PluginManager` documentation
+ for details on the plug-in specification. If the path does not exist,
+ then the value of `default` is used instead. This default value must be
+ a dotted string pointing to a subclass of `Plugin`. For example:
+
+ pm = PluginManager(config, parser, parsed_args, unparsed_argv)
+ pm.parse_args('Accounts', default='awsrun.plugins.accts.Identity')
+
+ If the `Accounts` key does not exist in the configuration, then the
+ `awsrun.plugins.accts.Identity` plug-in will be used instead.
+ """
+ path = self._config.get(*keys, "plugin") or default
+ LOG.info("loading plug-in: %s", path)
+
+ try:
+ plugin_class = load_dotted_object(path)
+
+ except ImportError as e:
+ raise ValueError(f"Error in config: {'->'.join(keys)}->plugin: {e}") from e
+
+ if not (isclass(plugin_class) and issubclass(plugin_class, Plugin)):
+ raise TypeError(
+ f"Error in config: {'->'.join(keys)}->plugin: '{path}' is not a {Plugin}"
+ )
+
+ # Create a new config callable that points directly to the options
+ # stored in the configuration. This will make it easy for plugin authors
+ # to query the config without having to specify the key path leading up
+ # to the options section of the config.
+ cfg = partial(self._config.get, *keys, "options")
+
+ plugin = plugin_class(self._parser, cfg)
+ self.args, self.remaining_argv = self._parser.parse_known_args(
+ self.remaining_argv, self.args
+ )
+ LOG.info("parsed args=%s remaining args=%s", self.args, self.remaining_argv)
+
+ self._plugins[keys] = plugin
+
def instantiate(self, *keys, default=None, must_be=None)
@@ -1025,6 +1149,51 @@ Methods
must_be is provided, the returned value from Plugin.instantiate() must
be an instance of the type specified, otherwise a TypeError is raised.
If must_be is not provided, the returned object can be of any type.
+
+
+Expand source code
+
+def instantiate(self, *keys, default=None, must_be=None):
+ """Returns the instantiated plug-in.
+
+ This method ultimately returns the value from `Plugin.instantiate`,
+ which is passed a reference to the `PluginManager.args` object, so the
+ plug-in can use any parsed command line arguments it had requested. It
+ is usual, but not necessary, to invoke `PluginManager.parse_args` for
+ each plug-in before calling this method. This allows all command line
+ processing, and more important errors, to be complete before the actual
+ instantiation of any plug-ins.
+
+ The `keys` varargs specifies the path to the plug-in specification
+ contained with the configuration. See the `PluginManager` documentation
+ for details on the plug-in specification. If the path does not exist,
+ then the value of `default` is used instead. This default value must be
+ a dotted string pointing to a subclass of `Plugin`. For example:
+
+ pm = PluginManager(config, parser, parsed_args, unparsed_argv)
+ acct_loader = pm.instantiate(
+ 'Accounts',
+ must_be=AccountLoader,
+ default='awsrun.plugins.accts.Identity')
+
+ If the `Accounts` key does not exist in the configuration, then the
+ `awsrun.plugins.accts.Identity` plug-in will be used instead. If
+ `must_be` is provided, the returned value from `Plugin.instantiate` must
+ be an instance of the type specified, otherwise a `TypeError` is raised.
+ If `must_be` is not provided, the returned object can be of any type.
+ """
+ if keys not in self._plugins:
+ self.parse_args(*keys, default)
+
+ instance = self._plugins[keys].instantiate(self.args)
+
+ if must_be and not isinstance(instance, must_be):
+ raise TypeError(
+ f"Error in config: {'->'.join(keys)}->plugin: plugin did not build a {must_be}"
+ )
+
+ return instance
+
@@ -1072,7 +1241,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/runner.html b/docs/runner.html
index 59cf319..097e73f 100644
--- a/docs/runner.html
+++ b/docs/runner.html
@@ -3,13 +3,13 @@
-
+
awsrun.runner API documentation
-
+
@@ -503,7 +503,7 @@ Collecting Results
"""
@classmethod
- def from_cli(cls, parser, argv, cfg): # pylint: disable=unused-argument
+ def from_cli(cls, parser, argv, cfg):
"""Factory to build the command from CLI args and user configuration.
*This method is only required if the command is intended for use with
@@ -709,7 +709,7 @@ Collecting Results
try:
print(get_result(), end="", flush=True)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s: error: %s", acct, e, exc_info=True)
print(f"{acct}: error: {e}", flush=True, file=sys.stderr)
@@ -775,7 +775,7 @@ Collecting Results
return command
@classmethod
- def regional_from_cli(cls, parser, argv, cfg): # pylint: disable=unused-argument
+ def regional_from_cli(cls, parser, argv, cfg):
"""Factory to build a regional command from CLI args and user configuration.
*This method is only required if the command is intended for use with
@@ -883,7 +883,7 @@ Collecting Results
for region, get_region_result in get_result():
self.regional_collect_results(acct, region, get_region_result)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s: error: %s", acct, e, exc_info=True)
print(f"{acct}: error: {e}", flush=True, file=sys.stderr)
@@ -913,7 +913,7 @@ Collecting Results
try:
print(get_result(), end="", flush=True)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s/%s: error: %s", acct, region, e, exc_info=True)
print(f"{acct}/{region}: error: {e}", flush=True, file=sys.stderr)
@@ -944,7 +944,7 @@ Collecting Results
def collect_results(self, acct, get_result):
try:
self.results[acct] = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
self.errors[acct] = e
@@ -999,7 +999,7 @@ Collecting Results
def regional_collect_results(self, acct, region, get_result):
try:
self.results[(acct, region)] = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
self.errors[(acct, region)] = e
@@ -1241,7 +1241,7 @@ Collecting Results
session = self.session_provider.session(acct_id)
return _wrap_result(cmd.execute, session, acct)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# NOTE: exceptions thrown by a Command's execute are not
# handled in this block, but in wrap_result above. This
# block handles exceptions that occur while obtaining a
@@ -1330,7 +1330,7 @@ Collecting Results
try:
result = fn(*args, **kwargs)
return lambda: result
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
return _wrap_exception(e)
@@ -1367,6 +1367,34 @@ Functions
account ID as a string from the account object.
Accounts are processed concurrently using a worker pool. The default number
of workers is specified by the max_workers argument, which defaults to 10.
+
+
+Expand source code
+
+def execute_function(session_provider, accounts, func, key=lambda x: x, max_workers=10):
+ """Executes a function across one or more accounts concurrently.
+
+ This is a convenience function that instantiates an `AccountRunner`, wraps a
+ `func` in a `CommandFunctionAdapter`, runs the command across a list of
+ `accounts`, and then returns a tuple of dicts representing the results and
+ errors after all accounts have been processed. The returned dicts are keyed
+ by the account.
+
+ The list of `accounts` can be a simple list of strings of account IDs or it
+ can be a list of objects that represent accounts. Passing objects can be
+ useful as each object is passed to `func` when processing an account. When
+ using account objects, you must provide a `key` function that returns the
+ account ID as a string from the account object.
+
+ Accounts are processed concurrently using a worker pool. The default number
+ of workers is specified by the `max_workers` argument, which defaults to 10.
+ """
+ command = CommandFunctionAdapter(func)
+ AccountRunner(session_provider, max_workers=max_workers).run(
+ command, accounts, key=key
+ )
+ return (command.results, command.errors)
+
def regional_execute_function(session_provider, accounts, regions, func, key=<function <lambda>>, max_workers=10)
@@ -1385,6 +1413,36 @@ Functions
account ID as a string from the account object.
Accounts are processed concurrently using a worker pool. The default number
of workers is specified by the max_workers argument, which defaults to 10.
+
+
+Expand source code
+
+def regional_execute_function(
+ session_provider, accounts, regions, func, key=lambda x: x, max_workers=10
+):
+ """Executes a function across one or more accounts and regions concurrently.
+
+ This is a convenience function that instantiates an `AccountRunner`, wraps a
+ `func` in a `RegionalCommandFunctionAdapter`, runs the command across a list
+ of `accounts` and `regions`, and then returns a tuple of dicts representing
+ the results and errors after all accounts have been processed. The returned
+ dicts are keyed by a tuple of account and region.
+
+ The list of `accounts` can be a simple list of strings of account IDs or it
+ can be a list of objects that represent accounts. Passing objects can be
+ useful as each object is passed to `func` when processing an account. When
+ using account objects, you must provide a `key` function that returns the
+ account ID as a string from the account object.
+
+ Accounts are processed concurrently using a worker pool. The default number
+ of workers is specified by the `max_workers` argument, which defaults to 10.
+ """
+ command = RegionalCommandFunctionAdapter(regions, func)
+ AccountRunner(session_provider, max_workers=max_workers).run(
+ command, accounts, key=key
+ )
+ return (command.results, command.errors)
+
def max_thread_limit(count)
@@ -1407,6 +1465,45 @@ Functions
Note: while this decorator can limit the number of concurrent executions, it
will not increase the number of workers in the AccountRunner worker pool.
This is a rate limiting decorator only.
+
+
+Expand source code
+
+def max_thread_limit(count):
+ """Decorator to limit maximum number of concurrent executions.
+
+ In some cases, the author of a `Command` may wish to restrict the number of
+ concurrent executions of a command, regardless of the number of workers the
+ `AccountRunner` has been instantiated with. awsrun CLI users can specify the
+ number of workers via the `--threads` flag, which defaults to 10.
+
+ Use this with `Command.execute` or `RegionalCommand.regional_execute` to
+ guarantee concurrent executions do not exceed the specified `count`. When
+ the number of concurrent executions exceed the value, they will block until
+ an existing execution has completed. For example, the following will limit
+ concurrent executions to one:
+
+ @max_thread_limit(1)
+ def regional_execute(self, session, acct, region):
+ pass
+
+ Note: while this decorator can limit the number of concurrent executions, it
+ will not increase the number of workers in the `AccountRunner` worker pool.
+ This is a rate limiting decorator only.
+ """
+
+ def decorator(func):
+ sem = threading.BoundedSemaphore(count)
+
+ @functools.wraps(func)
+ def wrapper(*args, **kwargs):
+ with sem:
+ return func(*args, **kwargs)
+
+ return wrapper
+
+ return decorator
+
def get_paginated_resources(client, paginator, page_key, predicate=<function <lambda>>, **kwargs)
@@ -1456,6 +1553,73 @@ Functions
lambda l: l["Protocol"] in ["UDP", "TCP_UDP"],
LoadBalancer=lb_arn)
+
+
+Expand source code
+
+def get_paginated_resources(
+ client,
+ paginator,
+ page_key,
+ predicate: Callable[[dict], bool] = lambda _: True,
+ **kwargs,
+):
+ """Return the full list of boto3 resources via a paginator.
+
+ `client` is a boto3 client. `paginator` is the name (string) of the
+ paginator. `page_key` is the name (string) of the dictionary key used in
+ the paginated responses.
+
+ `predicate` is a function that determines which resources are included in
+ the list. The function takes a single argument, the resource. If it returns
+ `True`, the resource is included, otherwise it is excluded.
+
+ The remaining `kwargs` are used to specify which resources to retrieve. Some
+ paginators do not require any additional arguments, but others do to
+ restrict the size of the response.
+
+ For example, to collect the list of load balancers, one might write:
+
+ def get_lbs(client):
+ lbs = []
+ paginator = client.get_paginator("describe_load_balancers")
+ for page in paginator.paginate():
+ for lb in page["LoadBalancers"]:
+ lbs.append(lb)
+ return lbs
+
+ This can be simplified by using `get_paginated_resources`:
+
+ lbs = get_paginated_resources(client, "describe_load_balancers", "LoadBalancers")
+
+ As another example, to collect the list of UDP listeners for a load
+ balancer, one might write:
+
+ def get_listeners(client, lb_arn):
+ listeners = []
+ paginator = client.get_paginator("describe_listeners")
+ for page in paginator.paginate(LoadBalancerArn=lb_arn):
+ for listener in page["Listeners"]:
+ if listener["Protocol"] in ["UDP", "TCP_UDP"]:
+ listeners.append(listener)
+ return listeners
+
+ This, too, can be simplified using `get_paginated_resources`:
+
+ listeners = get_paginated_resources(
+ client,
+ "describe_listeners",
+ "Listeners",
+ lambda l: l["Protocol"] in ["UDP", "TCP_UDP"],
+ LoadBalancer=lb_arn)
+ """
+ resources = []
+ for page in client.get_paginator(paginator).paginate(**kwargs):
+ for resource in page[page_key]:
+ if predicate(resource):
+ resources.append(resource)
+ return resources
+
@@ -1500,7 +1664,7 @@ Classes
"""
@classmethod
- def from_cli(cls, parser, argv, cfg): # pylint: disable=unused-argument
+ def from_cli(cls, parser, argv, cfg):
"""Factory to build the command from CLI args and user configuration.
*This method is only required if the command is intended for use with
@@ -1706,7 +1870,7 @@ Classes
try:
print(get_result(), end="", flush=True)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s: error: %s", acct, e, exc_info=True)
print(f"{acct}: error: {e}", flush=True, file=sys.stderr)
@@ -1831,6 +1995,25 @@ Methods
rather once before any accounts are processed.
To provide backwards compatibility, the default implementation invokes
Command.pre_hook().
+
+
+Expand source code
+
+def pre_hook_with_context(self, context):
+ """Invoked by `AccountRunner.run` before any account processing starts.
+
+ This method is invoked only once per invocation of `AccountRunner.run`.
+ The `context` parameter is the opaque object passed as the context
+ parameter to `AccountRunner.run`. It is intended to provide access to
+ additional runtime context information for the command to leverage.
+ The method is not executed before each account is processed, but
+ rather once before any accounts are processed.
+
+ To provide backwards compatibility, the default implementation invokes
+ `Command.pre_hook`.
+ """
+ self.pre_hook()
+
def pre_hook(self)
@@ -1843,6 +2026,22 @@ Methods
executed before each account is processed, but rather once before
any accounts are processed.
The default implementation does nothing.
+
+
+Expand source code
+
+def pre_hook(self):
+ """Invoked in the default implementation of `pre_hook_with_context`
+ before any account processing starts.
+
+ This method is invoked in the event a command does not override the
+ default implementation of `pre_hook_with_context`. The method is not
+ executed before each account is processed, but rather once before
+ any accounts are processed.
+
+ The default implementation does nothing.
+ """
+
def post_hook(self)
@@ -1853,6 +2052,20 @@ Methods
It is not executed after each account is processed, but rather once
after all accounts have been processed.
The default implementation does nothing.
+
+
+Expand source code
+
+def post_hook(self):
+ """Invoked by `AccountRunner.run` after all processing has completed.
+
+ This method is invoked only once per invocation of `AccountRunner.run`.
+ It is not executed after each account is processed, but rather once
+ after all accounts have been processed.
+
+ The default implementation does nothing.
+ """
+
def execute(self, session, acct)
@@ -1901,6 +2114,54 @@ Methods
+
+
+Expand source code
+
+def execute(self, session, acct):
+ """Invoked by `AccountRunner.run` to process an account.
+
+ This method is invoked once for each account. The `session` parameter is
+ a boto3 Session object, or similar cloud SDK session-like object, with
+ credentials for the account being processed. The `acct` will be the same
+ object that was passed in the list of accounts to `AccountRunner.run`.
+
+ The return value from this method can be of any type. By default, this
+ value will be printed to the console if a `Command.collect_results`
+ implementation is not provided. If an exception is raised during the
+ execution, it will be printed to the console on standard error.
+
+ Note the following items of importance:
+
+ 1. Command authors must implement this method. There is no default
+ implementation.
+
+ 2. Recognize that this method will be invoked concurrently, so
+ modification of instance variables from within this method requires
+ synchronization. If accumulating results, define a custom
+ `Command.collect_results` which is guaranteed to be invoked
+ sequentially.
+
+ 3. Do not call `sys.exit` or the entire program will terminate. The
+ proper way to exit from this method is either by returning a value or
+ by raising an exception.
+
+ 4. Do not print directly to the console as output will be interspersed
+ with other output from other concurrently running threads processing
+ other accounts. The best practice when printing is to accumulate a
+ string buffer and return the buffer at the end of the method:
+
+ def execute(self, session, acct):
+ out = io.StringIO()
+
+ # Do stuff
+ print('This will be printed to the console eventually', file=out)
+ # Do more stuff
+
+ return out.getvalue()
+ """
+ raise NotImplementedError
+
def collect_results(self, acct, get_result)
@@ -1922,6 +2183,38 @@ Methods
to the console on standard output. If the command's execution raises an
exception, it prints the exception to standard error and logs the stack
trace at WARN log level.
+
+
+Expand source code
+
+def collect_results(self, acct, get_result):
+ """Invoked by `AccountRunner.run` after processing an account.
+
+ This method is invoked by `AccountRunner.run` after each `acct` has been
+ processed by `Command.execute`. The results from execute are provided
+ via `get_result`, which is a callable that will either return the value
+ returned by execute or raise an exception if one was raised. The `acct`
+ parameter will be the same object that was passed in the list of
+ accounts to `AccountRunner.run`.
+
+ Note: `Command.collect_results` is guaranteed to be called sequentially
+ by the main thread, so it is safe to mutate instance variables attached
+ to the `Command` object from within this method. This allows command
+ authors to safely accumulate results of processing within instance
+ variables without the need for synchronization.
+
+ The default implementation prints the return value of `Command.execute`
+ to the console on standard output. If the command's execution raises an
+ exception, it prints the exception to standard error and logs the stack
+ trace at WARN log level.
+ """
+ try:
+ print(get_result(), end="", flush=True)
+
+ except Exception as e:
+ LOG.warning("%s: error: %s", acct, e, exc_info=True)
+ print(f"{acct}: error: {e}", flush=True, file=sys.stderr)
+
@@ -2021,7 +2314,7 @@ Methods
return command
@classmethod
- def regional_from_cli(cls, parser, argv, cfg): # pylint: disable=unused-argument
+ def regional_from_cli(cls, parser, argv, cfg):
"""Factory to build a regional command from CLI args and user configuration.
*This method is only required if the command is intended for use with
@@ -2129,7 +2422,7 @@ Methods
for region, get_region_result in get_result():
self.regional_collect_results(acct, region, get_region_result)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s: error: %s", acct, e, exc_info=True)
print(f"{acct}: error: {e}", flush=True, file=sys.stderr)
@@ -2159,7 +2452,7 @@ Methods
try:
print(get_result(), end="", flush=True)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s/%s: error: %s", acct, region, e, exc_info=True)
print(f"{acct}/{region}: error: {e}", flush=True, file=sys.stderr)
@@ -2287,6 +2580,65 @@ Methods
+def regional_execute(self, session, acct, region):
+ """Invoked by `AccountRunner.run` to process an account / region pair.
+
+ This method is invoked for each account and region pair. The `session`
+ parameter is a boto3 Session object, or similar cloud SDK session-like
+ object, with credentials for the account being processed. The `acct`
+ will be the same object that was passed in the list of accounts to
+ `AccountRunner.run`. `region` is a string representing the region name
+ such as "us-east-1".
+
+ The return value can be of any type. It will be, by default, printed to
+ the console if a `RegionalCommand.regional_collect_results`
+ implementation is not provided. If an exception is raised during the
+ execution, it will be printed to the console on standard error.
+
+ Note the following items of importance:
+
+ 1. Command authors must implement this method. There is no default
+ implementation.
+
+ 2. Recognize that this method will be invoked concurrently, so
+ modification of instance variables from within this method requires
+ synchronization. If accumulating results, define a custom
+ `RegionalCommand.regional_collect_results` which is guaranteed to be
+ invoked sequentially.
+
+ 3. Although accounts are processed concurrently, the regions are
+ processed sequentially for each account. This ensures that multiple
+ regions for the same account are never executed at the same time. It
+ provides command authors a guarantee that an account and all its
+ regions will be processed sequentially. The same session object and
+ credentials are provided to this method for each region being
+ processed. Credentials could expire if processing of regions takes a
+ significant amount of time.
+
+ 4. Do not call `sys.exit` or the entire program will terminate. The
+ proper way to exit from this method is either by returning a value or
+ by raising an exception.
+
+ 5. Do not print directly to the console as output will be interspersed
+ with other output from other concurrently running threads processing
+ other accounts. The best practice when printing is to accumulate a
+ string buffer and return the buffer at the end of the method:
+
+ def regional_execute(self, session, acct, region):
+ out = io.StringIO()
+
+ # Do stuff
+ print('This will be printed to the console eventually', file=out)
+ # Do more stuff
+
+ return out.getvalue()
+ """
+ raise NotImplementedError
def regional_collect_results(self, acct, region, get_result)
@@ -2310,6 +2662,40 @@ Methods
RegionalCommand.regional_execute() to the console on standard output. If
the command's execution raises an exception, it prints the exception to
standard error and logs the stack trace at WARN log level.
+
+
+Expand source code
+
+def regional_collect_results(self, acct, region, get_result):
+ """Invoked by `AccountRunner.run` after processing an account and region.
+
+ This method is invoked by `AccountRunner.run` after each `acct` /
+ `region` pair has been processed by `RegionalCommand.regional_execute`.
+ The results from execute are provided via `get_result`, which is a
+ callable that will either return the value returned by the regional
+ execute method or raise an exception if one was raised. The `acct`
+ parameter will be the same object that was passed in the list of
+ accounts to `AccountRunner.run`.
+
+ Note: `RegionalCommand.regional_collect_results` is guaranteed to be
+ called sequentially by the main thread, so it is safe to mutate instance
+ variables attached to the `RegionalCommand` object from within this
+ method. This allows command authors to safely accumulate results of
+ processing within instance variables without the need for
+ synchronization.
+
+ The default implementation prints the return value of
+ `RegionalCommand.regional_execute` to the console on standard output. If
+ the command's execution raises an exception, it prints the exception to
+ standard error and logs the stack trace at WARN log level.
+ """
+ try:
+ print(get_result(), end="", flush=True)
+
+ except Exception as e:
+ LOG.warning("%s/%s: error: %s", acct, region, e, exc_info=True)
+ print(f"{acct}/{region}: error: {e}", flush=True, file=sys.stderr)
+
Inherited members
@@ -2367,7 +2753,7 @@ Inherited members
def collect_results(self, acct, get_result):
try:
self.results[acct] = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
self.errors[acct] = e
def run(self, cmd, accounts, key=lambda x: x, context=None):
+ """Execute a command concurrently on the specified accounts.
+
+ This method will block until all accounts have been processed. The
+ return value is the number of seconds it took to process the accounts.
+
+ The `cmd` must be a subclass of `Command`. The runner will invoke
+ `Command.pre_hook_with_context` once before it starts processing any
+ accounts passing it the `context` argument, which is simply passed
+ through as an opaque object. This can be used to pass a runtime context
+ to a `Command`.
+
+ After the pre-hook has been invoked, accounts are then processed
+ concurrently and `Command.execute` is invoked by a worker for each
+ account. As each execute method returns, the main thread will invoke
+ `Command.collect_results`, which ensures results are collected
+ sequentially. Finally, after all accounts have been processed,
+ `Command.post_hook` is called.
+
+ The specified list of `accounts` can be of any type as long as the
+ function specified by the `key` parameter returns a string representing
+ the cloud account ID when passed one of these accounts. This allows
+ users to pass any object representing an account all the way through to
+ `Command.execute`. The only contract is that a `key` function must be
+ provided, so workers can obtain the account ID, which is used to request
+ a session for the account.
+
+ For example, `accounts` could be a simple list of strings of AWS account
+ IDs. The default value of `key` is the identity function, which returns
+ the string itself satisfying the contract above. Alternatively,
+ `accounts` could be a list of dicts containing metadata for an account,
+ which would then be available for command authors in `Command.execute`.
+ If the list of accounts specified contained the following:
+
+ [
+ {'id': '100200300400', 'env': 'prod', 'status': 'active'},
+ {'id': '200300400100', 'env': 'dev', 'status': 'active'},
+ {'id': '300400100200', 'env': 'dev', 'status': 'active'},
+ ]
+
+ Then, the `key` argument must be specified as `lambda x: x['id']`, which
+ will return the account ID string satisfying the contract above.
+ Likewise, if accounts were a list of objects that contained an `acct_id`
+ attribute, `key` must be defined as `lambda x: x.acct_id` to satisfy the
+ contract. If the key function does not return a string or throws an
+ exception, then an `InvalidAccountIDError` is raised in the worker
+ thread processing the account, which will then propagate to the
+ `Command.collect_results`.
+
+ The optional `context` parameter can be any value. It is passed
+ as-is to `Command.pre_hook_with_context`. It provides a mechanism
+ for the caller to pass a runtime context to the command being
+ executed. See the [Context With Pre-Hook](#context-with-pre-hook)
+ section of the use guide for an example.
+ """
+ # This will ensure v1 users of awsrun aren't mixing v1 Command's with
+ # the v2 framework.
+ if not isinstance(cmd, Command):
+ raise TypeError(f"'{cmd}' must be a subclass of awsrun.runner.Command")
+
+ # Wrapper to ensure the user-supplied key function returns an string of
+ # digits (an AWS account id). It will throw an InvalidAccountIDError
+ # if there are any exceptions thrown from use of their key function.
+ key = _valid_key_fn(key)
+
+ start = time.time()
+ cmd.pre_hook_with_context(context)
+
+ with ThreadPoolExecutor(max_workers=self.max_workers) as pool:
+ # The worker task processes a single account. The worker task takes
+ # care to capture the result of the command's execute method. We
+ # don't want a poorly written command that raises an exception to
+ # terminate the main program, so the return value of the command's
+ # execute method or any exception raised is wrapped in a callable
+ # that is provided back to the command via its collect_results
+ # method. When the callable is later invoked, it will return the
+ # return value from execute or it will raise the caught exception.
+ def worker_task(acct):
+ try:
+ acct_id = key(acct) # Get the acct id from the account obj
+ session = self.session_provider.session(acct_id)
+ return _wrap_result(cmd.execute, session, acct)
+
+ except Exception as e:
+ # NOTE: exceptions thrown by a Command's execute are not
+ # handled in this block, but in wrap_result above. This
+ # block handles exceptions that occur while obtaining a
+ # session for the account.
+ return _wrap_exception(e)
+
+ # Submit all of the jobs for execution to the thread pool.
+ f2a = {pool.submit(worker_task, a): a for a in accounts}
+
+ # NOTE: collect_results is called by the main thread sequentially
+ # after each worker completes their task. This is a guarantee for
+ # Command authors as it allows them to safely update instance vars
+ # in the Command because it is not safe to do so in the execute
+ # method which is invoked in a concurrently running worker thread.
+ for future in as_completed(f2a):
+ acct = f2a[future]
+ cmd.collect_results(acct, future.result())
+
+ cmd.post_hook()
+ return time.time() - start
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/session/aws.html b/docs/session/aws.html
index a9d5e1d..c059efe 100644
--- a/docs/session/aws.html
+++ b/docs/session/aws.html
@@ -3,13 +3,13 @@
-
+
awsrun.session.aws API documentation
-
+
@@ -78,21 +78,23 @@ Module awsrun.session.aws
Obtain boto3 sessions and credentials via a variety of means.
Overview
-This module provides a SessionProvider interface to obtain AWS credentials via
-one of several mechanisms: the standard AWS CLI configuration files, Single Sign
-On (SSO) via federated SAML authentication, or cross-account access initiated
-from a base account. Regardless of the mechanism, the session provider is
-responsible for returning a boto3 Session that contains the credentials for a
-requested account. In some cases, those credentials are cached to limit the
-number of API calls to AWS and/or Identity Providers (IdP) should a session be
-requested for the same account again.
-There are three concrete session provider implementations included in this
+
This module provides a SessionProvider interface to obtain AWS credentials
+via one of several mechanisms: the standard AWS CLI configuration files,
+Single Sign On (SSO) via federated SAML/OIDC, or cross-account access
+initiated from a base account. Regardless of the mechanism, the session
+provider is responsible for returning a boto3 Session that contains the
+credentials for a requested account. In some cases, those credentials are
+cached to limit the number of API calls to AWS and/or Identity Providers (IdP)
+should a session be requested for the same account again.
+There are four concrete session provider implementations included in this
module:
CredsViaProfile- Credentials are obtained from AWS configuration and credential files.
CredsViaSAML- Credentials are obtained via a role assumed via SAML-based federation.
+CredsViaOAuth2ROPC- Credentials are obtained via a role assumed via OAuth2 ROPC authentication.
CredsViaCrossAccount respectively.- Credentials are obtained via a role assumed from a base account.
@@ -175,6 +177,38 @@ SAML SSO
Please refer to Using SAML-Based Federation for API Access to
AWS
for additional details on the use of federated SSO with SAML.
+OIDC via OAuth2 ROPC
+AWS supports federated SSO access via OpenID Connect (OIDC). In this scenario,
+rather than defining AWS credentials in AWS configuration files (see first
+section), the user obtains an access token from an OAuth2 authorization server
+using the Resource Owner Password Credential (ROPC) flow. The access token is
+then sent to AWS to assume an AWS role within an account to obtain credentials
+for API usage.
+To create a SessionProvider that will obtain credentials via OAuth2
+ROPC-based federated access, create an instance of CredsViaOAuth2ROPC. In
+the example below, Microsoft Entra ID (formerly Azure AD) is used as the OAuth2
+authorization server:
+# Instantiate a single session provider and reuse it
+session_provider = CredsViaOAuth2ROPC(
+ role_arn='arn:aws:iam::111222333444:role/RoleName',
+ url='https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/token"',
+ username='example@example.com',
+ password='EXAMPLE_PASSWORD',
+ oauth2_params={
+ 'client_id': 'CLIENT_ID',
+ 'scope': 'api://CLIENT_ID/AssumeRoleWithWebIdentity'
+ })
+
+# Obtain boto3 sessions for one or more accounts
+session1 = session_provider.session('111111111111')
+session2 = session_provider.session('222222222222')
+
+# Use the sessions to interact with AWS
+ec2 = session1.resource('ec2', region_name='us-east-1')
+iam = session2.resource('iam', region_name='us-east-1')
+
+Please refer to AWS OIDC federation guide
+for additional details on the use of federated access with OIDC.
Cross Account
In large companies with a significant number of AWS accounts, it is more common
to use cross-account access to simplify management of credentials and limit
@@ -205,8 +239,9 @@
Cross Account
ec2 = session1.resource('ec2', region_name='us-east-1')
iam = session2.resource('iam', region_name='us-east-1')
-In large companies, it's likely more common that SAML is used for direct access -to the base account, and then cross-account access to the remainder of accounts:
+In large companies, it's likely more common that SAML or OIDC is used for +direct access to the base account, and then cross-account access to the +remainder of accounts:
# Instantiate a session provider for the base account
session_provider = CredsViaSAML(
role='BaseRoleName', # Name (not ARN) of the AWS role to assume
@@ -224,29 +259,33 @@ Cross Account
user/role will need the AssumeRole permission, and the cross-account roles will
need to have an assume role policy document that permits access from the base
account.
-Although, the CredsViaProfile session provider can support cross-account
-access, it does require that every account must be defined in the local AWS
-files. For a corporation with hundreds of accounts that change all the time,
-this may not be practical. CredsViaCrossAccount does not use the local AWS
-files. The only use of AWS files would be limited to the base account if
-CredsViaProfile is used. If, on the other hand, SAML authentication is used
-for the base account via CredsViaSAML, then the local AWS files aren't used at
-all.
+Although, the CredsViaProfile session provider could support direct access
+to multiple accounts, it would require that every account be defined in local
+AWS credential files. For a corporation with hundreds of accounts that change
+all the time, this may not be practical. CredsViaCrossAccount does not use
+the local AWS files. The only use of AWS files would be limited to the base
+account if CredsViaProfile is used. If, on the other hand, SAML or OIDC
+authentication is used for the base account via CredsViaSAML or
+CredsViaOAuth2ROPC, then the local AWS files aren't used at all.
Please refer to Providing Access to an IAM User in Another AWS Account That You
Own
for additional details on cross-account access.
Caching
-AWS credentials obtained via SAML-based federation (assume_role_with_saml) or
-cross account access (assume_role) are cached in memory by default for 1 hour
-(3600 seconds). This means subsequent calls to SessionProvider.session for the
-same account will return the cached set of credentials unless half of the cache
-duration has transpired, in which case a new set of credentials will be
-returned. If AWS credentials are expired sooner by a local policy, then lower
-the duration in the CredsViaSAML and CredsViaCrossAccount constructors.
+AWS credentials obtained via SAML (assume_role_with_saml), OIDC
+(assume_role_with_web_identity), or cross account access (assume_role) are
+cached in memory by default for 1 hour (3600 seconds). This means subsequent
+calls to SessionProvider.session for the same account will return the cached
+set of credentials unless half of the cache duration has transpired, in which
+case a new set of credentials will be returned. If AWS credentials are expired
+sooner by a local policy, then lower the duration in the CredsViaSAML,
+CredsViaOAuth2ROPC and CredsViaCrossAccount constructors.
Likewise, the SAML assertion obtained via the IdP by CredsViaSAML is cached
for 5 minutes (300 seconds). This, too, can be adjusted in the constructor. If
the IdP expires SAML assertions sooner, then the saml_duration value must be
set appropriately.
+If using CredsViaOAuth2ROPC, the access token obtained from the OAuth2
+server is cached based on the expiration provided in the token response. There
+is no need to adjust any caching parameters in this case.
Finally, it is important to note that caching in this context has no relevance
to refreshable credentials that boto3 can provide when assuming roles. This
module does not provide refreshable credentials at this time. At some point in
@@ -259,9 +298,9 @@
Exceptions
IDPAccessDeniedExceptionRaised if the user could not be authenticated with IdP.
IDPInvalidResponseExceptionRaised if the SAML response cannot be found from IdP.
+Raised if the SAML or OAuth2 response is not valid.
IDPInvalidRoleExceptionRaised if the user does not have access to the role.
+Raised if the user does not have access to the role in a SAML flow.
AWSAssumeRoleExceptionRaised if the assume_role* calls fail.
@@ -285,16 +324,16 @@ Thread Safety
## Overview
-This module provides a `SessionProvider` interface to obtain AWS credentials via
-one of several mechanisms: the standard AWS CLI configuration files, Single Sign
-On (SSO) via federated SAML authentication, or cross-account access initiated
-from a base account. Regardless of the mechanism, the session provider is
-responsible for returning a boto3 Session that contains the credentials for a
-requested account. In some cases, those credentials are cached to limit the
-number of API calls to AWS and/or Identity Providers (IdP) should a session be
-requested for the same account again.
+This module provides a `SessionProvider` interface to obtain AWS credentials
+via one of several mechanisms: the standard AWS CLI configuration files,
+Single Sign On (SSO) via federated SAML/OIDC, or cross-account access
+initiated from a base account. Regardless of the mechanism, the session
+provider is responsible for returning a boto3 Session that contains the
+credentials for a requested account. In some cases, those credentials are
+cached to limit the number of API calls to AWS and/or Identity Providers (IdP)
+should a session be requested for the same account again.
-There are three concrete session provider implementations included in this
+There are four concrete session provider implementations included in this
module:
`CredsViaProfile`
@@ -303,6 +342,9 @@ Thread Safety
`CredsViaSAML`
: Credentials are obtained via a role assumed via SAML-based federation.
+`CredsViaOAuth2ROPC`
+: Credentials are obtained via a role assumed via OAuth2 ROPC authentication.
+
`CredsViaCrossAccount` respectively.
: Credentials are obtained via a role assumed from a base account.
@@ -397,6 +439,42 @@ Thread Safety
AWS](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_saml.html)
for additional details on the use of federated SSO with SAML.
+### OIDC via OAuth2 ROPC
+
+AWS supports federated SSO access via OpenID Connect (OIDC). In this scenario,
+rather than defining AWS credentials in AWS configuration files (see first
+section), the user obtains an access token from an OAuth2 authorization server
+using the Resource Owner Password Credential (ROPC) flow. The access token is
+then sent to AWS to assume an AWS role within an account to obtain credentials
+for API usage.
+
+To create a `SessionProvider` that will obtain credentials via OAuth2
+ROPC-based federated access, create an instance of `CredsViaOAuth2ROPC`. In
+the example below, Microsoft Entra ID (formerly Azure AD) is used as the OAuth2
+authorization server:
+
+ # Instantiate a single session provider and reuse it
+ session_provider = CredsViaOAuth2ROPC(
+ role_arn='arn:aws:iam::111222333444:role/RoleName',
+ url='https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/token"',
+ username='example@example.com',
+ password='EXAMPLE_PASSWORD',
+ oauth2_params={
+ 'client_id': 'CLIENT_ID',
+ 'scope': 'api://CLIENT_ID/AssumeRoleWithWebIdentity'
+ })
+
+ # Obtain boto3 sessions for one or more accounts
+ session1 = session_provider.session('111111111111')
+ session2 = session_provider.session('222222222222')
+
+ # Use the sessions to interact with AWS
+ ec2 = session1.resource('ec2', region_name='us-east-1')
+ iam = session2.resource('iam', region_name='us-east-1')
+
+Please refer to AWS [OIDC federation guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html)
+for additional details on the use of federated access with OIDC.
+
### Cross Account
In large companies with a significant number of AWS accounts, it is more common
@@ -430,8 +508,9 @@ Thread Safety
ec2 = session1.resource('ec2', region_name='us-east-1')
iam = session2.resource('iam', region_name='us-east-1')
-In large companies, it's likely more common that SAML is used for direct access
-to the base account, and then cross-account access to the remainder of accounts:
+In large companies, it's likely more common that SAML or OIDC is used for
+direct access to the base account, and then cross-account access to the
+remainder of accounts:
# Instantiate a session provider for the base account
session_provider = CredsViaSAML(
@@ -451,14 +530,14 @@ Thread Safety
need to have an assume role policy document that permits access from the base
account.
-Although, the `CredsViaProfile` session provider can support cross-account
-access, it does require that every account must be defined in the local AWS
-files. For a corporation with hundreds of accounts that change all the time,
-this may not be practical. `CredsViaCrossAccount` does not use the local AWS
-files. The only use of AWS files would be limited to the base account if
-`CredsViaProfile` is used. If, on the other hand, SAML authentication is used
-for the base account via `CredsViaSAML`, then the local AWS files aren't used at
-all.
+Although, the `CredsViaProfile` session provider could support direct access
+to multiple accounts, it would require that every account be defined in local
+AWS credential files. For a corporation with hundreds of accounts that change
+all the time, this may not be practical. `CredsViaCrossAccount` does not use
+the local AWS files. The only use of AWS files would be limited to the base
+account if `CredsViaProfile` is used. If, on the other hand, SAML or OIDC
+authentication is used for the base account via `CredsViaSAML` or
+`CredsViaOAuth2ROPC`, then the local AWS files aren't used at all.
Please refer to [Providing Access to an IAM User in Another AWS Account That You
Own](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_aws-accounts.html)
@@ -466,19 +545,24 @@ Thread Safety
## Caching
-AWS credentials obtained via SAML-based federation (assume_role_with_saml) or
-cross account access (assume_role) are cached in memory by default for 1 hour
-(3600 seconds). This means subsequent calls to `SessionProvider.session` for the
-same account will return the cached set of credentials unless half of the cache
-duration has transpired, in which case a new set of credentials will be
-returned. If AWS credentials are expired sooner by a local policy, then lower
-the duration in the `CredsViaSAML` and `CredsViaCrossAccount` constructors.
+AWS credentials obtained via SAML (assume_role_with_saml), OIDC
+(assume_role_with_web_identity), or cross account access (assume_role) are
+cached in memory by default for 1 hour (3600 seconds). This means subsequent
+calls to `SessionProvider.session` for the same account will return the cached
+set of credentials unless half of the cache duration has transpired, in which
+case a new set of credentials will be returned. If AWS credentials are expired
+sooner by a local policy, then lower the duration in the `CredsViaSAML`,
+`CredsViaOAuth2ROPC` and `CredsViaCrossAccount` constructors.
Likewise, the SAML assertion obtained via the IdP by `CredsViaSAML` is cached
for 5 minutes (300 seconds). This, too, can be adjusted in the constructor. If
the IdP expires SAML assertions sooner, then the `saml_duration` value must be
set appropriately.
+If using `CredsViaOAuth2ROPC`, the access token obtained from the OAuth2
+server is cached based on the expiration provided in the token response. There
+is no need to adjust any caching parameters in this case.
+
Finally, it is important to note that caching in this context has no relevance
to refreshable credentials that boto3 can provide when assuming roles. This
module does not provide refreshable credentials at this time. At some point in
@@ -494,10 +578,10 @@ Thread Safety
: Raised if the user could not be authenticated with IdP.
`IDPInvalidResponseException`
-: Raised if the SAML response cannot be found from IdP.
+: Raised if the SAML or OAuth2 response is not valid.
`IDPInvalidRoleException`
-: Raised if the user does not have access to the role.
+: Raised if the user does not have access to the role in a SAML flow.
`AWSAssumeRoleException`
: Raised if the assume_role* calls fail.
@@ -523,7 +607,7 @@ Thread Safety
import requests
from bs4 import BeautifulSoup
-from awsrun.cache import ExpiringValue
+from awsrun.cache import ExpiringValue, ValueWithExpiry
from awsrun.session import SessionProvider
LOG = logging.getLogger(__name__)
@@ -535,7 +619,13 @@ Thread Safety
This class cannot be instantiated directly. Users of this class must provide
an implementation for `credentials` and must invoke its constructor. Caching
is based on the IAM `role` being assumed and the the account ID. Credentials
- are cached for `duration` seconds, which defaults to 1 hour.
+ are cached for the expiry returned from AWS assume_role calls.
+
+ For backwards compatibility, the `duration` parameter is provided to
+ specify the maximum cache duration. If the credentials returned from AWS
+ have an expiration, then that expiration is used to determine when to
+ refresh the credentials. If not, then the `duration` parameter is used to
+ determine when to refresh the credentials.
"""
def __init__(self, role, duration=3600):
@@ -565,7 +655,10 @@ Thread Safety
# the lock may seem redundant, but it's safer to make this explicit.
ev = self._creds.setdefault(
(acct_id, self._role),
- ExpiringValue(lambda: self.credentials(acct_id), self._duration / 2),
+ ExpiringValue(
+ lambda: self._expiry_value(self.credentials(acct_id)),
+ self._duration,
+ ),
)
creds = ev.value()
@@ -576,6 +669,22 @@ Thread Safety
aws_session_token=creds["SessionToken"],
)
+ # This function is used to wrap the credentials returned by the subclass's
+ # `credentials` method in a ValueWithExpiry object. If the credentials
+ # include an "Expiration" key, then the expiration time is used to set the
+ # expiration time of the ValueWithExpiry object. If not, then the
+ # credentials are returned as-is and the ExpiringValue will use the
+ # max_age specified in the constructor.
+ @staticmethod
+ def _expiry_value(creds):
+ if not creds.get("Expiration"):
+ return creds
+
+ # Subtract 5 mins from the expiration time to avoid edge cases where the
+ # credentials are expired by the time they are used.
+ expiration = creds["Expiration"].timestamp() - 300
+ return ValueWithExpiry(creds, expires_at=expiration)
+
def credentials(self, acct_id):
"""Returns a dict containing AWS credentials for the requested account.
@@ -629,6 +738,122 @@ Thread Safety
return boto3.Session()
+class CredsViaOAuth2ROPC(CachingSessionProvider):
+ """A session provider that uses OAuth2 ROPC authentication.
+
+ This class relies on an OAuth2 authorization server using the Resource
+ Owner Password Credential (ROPC) flow to provide an access token to an
+ authenticated user, which is then sent to AWS to obtain the credentials for
+ an AWS account.
+
+ The `role` is the IAM role name (not ARN) to assume in the AWS account.
+ This is used along with the account to construct the ARN when making the
+ underlying AWS `assume_role_with_web_identity` API call.
+
+ The `url` is the URL to the OAuth2 server that will provide a token to the
+ user once they have authenticated. The client will authenticate using the
+ `username` and `password` parameters. If additional OAuth2 parameters must
+ be provided to the server in the authentication request, specify a dict of
+ parameter/value pairs via the `oauth2_params` argument. If additional HTTP
+ headers must be provided, specify a dict of header/value pairs via the
+ `headers` argument. To disable certificate verification, which is strongly
+ discouraged, set `no_verify` to `True`.
+
+ The AWS credentials for the role and account are cached for `duration`
+ seconds, which defaults to 1 hour. Likewise, the token obtained from the
+ OAuth2 server is cached based on the expiration provided in the token.
+ """
+
+ def __init__(
+ self,
+ role,
+ url,
+ username,
+ password,
+ headers=None,
+ oauth2_params=None,
+ duration=3600,
+ no_verify=False,
+ ):
+ super().__init__(role, duration)
+ self._role = role
+ self._url = url
+ self._username = username
+ self._password = password
+ self._headers = {} if headers is None else headers
+ self._oauth2_params = {} if oauth2_params is None else oauth2_params
+ self._no_verify = no_verify
+
+ # `max_age` isn't used as we use the expiration that is provided in
+ # token response. While, `ExpriringValue` requires a max_age, it is
+ # only used if the return value from the `refresh_fn` isn't
+ # a `ValueWithExpiry`, which ours always is. For safety, we'll set
+ # it to 60 seconds just in case, so we never overload the server.
+ self._cached_token = ExpiringValue(self._request_token, max_age=60)
+
+ def token(self, refresh=False):
+ """Returns an OAuth2 access token from the OAuth2 server.
+
+ This value is cached by default. If refresh is True, the value is
+ refreshed first, then returned. See the module documentation for the
+ exceptions that may be raised.
+ """
+ return self._cached_token.value(refresh)
+
+ def _request_token(self):
+ """Returns a non-cached OAuth2 access token from the OAuth2 server.
+
+ See the module documentation for the exceptions that may be raised.
+ """
+ LOG.info("Fetching OAuth2 access token")
+ with requests.Session() as s:
+ s.headers.update(self._headers)
+ form = {
+ "userName": self._username,
+ "password": self._password,
+ "grant_type": "password",
+ }
+ form.update(self._oauth2_params)
+ resp = s.post(self._url, data=form, verify=not self._no_verify)
+
+ if resp.status_code == 400: # RFC6749 section 5.2
+ try:
+ desc = resp.json().get("error_description", "unknown error")
+ except ValueError:
+ desc = f"non-JSON response: {resp.text}"
+ raise IDPAccessDeniedException(f"Could not authenticate: {desc}")
+ if not 200 <= resp.status_code < 300:
+ raise IDPInvalidResponseException(
+ f"{resp.status_code} response from {self._url}"
+ )
+
+ token_data = resp.json()
+ access_token = token_data.get("access_token")
+ expires_in = token_data.get("expires_in")
+
+ if not access_token or not expires_in:
+ raise IDPInvalidResponseException(
+ "Cannot extract access token or expiration from response"
+ )
+
+ return ValueWithExpiry(access_token, ttl=expires_in)
+
+ def credentials(self, acct_id):
+ role_arn = f"arn:aws:iam::{acct_id}:role/{self._role}"
+ LOG.info("Assuming role with web identity for %s", role_arn)
+ assumed_role = boto3.client("sts").assume_role_with_web_identity(
+ RoleArn=role_arn,
+ RoleSessionName=f"AWSRunSession-{self._username}",
+ WebIdentityToken=self.token(),
+ DurationSeconds=self._duration,
+ )
+
+ if not assumed_role:
+ raise AWSAssumeRoleException(f"Cannot assume role: {role_arn}")
+
+ return assumed_role["Credentials"]
+
+
class CredsViaSAML(CachingSessionProvider):
"""A session provider that uses federated SAML authentication.
@@ -637,7 +862,7 @@ Thread Safety
credentials for an AWS account.
The `role` is the name of the IAM role (not the ARN) to assume in the AWS
- account. This is used when making the underlying AWS assume_role_with_saml
+ account. This is used when making the underlying AWS `assume_role_with_saml`
API call.
The `url` is the URL to the IdP web server that will provide a SAML
@@ -865,7 +1090,12 @@ Classes
This class cannot be instantiated directly. Users of this class must provide
an implementation for credentials and must invoke its constructor. Caching
is based on the IAM role being assumed and the the account ID. Credentials
-are cached for duration seconds, which defaults to 1 hour.
+are cached for the expiry returned from AWS assume_role calls.
+For backwards compatibility, the duration parameter is provided to
+specify the maximum cache duration. If the credentials returned from AWS
+have an expiration, then that expiration is used to determine when to
+refresh the credentials. If not, then the duration parameter is used to
+determine when to refresh the credentials.
Expand source code
@@ -876,7 +1106,13 @@ Classes
This class cannot be instantiated directly. Users of this class must provide
an implementation for `credentials` and must invoke its constructor. Caching
is based on the IAM `role` being assumed and the the account ID. Credentials
- are cached for `duration` seconds, which defaults to 1 hour.
+ are cached for the expiry returned from AWS assume_role calls.
+
+ For backwards compatibility, the `duration` parameter is provided to
+ specify the maximum cache duration. If the credentials returned from AWS
+ have an expiration, then that expiration is used to determine when to
+ refresh the credentials. If not, then the `duration` parameter is used to
+ determine when to refresh the credentials.
"""
def __init__(self, role, duration=3600):
@@ -906,7 +1142,10 @@ Classes
# the lock may seem redundant, but it's safer to make this explicit.
ev = self._creds.setdefault(
(acct_id, self._role),
- ExpiringValue(lambda: self.credentials(acct_id), self._duration / 2),
+ ExpiringValue(
+ lambda: self._expiry_value(self.credentials(acct_id)),
+ self._duration,
+ ),
)
creds = ev.value()
@@ -917,6 +1156,22 @@ Classes
aws_session_token=creds["SessionToken"],
)
+ # This function is used to wrap the credentials returned by the subclass's
+ # `credentials` method in a ValueWithExpiry object. If the credentials
+ # include an "Expiration" key, then the expiration time is used to set the
+ # expiration time of the ValueWithExpiry object. If not, then the
+ # credentials are returned as-is and the ExpiringValue will use the
+ # max_age specified in the constructor.
+ @staticmethod
+ def _expiry_value(creds):
+ if not creds.get("Expiration"):
+ return creds
+
+ # Subtract 5 mins from the expiration time to avoid edge cases where the
+ # credentials are expired by the time they are used.
+ expiration = creds["Expiration"].timestamp() - 300
+ return ValueWithExpiry(creds, expires_at=expiration)
+
def credentials(self, acct_id):
"""Returns a dict containing AWS credentials for the requested account.
@@ -935,6 +1190,7 @@ Ancestors
Subclasses
Methods
@@ -951,6 +1207,41 @@ Methods
event this method is invoked multiple times for the same account. Users
are guaranteed that the credentials will be valid for half of the cache
duration time specified in the constructor.
+
+
+Expand source code
+
+def session(self, acct_id):
+ """Returns a boto3 Session with credentials for the requested account.
+
+ The `acct_id` is a string containing the AWS account ID. The returned
+ boto3 Session object is ready to use and loaded with the requested
+ credentials.
+
+ The credentials loaded into the boto3 Session object are cached in the
+ event this method is invoked multiple times for the same account. Users
+ are guaranteed that the credentials will be valid for half of the cache
+ duration time specified in the constructor.
+ """
+ with self._lock:
+ # setdefault is technically atomic in cpython 2.7 or 3.2 higher, so
+ # the lock may seem redundant, but it's safer to make this explicit.
+ ev = self._creds.setdefault(
+ (acct_id, self._role),
+ ExpiringValue(
+ lambda: self._expiry_value(self.credentials(acct_id)),
+ self._duration,
+ ),
+ )
+
+ creds = ev.value()
+
+ return boto3.Session(
+ aws_access_key_id=creds["AccessKeyId"],
+ aws_secret_access_key=creds["SecretAccessKey"],
+ aws_session_token=creds["SessionToken"],
+ )
+
def credentials(self, acct_id)
@@ -961,6 +1252,21 @@ Methods
dict must include the following keys: "AccessKeyId", "SecretAccessKey",
and "SessionToken". This dict will be cached by the session provider.
Refer to the module documentation for the exceptions that may be raised.
+
+
+Expand source code
+
+def credentials(self, acct_id):
+ """Returns a dict containing AWS credentials for the requested account.
+
+ The `acct_id` is a string containing the AWS account ID. The returned
+ dict must include the following keys: "AccessKeyId", "SecretAccessKey",
+ and "SessionToken". This dict will be cached by the session provider.
+
+ Refer to the module documentation for the exceptions that may be raised.
+ """
+ raise NotImplementedError
+
@@ -1049,6 +1355,190 @@ Inherited members
+
+class CredsViaOAuth2ROPC
+(role, url, username, password, headers=None, oauth2_params=None, duration=3600, no_verify=False)
+
+A session provider that uses OAuth2 ROPC authentication.
+This class relies on an OAuth2 authorization server using the Resource
+Owner Password Credential (ROPC) flow to provide an access token to an
+authenticated user, which is then sent to AWS to obtain the credentials for
+an AWS account.
+The role is the IAM role name (not ARN) to assume in the AWS account.
+This is used along with the account to construct the ARN when making the
+underlying AWS assume_role_with_web_identity API call.
+The url is the URL to the OAuth2 server that will provide a token to the
+user once they have authenticated. The client will authenticate using the
+username and password parameters. If additional OAuth2 parameters must
+be provided to the server in the authentication request, specify a dict of
+parameter/value pairs via the oauth2_params argument. If additional HTTP
+headers must be provided, specify a dict of header/value pairs via the
+headers argument. To disable certificate verification, which is strongly
+discouraged, set no_verify to True.
+The AWS credentials for the role and account are cached for duration
+seconds, which defaults to 1 hour. Likewise, the token obtained from the
+OAuth2 server is cached based on the expiration provided in the token.
+Expand source code
+
+class CredsViaOAuth2ROPC(CachingSessionProvider):
+ """A session provider that uses OAuth2 ROPC authentication.
+
+ This class relies on an OAuth2 authorization server using the Resource
+ Owner Password Credential (ROPC) flow to provide an access token to an
+ authenticated user, which is then sent to AWS to obtain the credentials for
+ an AWS account.
+
+ The `role` is the IAM role name (not ARN) to assume in the AWS account.
+ This is used along with the account to construct the ARN when making the
+ underlying AWS `assume_role_with_web_identity` API call.
+
+ The `url` is the URL to the OAuth2 server that will provide a token to the
+ user once they have authenticated. The client will authenticate using the
+ `username` and `password` parameters. If additional OAuth2 parameters must
+ be provided to the server in the authentication request, specify a dict of
+ parameter/value pairs via the `oauth2_params` argument. If additional HTTP
+ headers must be provided, specify a dict of header/value pairs via the
+ `headers` argument. To disable certificate verification, which is strongly
+ discouraged, set `no_verify` to `True`.
+
+ The AWS credentials for the role and account are cached for `duration`
+ seconds, which defaults to 1 hour. Likewise, the token obtained from the
+ OAuth2 server is cached based on the expiration provided in the token.
+ """
+
+ def __init__(
+ self,
+ role,
+ url,
+ username,
+ password,
+ headers=None,
+ oauth2_params=None,
+ duration=3600,
+ no_verify=False,
+ ):
+ super().__init__(role, duration)
+ self._role = role
+ self._url = url
+ self._username = username
+ self._password = password
+ self._headers = {} if headers is None else headers
+ self._oauth2_params = {} if oauth2_params is None else oauth2_params
+ self._no_verify = no_verify
+
+ # `max_age` isn't used as we use the expiration that is provided in
+ # token response. While, `ExpriringValue` requires a max_age, it is
+ # only used if the return value from the `refresh_fn` isn't
+ # a `ValueWithExpiry`, which ours always is. For safety, we'll set
+ # it to 60 seconds just in case, so we never overload the server.
+ self._cached_token = ExpiringValue(self._request_token, max_age=60)
+
+ def token(self, refresh=False):
+ """Returns an OAuth2 access token from the OAuth2 server.
+
+ This value is cached by default. If refresh is True, the value is
+ refreshed first, then returned. See the module documentation for the
+ exceptions that may be raised.
+ """
+ return self._cached_token.value(refresh)
+
+ def _request_token(self):
+ """Returns a non-cached OAuth2 access token from the OAuth2 server.
+
+ See the module documentation for the exceptions that may be raised.
+ """
+ LOG.info("Fetching OAuth2 access token")
+ with requests.Session() as s:
+ s.headers.update(self._headers)
+ form = {
+ "userName": self._username,
+ "password": self._password,
+ "grant_type": "password",
+ }
+ form.update(self._oauth2_params)
+ resp = s.post(self._url, data=form, verify=not self._no_verify)
+
+ if resp.status_code == 400: # RFC6749 section 5.2
+ try:
+ desc = resp.json().get("error_description", "unknown error")
+ except ValueError:
+ desc = f"non-JSON response: {resp.text}"
+ raise IDPAccessDeniedException(f"Could not authenticate: {desc}")
+ if not 200 <= resp.status_code < 300:
+ raise IDPInvalidResponseException(
+ f"{resp.status_code} response from {self._url}"
+ )
+
+ token_data = resp.json()
+ access_token = token_data.get("access_token")
+ expires_in = token_data.get("expires_in")
+
+ if not access_token or not expires_in:
+ raise IDPInvalidResponseException(
+ "Cannot extract access token or expiration from response"
+ )
+
+ return ValueWithExpiry(access_token, ttl=expires_in)
+
+ def credentials(self, acct_id):
+ role_arn = f"arn:aws:iam::{acct_id}:role/{self._role}"
+ LOG.info("Assuming role with web identity for %s", role_arn)
+ assumed_role = boto3.client("sts").assume_role_with_web_identity(
+ RoleArn=role_arn,
+ RoleSessionName=f"AWSRunSession-{self._username}",
+ WebIdentityToken=self.token(),
+ DurationSeconds=self._duration,
+ )
+
+ if not assumed_role:
+ raise AWSAssumeRoleException(f"Cannot assume role: {role_arn}")
+
+ return assumed_role["Credentials"]
+
+Ancestors
+
+Methods
+
+
+def token(self, refresh=False)
+-
+
Returns an OAuth2 access token from the OAuth2 server.
+This value is cached by default. If refresh is True, the value is
+refreshed first, then returned.
+See the module documentation for the
+exceptions that may be raised.
+Expand source code
+
+def token(self, refresh=False):
+ """Returns an OAuth2 access token from the OAuth2 server.
+
+ This value is cached by default. If refresh is True, the value is
+ refreshed first, then returned. See the module documentation for the
+ exceptions that may be raised.
+ """
+ return self._cached_token.value(refresh)
+
+
+
+Inherited members
+
+CachingSessionProvider:
+
+credentialssession
+
+
class CredsViaSAML
(role, url, auth, http_method, headers=None, duration=3600, saml_duration=300, no_verify=False)
@@ -1059,7 +1549,7 @@ Inherited members
assertion to an authenticated user, which is then sent to AWS to obtain the
credentials for an AWS account.
The role is the name of the IAM role (not the ARN) to assume in the AWS
-account. This is used when making the underlying AWS assume_role_with_saml
+account. This is used when making the underlying AWS assume_role_with_saml
API call.
The url is the URL to the IdP web server that will provide a SAML
assertion to the user once they have authenticated. The HTTP authentication
@@ -1087,7 +1577,7 @@
Inherited members
credentials for an AWS account.
The `role` is the name of the IAM role (not the ARN) to assume in the AWS
- account. This is used when making the underlying AWS assume_role_with_saml
+ account. This is used when making the underlying AWS `assume_role_with_saml`
API call.
The `url` is the URL to the IdP web server that will provide a SAML
@@ -1239,6 +1729,19 @@ Methods
refreshed first, then returned.
See the module documentation for the
exceptions that may be raised.
+
+
+Expand source code
+
+def assertion(self, refresh=False):
+ """Returns a SAML assertion from the IdP.
+
+ This value is cached by default. If refresh is True, the value is
+ refreshed first, then returned. See the module documentation for the
+ exceptions that may be raised.
+ """
+ return self._cached_saml.value(refresh)
+
Inherited members
@@ -1438,6 +1941,7 @@ Index
Quick Start
@@ -1465,6 +1969,12 @@ CredsViaProfile
+CredsViaOAuth2ROPC
+
+token
+
+
CredsViaSAML
assertion
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/session/azure.html b/docs/session/azure.html
index fddd162..1b45428 100644
--- a/docs/session/azure.html
+++ b/docs/session/azure.html
@@ -3,13 +3,13 @@
-
+
awsrun.session.azure API documentation
-
+
@@ -204,6 +204,7 @@ Thread Safety
token, to populate the token cache or refresh it, before allowing other threads
to proceed concurrently.
"""
+
import functools
import threading
@@ -261,9 +262,6 @@ Thread Safety
return wrapper
-# pylint: disable=too-few-public-methods
-
-
class CredsViaAzureDefault(SessionProvider):
"""A session provider that obtains credentials from a variety of sources.
@@ -540,7 +538,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/docs/session/index.html b/docs/session/index.html
index f4b993b..600939f 100644
--- a/docs/session/index.html
+++ b/docs/session/index.html
@@ -3,13 +3,13 @@
-
+
awsrun.session API documentation
-
+
@@ -198,6 +198,19 @@ Methods
The acct_id is a string representing an account within a CSP. The
returned session object is ready to use and loaded with the requested
credentials.
+
+
+Expand source code
+
+def session(self, acct_id):
+ """Returns a session with credentials for the requested account.
+
+ The `acct_id` is a string representing an account within a CSP. The
+ returned session object is ready to use and loaded with the requested
+ credentials.
+ """
+ raise NotImplementedError
+
@@ -237,7 +250,7 @@
-Generated by pdoc 0.11.1.
+Generated by pdoc 0.11.6.
diff --git a/pyproject.toml b/pyproject.toml
index 4d82f71..3093b18 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -50,7 +50,6 @@ dx-status = ["colorama", "asciichartpy>=1.5.25", "pysparklines>=1.0"]
# dependencies for development
dev = [
"black",
- "flake8",
"freezegun",
"isort",
"pdoc3",
@@ -67,3 +66,6 @@ homepage = "https://github.com/fidelity/awsrun"
[tool.setuptools.dynamic]
version = { attr = "awsrun.__version__" }
+
+[tool.ruff.lint]
+ignore = ["SIM117", "BLE001", "SIM103", "FA100", "TRY002", "EXE002", "TRY004"]
diff --git a/src/awsrun/__init__.py b/src/awsrun/__init__.py
index a8d690d..e7b6f13 100644
--- a/src/awsrun/__init__.py
+++ b/src/awsrun/__init__.py
@@ -122,4 +122,4 @@
"""
name = "awsrun"
-__version__ = "3.2.1"
+__version__ = "3.3.0"
diff --git a/src/awsrun/acctload.py b/src/awsrun/acctload.py
index 3115714..4d3c2ee 100644
--- a/src/awsrun/acctload.py
+++ b/src/awsrun/acctload.py
@@ -320,7 +320,7 @@ def __init__(
)
self.CustomAccount._str_template = ( # type: ignore
self.str_template
- ) # pylint: disable=protected-access
+ )
self.accts, self.attrs = self._parse(accts)
LOG.info(
@@ -949,6 +949,9 @@ def __call__(self, text):
return json.loads(text, **self.kwargs)
+_DEFAULT_JSON_PARSER = JSONParser()
+
+
class YAMLParser:
"""Returns a list or dict from a buffer of YAML-formatted text.
@@ -1050,7 +1053,7 @@ class URLAccountLoader(MetaAccountLoader):
def __init__(
self,
url,
- parser=JSONParser(),
+ parser=_DEFAULT_JSON_PARSER,
auth=None,
max_age=0,
id_attr="id",
@@ -1127,10 +1130,10 @@ def __getattr__(self, name):
return value
def __eq__(self, other):
- return self._attrs == other._attrs # pylint: disable=protected-access
+ return self._attrs == other._attrs
def __repr__(self):
- pairs = (f"{k}={repr(v)}" for k, v in self._attrs.items())
+ pairs = (f"{k}={v!r}" for k, v in self._attrs.items())
return f"Account({', '.join(pairs)})"
def __str__(self):
@@ -1183,7 +1186,7 @@ def _convert_keys_to_valid_attribute_names(d):
restrictions, this function munges any dict keys that are not valid
attribute names. For efficiency, the dictionary is modified in place.
"""
- invalid_keys = [k for k in d.keys() if not k.isidentifier() or keyword.iskeyword(k)]
+ invalid_keys = [k for k in d if not k.isidentifier() or keyword.iskeyword(k)]
for key in invalid_keys:
d[_make_valid_attribute_name(key)] = d.pop(key)
diff --git a/src/awsrun/cache.py b/src/awsrun/cache.py
index 7c1a688..44cc92f 100644
--- a/src/awsrun/cache.py
+++ b/src/awsrun/cache.py
@@ -28,6 +28,28 @@
is before the value would have expired, and thus the cached result is returned.
The third value, however, is ten seconds later because by the time the third
invocation of `value` took place, the original value expired after 10 seconds.
+
+## Dynamic Expiration
+
+In some cases, the expiration time is only known after the value has been
+obtained (e.g., OAuth2 tokens include expiration in the response). Use
+`ValueWithExpiry` to wrap the value with a custom expiration. This overrides
+the `max_age` specified in the `ExpiringValue` constructor. For example:
+
+ >>> def fetch_token():
+ ... token = {'access_token': 'EXAMPLE_TOKEN', 'expires_in': 3600}
+ ... return ValueWithExpiry(token, ttl=token['expires_in'])
+ >>> ev = ExpiringValue(refresh_fn=fetch_token, max_age=0)
+ >>> ev.value()
+ {'access_token': 'EXAMPLE_TOKEN', 'expires_in': 3600}
+
+The `ValueWithExpiry` wrapper supports both TTL (time-to-live in seconds) and
+absolute timestamps via the `expires_at` parameter.
+
+When using dynamic expiry, `max_age` becomes relevant if the refresh function
+can return a mix of both plain values and `ValueWithExpiry` instances. When
+a plain value is returned, it is cached for `max_age`. If only dynamic expiry
+is used, then `max_age` is not used.
"""
import json
@@ -39,17 +61,53 @@
LOG = logging.getLogger(__name__)
+class ValueWithExpiry:
+ """Wrapper to return a value with a custom expiration time.
+
+ Use this when your refresh function needs to specify its own expiration
+ rather than using the default max_age. The expiry can be specified as
+ either a TTL in seconds or an absolute timestamp.
+
+ Example usage with TTL::
+
+ def refresh_oauth_token():
+ token = fetch_token() # Returns {'access_token': '...', 'expires_in': 3600}
+ return ValueWithExpiry(token, ttl=token['expires_in'])
+
+ Example usage with absolute timestamp::
+
+ def refresh_oauth_token():
+ token = fetch_token() # Returns {'access_token': '...', 'expires_at': 1702915200}
+ return ValueWithExpiry(token, expires_at=token['expires_at'])
+ """
+
+ def __init__(self, value, *, ttl=None, expires_at=None):
+ if ttl is None and expires_at is None:
+ raise ValueError("Must specify either ttl or expires_at")
+ if ttl is not None and expires_at is not None:
+ raise ValueError("Cannot specify both ttl and expires_at")
+
+ self.value = value
+ self.expires_at = expires_at if expires_at else time.time() + ttl
+
+ def __repr__(self):
+ return f"ValueWithExpiry(value={self.value!r}, expires_at={self.expires_at})"
+
+
class AbstractExpiringValue:
"""Abstract base class to represent a value that expires.
An `AbstractExpiringValue` represents a lazily loaded value that will expire
over time. The constructor takes a `refresh_fn` function of zero arguments,
- which is called to obtain the value to be cached for `max_age` seconds.
+ which is called to obtain the value to be cached. The value is cached for
+ `max_age` seconds unless the `refresh_fn` returns a `ValueWithExpiry` object
+ that specifies a custom expiration time. If so, the default `max_age` is
+ ignored.
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
- method is called. This class is thread-safe. Subclasses must provide
+ method is called. The `value` method is thread-safe. Subclasses must provide
implementations for `is_expired`, `load`, and `save`.
"""
@@ -66,19 +124,35 @@ def value(self, refresh=False):
invocations of this method will return the cached value until it
expires. If you set `refresh` parameter to `True`, the value will be
refreshed and the expiration will be reset before being returned.
+
+ This method is thread-safe.
"""
with self._lock:
if not refresh and not self.is_expired():
return self.load()
- value = self._refresh_fn()
- self.save(value)
+ result = self._refresh_fn()
+
+ # Unwrap and extract expiry if ValueWithExpiry
+ if isinstance(result, ValueWithExpiry):
+ value = result.value
+ expiry = result.expires_at
+ else:
+ value = result
+ expiry = None
+
+ self.save(value, expiry)
LOG.info("refreshed data and saved in cache")
return value
def is_expired(self):
"""Returns `True` if the value needs to be refreshed, `False` otherwise.
+ A value needs to be refreshed when it has expired. This is determined
+ by the implementation of this method. By default, the value is cached for
+ `max_age` seconds or the dynamic expiry specified by the `refresh_fn`
+ return value (if it is a `ValueWithExpiry` instance).
+
If this returns `True` during the invocation of
`AbstractExpiringValue.value`, the `refresh_fn` will be called, followed
by `save`, to renew the cached value. When this returns `False`, `load`
@@ -95,12 +169,14 @@ def load(self):
"""
raise NotImplementedError
- def save(self, value):
+ def save(self, value, expiry=None):
"""Saves the value to the cache.
If `is_expired` returns `True` during the invocation of
`AbstractExpiringValue.value`, this method is invoked to save the new
- value to the cache.
+ value to the cache. If `expiry` is provided, it specifies the absolute
+ timestamp when the value should expire. Otherwise, the default `max_age`
+ should be used.
"""
raise NotImplementedError
@@ -116,7 +192,7 @@ class ExpiringValue(AbstractExpiringValue):
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
- method is called. This class is thread-safe.
+ method is called. The `value` method is thread-safe.
"""
def __init__(self, refresh_fn, max_age):
@@ -131,10 +207,10 @@ def load(self):
LOG.debug("Loading data from cache")
return self._value
- def save(self, value):
- LOG.debug("Saving value to cache")
+ def save(self, value, expiry=None):
self._value = value
- self._expiry = time.time() + self._max_age
+ self._expiry = expiry if expiry is not None else time.time() + self._max_age
+ LOG.debug("Saving value to cache, will expire at %s", time.ctime(self._expiry))
class PersistentExpiringValue(ExpiringValue):
@@ -149,17 +225,24 @@ class PersistentExpiringValue(ExpiringValue):
At the time of instantiation, the value is not retrieved, it is only
retrieved the first time the value method is invoked. Likewise, the value is
not refreshed at the time it expires, but only the next time the value
- method is called. This class is thread-safe. If the value cannot be
+ method is called. The `value` method is thread-safe. If the value cannot be
persisted as JSON, a TypeError is thrown.
"""
def __init__(self, refresh_fn, path, max_age):
super().__init__(refresh_fn, max_age)
self._path = path if isinstance(path, Path) else Path(path)
+ self._expiry_path = self._path.with_suffix(self._path.suffix + ".expiry")
def is_expired(self):
if not self._path.exists():
return True
+ # Check for explicit expiry file first
+ if self._expiry_path.exists():
+ expiry = float(self._expiry_path.read_text(encoding="utf-8").strip())
+ return time.time() > expiry
+
+ # Otherwise fall back to file modification time + max_age
last_modification = self._path.stat().st_mtime
return time.time() > last_modification + self._max_age
@@ -168,9 +251,9 @@ def load(self):
with self._path.open("r", encoding="utf-8") as file:
return json.load(file)
- def save(self, value):
- # No need to persist the file if max_age is 0 seconds.
- if self._max_age == 0:
+ def save(self, value, expiry=None):
+ # No need to persist the file if max_age is 0 seconds (and not dynamic).
+ if self._max_age == 0 and expiry is None:
return
LOG.debug("Saving data to cache file %s", self._path)
@@ -180,3 +263,9 @@ def save(self, value):
# Pathlib.replace uses os.replace which is atomic on POSIX systems
tmp.replace(self._path)
+
+ # Write or remove expiry file
+ if expiry is not None:
+ self._expiry_path.write_text(str(expiry), encoding="utf-8")
+ elif self._expiry_path.exists():
+ self._expiry_path.unlink()
diff --git a/src/awsrun/cli.py b/src/awsrun/cli.py
index 13396f5..18bc143 100755
--- a/src/awsrun/cli.py
+++ b/src/awsrun/cli.py
@@ -33,7 +33,7 @@
For the remainder of this guide, we'll be using the general purpose
`awsrun.commands.aws.aws` command, which is an adapter for the AWS CLI tool.
This allows users to concurrently run the AWS CLI across multiple accounts. When
-combined with other awsrun features such as SAML authentication, cross-account
+combined with other awsrun features such as SAML/OIDC federation, cross-account
access, and metadata account filters, it makes the AWS CLI tool even more
powerful than it already is as you'll see.
@@ -97,7 +97,8 @@
default, awsrun will use the same profiles from your ~/.aws/credentials file. If
a profile does not exist for an account, awsrun will fallback to the "default"
profile if you have one configured. As you'll see later, we can use other
-plug-ins to obtain credentials for accounts via SAML and cross-account access.
+plug-ins to obtain credentials for accounts via SAML/OIDC federation and
+cross-account access.
Using awsrun to run a command over two accounts may not seem that interesting
yet, but when running the same AWS CLI command over more than a handful of
@@ -186,16 +187,16 @@
Up until this point, we have been using the default credential loader called
`awsrun.plugins.creds.aws.Profile`, which looks for account profiles in your
`$HOME/.aws/credentials` or `$HOME/.aws/config`. Let's look at some of the other
-mechanisms included with awsrun such as the SAML and cross-account access
+mechanisms included with awsrun such as the SAML, OIDC, and cross-account access
plug-ins. If you are only seeking reference material, please refer to the
`awsrun.plugins.creds` page instead.
-Within many enterprises, the use of single sign-on (SSO) is prevalent and allows
-for centralized account management. AWS supports federated users and SSO via the
-use of a SAML-compliant Identity Provider (IdP). Rather than define IAM users in
-each account, an IAM role can be created that allows for federated access via
-your IdP. To use SAML-based access with AWS, you need to obtain temporary tokens
-from the AWS STS service.
+Within many enterprises, the use of federated access is prevalent and allows for
+centralized account management. AWS supports federated users and SSO via the use
+of a SAML- or OIDC-compliant Identity Provider (IdP). Rather than define IAM
+users in each account, an IAM role can be created that allows for federated
+access via your IdP. To use SAML- or OIDC-based access with AWS, you need to
+obtain temporary tokens from the AWS STS service.
The `awsrun.plugins.creds.aws.SAML` credential plug-in simplifies this process
for you. At a minimum, you'll need the IAM role that has been setup for SAML as
@@ -777,7 +778,7 @@ def main():
csp = _CSP.from_prog_name(sys.argv[0])
_cli(csp)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# Don't print stack traces by default as it can be overwhelming (scary)
# for those not familiar with Python development.
if os.getenv("AWSRUN_TRACE"):
@@ -1104,8 +1105,8 @@ def _print_valid_commands(commands, out=sys.stdout):
return
print("The following are the available commands:\n", file=out)
- max_cmd_len = max(len(name) for name in commands.keys())
- for name in sorted(commands.keys()):
+ max_cmd_len = max(len(name) for name in commands)
+ for name in sorted(commands):
# By convention, as documented in user documentation, class docstring
# is used when printing a summary of commands.
docstring = commands[name].__doc__ or ""
diff --git a/src/awsrun/cloudwatch.py b/src/awsrun/cloudwatch.py
index 4b79744..285a5a6 100644
--- a/src/awsrun/cloudwatch.py
+++ b/src/awsrun/cloudwatch.py
@@ -68,7 +68,7 @@ def __init__(self, client, last=3600, samples=60, ingestion_interval=60):
self._samples = samples
self._ingestion_interval = ingestion_interval
self._period = self._compute_period()
- self._beg = self._end = datetime.now()
+ self._beg = self._end = datetime.now(timezone.utc)
_LOG.info(
"CWMetrics(client, last=%d, samples=%d, ingestion_interval=%d)",
diff --git a/src/awsrun/commands/aws/access_report.py b/src/awsrun/commands/aws/access_report.py
index 1e49131..d7bd7a2 100644
--- a/src/awsrun/commands/aws/access_report.py
+++ b/src/awsrun/commands/aws/access_report.py
@@ -87,7 +87,7 @@ def collect_results(self, acct, get_result):
self.total += 1
try:
result = get_result()
- except Exception: # pylint: disable=broad-except
+ except Exception:
self.no_access.append(acct)
return
diff --git a/src/awsrun/commands/aws/aws.py b/src/awsrun/commands/aws/aws.py
index 1403393..72028f5 100644
--- a/src/awsrun/commands/aws/aws.py
+++ b/src/awsrun/commands/aws/aws.py
@@ -507,9 +507,8 @@ def regional_execute(self, session, acct, region):
cmd,
env=env,
check=False,
- universal_newlines=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ text=True,
+ capture_output=True,
)
# Lastly, we return the ProcessCompleted object from the run() method.
@@ -543,7 +542,7 @@ def annotate_json(text):
# ProcessCompleted object from the subprocess.run() method above ...
result = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# ... unless there was an exception in which case it is raised by
# the call to get_result and we handle it here.
LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
diff --git a/src/awsrun/commands/aws/cidr_overlap.py b/src/awsrun/commands/aws/cidr_overlap.py
index eaa70dd..5c539c7 100644
--- a/src/awsrun/commands/aws/cidr_overlap.py
+++ b/src/awsrun/commands/aws/cidr_overlap.py
@@ -126,7 +126,7 @@ def regional_collect_results(self, acct, region, get_result):
def post_hook(self):
overlap = []
- for i in range(0, len(self.cidrs)): # pylint: disable=consider-using-enumerate
+ for i in range(len(self.cidrs)):
c1 = self.cidrs[i]
for j in range(i + 1, len(self.cidrs)):
c2 = self.cidrs[j]
diff --git a/src/awsrun/commands/aws/dx_maint.py b/src/awsrun/commands/aws/dx_maint.py
index 081f9b5..3168823 100644
--- a/src/awsrun/commands/aws/dx_maint.py
+++ b/src/awsrun/commands/aws/dx_maint.py
@@ -263,7 +263,7 @@ def regional_collect_results(self, acct, region, get_result):
# in this method, but not the execute method above.
self.all_results.extend(get_result())
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s/%s: error: %s", acct, region, e, exc_info=True)
print(f"{acct}/{region}: error: {e}", flush=True, file=sys.stderr)
diff --git a/src/awsrun/commands/aws/kubectl.py b/src/awsrun/commands/aws/kubectl.py
index 280eada..9e9dfd1 100644
--- a/src/awsrun/commands/aws/kubectl.py
+++ b/src/awsrun/commands/aws/kubectl.py
@@ -257,12 +257,15 @@ def regional_from_cli(cls, parser, argv, cfg):
parser.error("Do not specify --{arg} with awsrun kubectl")
delattr(args, attr)
- if args.awsrun_annotate and args.output:
- if args.awsrun_annotate != "text":
- if args.awsrun_annotate != args.output:
- parser.error(
- "When specifying --awsrun-annotate, you do not need the --output flag"
- )
+ if (
+ args.awsrun_annotate
+ and args.output
+ and args.awsrun_annotate != "text"
+ and args.awsrun_annotate != args.output
+ ):
+ parser.error(
+ "When specifying --awsrun-annotate, you do not need the --output flag"
+ )
return cls(**vars(args))
@@ -363,9 +366,8 @@ def regional_execute(self, session, acct, region):
cmd,
env=os.environ,
check=False,
- universal_newlines=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ text=True,
+ capture_output=True,
)
results.append(_Result(result.stdout, result.stderr, name, namespace))
@@ -392,7 +394,7 @@ def annotate_format(result, loader, dumper):
d["Results"] = loader(result.stdout)
dumper(d, sys.stdout, indent=4)
print()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
annotate_lines(result, f"cannot parse output: {e}", file=sys.stderr)
try:
@@ -400,7 +402,7 @@ def annotate_format(result, loader, dumper):
# ProcessCompleted object from the subprocess.run() method above ...
results = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# ... unless there was an exception in which case it is raised by
# the call to get_result and we handle it here.
LOG.info("%s/%s: error: %s", acct, region, e, exc_info=True)
@@ -481,7 +483,8 @@ def annotate_format(result, loader, dumper):
def _save_output(name, text):
- with name.open("w") as out:
+ fd = os.open(name, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+ with os.fdopen(fd, "w") as out:
out.write(text)
@@ -500,7 +503,7 @@ def _save_kubecfg(name, namespace, account_id, region, cluster, session):
}
kubedir = Path.home() / Path(".kube")
- kubedir.mkdir(parents=True, exist_ok=True)
+ kubedir.mkdir(mode=0o700, parents=True, exist_ok=True)
filename = kubedir / Path(f"awsrun-{account_id}-{region}-{name}-{namespace}")
_save_output(filename, _KUBECONFIG.format(**substitions))
@@ -508,8 +511,8 @@ def _save_kubecfg(name, namespace, account_id, region, cluster, session):
def _list_clusters(eks):
- clusters = []
- for page in eks.get_paginator("list_clusters").paginate():
- for name in page["clusters"]:
- clusters.append(name)
- return clusters
+ return [
+ name
+ for page in eks.get_paginator("list_clusters").paginate()
+ for name in page["clusters"]
+ ]
diff --git a/src/awsrun/commands/aws/last.py b/src/awsrun/commands/aws/last.py
index 8040eed..8a2eafc 100644
--- a/src/awsrun/commands/aws/last.py
+++ b/src/awsrun/commands/aws/last.py
@@ -144,7 +144,7 @@
from datetime import datetime, timedelta, timezone
from functools import partial
from itertools import chain, cycle
-from typing import Optional
+from typing import ClassVar, Optional
from awsrun.argparse import AppendAttributeValuePair
from awsrun.config import Bool, Dict, Int, List, Str
@@ -932,7 +932,7 @@ class EventViewer(App):
"""
TITLE = "CloudTrail Viewer"
- BINDINGS = [
+ BINDINGS: ClassVar[list] = [
("q", "quit", "Quit"),
("c", "copy", "Copy"),
("e", "export_popup", "Export"),
@@ -945,7 +945,7 @@ class EventViewer(App):
total_count = reactive(0)
filtered_count = reactive(0)
- theme = {
+ theme: ClassVar[dict] = {
"dark": Syntax.get_theme("dracula"),
"light": Syntax.get_theme("friendly"),
}
diff --git a/src/awsrun/commands/aws/list_iam_policies.py b/src/awsrun/commands/aws/list_iam_policies.py
index fb1881e..30cb4a1 100644
--- a/src/awsrun/commands/aws/list_iam_policies.py
+++ b/src/awsrun/commands/aws/list_iam_policies.py
@@ -436,8 +436,8 @@ def execute(self, session, acct):
if self.include_roles:
identities["role"] = get_identities(iam.roles, iam.Role, self.search_roles)
- for i_type in identities: # pylint: disable=consider-using-dict-items
- for identity in identities[i_type]:
+ for i_type, identity_list in identities.items():
+ for identity in identity_list:
ip = IdentityPrinter(out, f"{acct}: identity={i_type}:{identity.name}")
self.show_inline_policies(identity, ip)
self.show_attached_policies(identity, ip)
@@ -450,12 +450,11 @@ def show_inline_policies(self, identity, ip):
return
for inline in identity.policies.all():
- # pylint: disable=cell-var-from-loop
# We wrap the policy_document in a lambda so boto3 resource is not
- # fetched unless it is really needed. Although pylint complains
- # about wrapping the looping var in a lambda, we use the lambda
- # immediately if needed.
- if self.should_skip(inline.policy_name, lambda: inline.policy_document):
+ # fetched unless it is really needed.
+ if self.should_skip(
+ inline.policy_name, lambda inline=inline: inline.policy_document
+ ):
continue
ip.print(f"policy=inline:{inline.policy_name}")
@@ -468,11 +467,11 @@ def show_attached_policies(self, identity, ip):
return
for attached in identity.attached_policies.all():
- # pylint: disable=cell-var-from-loop
# We wrap the default_version.document in a lambda so boto3 resource
# is not fetched unless it is really needed.
if self.should_skip(
- attached.policy_name, lambda: attached.default_version.document
+ attached.policy_name,
+ lambda attached=attached: attached.default_version.document,
):
continue
@@ -604,4 +603,4 @@ def identity_exists(identity):
except ClientError as e:
if e.response["Error"]["Code"] == "NoSuchEntity":
return False
- raise e
+ raise
diff --git a/src/awsrun/commands/aws/list_iam_roles.py b/src/awsrun/commands/aws/list_iam_roles.py
index 6dfe33b..e844571 100644
--- a/src/awsrun/commands/aws/list_iam_roles.py
+++ b/src/awsrun/commands/aws/list_iam_roles.py
@@ -121,7 +121,7 @@ def _trusted_arns(policy):
arns = []
for statement in policy.get("Statement", []):
- if not statement.get("Effect", "") == "Allow":
+ if statement.get("Effect", "") != "Allow":
continue
for arn in statement.get("Principal", {}).values():
if isinstance(arn, list):
diff --git a/src/awsrun/commands/azure/az.py b/src/awsrun/commands/azure/az.py
index 66c9522..02244d1 100644
--- a/src/awsrun/commands/azure/az.py
+++ b/src/awsrun/commands/azure/az.py
@@ -348,9 +348,8 @@ def execute(self, session, acct):
result = subprocess.run(
cmd,
check=False,
- universal_newlines=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ text=True,
+ capture_output=True,
)
# Lastly, we return the ProcessCompleted object from the run() method.
@@ -364,7 +363,7 @@ def collect_results(self, acct, get_result):
def annotate_lines(text, delimiter=": ", file=sys.stdout, separator=False):
for line in filter(None, text.split("\n")):
print(f"{acct}{delimiter}{line}", file=file, flush=True)
- if separator and not text == "\n":
+ if separator and text != "\n":
print()
def annotate_json(text):
@@ -398,7 +397,7 @@ def annotate_yaml(text):
# ProcessCompleted object from the subprocess.run() method above ...
result = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# ... unless there was an exception in which case it is raised by
# the call to get_result and we handle it here.
LOG.info("%s: error: %s", acct, e, exc_info=True)
diff --git a/src/awsrun/commands/azure/cidr_overlap.py b/src/awsrun/commands/azure/cidr_overlap.py
index 3e43d15..4b30b7d 100644
--- a/src/awsrun/commands/azure/cidr_overlap.py
+++ b/src/awsrun/commands/azure/cidr_overlap.py
@@ -122,7 +122,7 @@ def collect_results(self, acct, get_result):
def post_hook(self):
overlap = []
- for i in range(0, len(self.cidrs)): # pylint: disable=consider-using-enumerate
+ for i in range(len(self.cidrs)):
c1 = self.cidrs[i]
for j in range(i + 1, len(self.cidrs)):
c2 = self.cidrs[j]
diff --git a/src/awsrun/config.py b/src/awsrun/config.py
index d5a00b6..4e268b3 100644
--- a/src/awsrun/config.py
+++ b/src/awsrun/config.py
@@ -73,13 +73,12 @@
import re
from functools import reduce
from pathlib import Path
+from typing import ClassVar
import yaml
LOG = logging.getLogger(__name__)
-# pylint: disable=unidiomatic-typecheck
-#
# Because isinstance(True, int) is true, we do not rely on isinstance for our
# type checking in this module as we want to match exact types. We don't want to
# consider subclasses and True should not type check successfully against an
@@ -97,7 +96,7 @@ class Config:
load configs from files.
"""
- _filetypes = {}
+ _filetypes: ClassVar[dict] = {}
@classmethod
def register_filetype(cls, config_class, *extensions):
@@ -164,8 +163,6 @@ def get(self, *keys, default=None, type=None, must_exist=False):
c.get('path', 'to', 'value', type=And(StrMatch(r'\\d+'), StrMatch(r'[A-Z]')))
c.get('path', 'to', 'value', type=Not(Or(Int, Float)))
"""
- # pylint: disable=redefined-builtin
-
# This one-liner will recursively follow a list of keys into a
# dictionary and return the value. If a key does not exist, return an
# empty dict.
@@ -199,9 +196,7 @@ def get(self, *keys, default=None, type=None, must_exist=False):
return value
# Finally, all other cases indicate a type error.
- raise TypeError(
- f"Error in config: {'->'.join(keys)}: not a {type}: {repr(value)}"
- )
+ raise TypeError(f"Error in config: {'->'.join(keys)}: not a {type}: {value!r}")
EmptyConfig = Config({})
@@ -309,7 +304,7 @@ def type_check(self, obj):
# we did not check types, then this would report incorrect results.
# Likewise, we cannot use isinstance here either as a bool is a subclass
# of int, so it would also report incorrect results.
- if type(obj) != type(self.const): # noqa: E721
+ if type(obj) != type(self.const):
return False
return obj == self.const
@@ -338,7 +333,7 @@ def __init__(self, type_):
self.type = type_
def type_check(self, obj):
- return type(obj) == self.type # noqa: E721
+ return type(obj) == self.type
def __str__(self):
return self.type.__name__
@@ -354,7 +349,7 @@ def __init__(self, pattern):
self.pattern = pattern
def type_check(self, obj):
- if type(obj) != str: # noqa: E721
+ if type(obj) != str:
return False
return bool(re.search(self.pattern, obj))
@@ -366,7 +361,7 @@ class IpAddress(Type):
"""Represents a string matching an IP address (v4 or v6)."""
def type_check(self, obj):
- if type(obj) != str: # noqa: E721
+ if type(obj) != str:
return False
try:
ipaddress.ip_address(obj)
@@ -382,7 +377,7 @@ class IpNetwork(Type):
"""Represents a string matching an IP network (v4 or v6)."""
def type_check(self, obj):
- if type(obj) != str: # noqa: E721
+ if type(obj) != str:
return False
try:
ipaddress.ip_network(obj)
@@ -398,7 +393,7 @@ class FileType(Type):
"""Represents a string pointing to an existing file."""
def type_check(self, obj):
- if type(obj) != str: # noqa: E721
+ if type(obj) != str:
return False
return Path(obj).exists()
@@ -462,7 +457,7 @@ def __init__(self, element_type):
self.element_type = element_type
def type_check(self, obj):
- if type(obj) != list: # noqa: E721
+ if type(obj) != list:
return False
return all(self.element_type.type_check(e) for e in obj)
@@ -485,9 +480,9 @@ def __init__(self, key_type, value_type):
self.value_type = value_type
def type_check(self, obj):
- if type(obj) != dict: # noqa: E721
+ if type(obj) != dict:
return False
- return all(self.key_type.type_check(k) for k in obj.keys()) and all(
+ return all(self.key_type.type_check(k) for k in obj) and all(
self.value_type.type_check(v) for v in obj.values()
)
diff --git a/src/awsrun/plugins/creds/aws.py b/src/awsrun/plugins/creds/aws.py
index a650ebf..dd3c4ac 100644
--- a/src/awsrun/plugins/creds/aws.py
+++ b/src/awsrun/plugins/creds/aws.py
@@ -46,12 +46,17 @@
from awsrun.config import URL, Bool, Choice, Dict, Int, Str
from awsrun.plugmgr import Plugin
-from awsrun.session.aws import CredsViaCrossAccount, CredsViaProfile, CredsViaSAML
+from awsrun.session.aws import (
+ CredsViaCrossAccount,
+ CredsViaOAuth2ROPC,
+ CredsViaProfile,
+ CredsViaSAML,
+)
# This is only used to prevent pdoc (the doc generator) from exposing
# AbstractCrossAccount in the module's documentation, which is intended for CLI
# users and not programmers as this module deals with plugin configurations.
-__all__ = ["Profile", "SAML", "ProfileCrossAccount", "SAMLCrossAccount"]
+__all__ = ["SAML", "Profile", "ProfileCrossAccount", "SAMLCrossAccount"]
_AUTH_CLASSES = {"basic": HTTPBasicAuth, "digest": HTTPDigestAuth, "ntlm": HttpNtlmAuth}
@@ -297,6 +302,172 @@ def instantiate(self, args):
return session_provider
+class OAuth2(Plugin):
+ """CLI plug-in that uses federated authentication via OAuth2 for credentials.
+
+ AWS supports federated access via OpenID Connect (OIDC). With this
+ plug-in, the user authenticates to an OAuth2 server using the Resource
+ Owner Password Credential (ROPC) flow, such as Microsoft Entra, to obtain
+ an access token that is used to assume an AWS role within an account to
+ obtain STS credentials. Unlike the `Profile` plug-in, this plug-in does
+ not rely on the standard AWS credentials or configuration files, and thus
+ does not require defining all accounts ahead of time. This plug-in does,
+ however, require that the OAuth2 server is configured to grant a user
+ access to authorized AWS accounts.
+
+ Please refer to AWS [OIDC federation guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html)
+ for additional details on the use of federated access with OIDC.
+
+ ## Configuration
+
+ Options with an asterisk are mandatory and must be provided:
+
+ Credentials:
+ plugin: awsrun.plugins.creds.aws.OAuth2
+ options:
+ username: STRING
+ password: STRING
+ role: STRING*
+ url: STRING*
+ http_headers:
+ STRING: STRING
+ oauth2_params:
+ STRING: STRING
+ no_verify: BOOLEAN
+ duration: INTEGER
+
+ ## Plug-in Options
+
+ Some options can be overridden on the awsrun CLI via command line flags.
+ In those cases, the CLI flags are specified next to the option name below.
+
+ `username`, `--oauth2-username`
+ : The username to use when authenticating with the OAuth2 server for the base
+ account. If this value is not provided, the following environment variables
+ are checked in order: LOGNAME, USER, LNAME and USERNAME.
+
+ `password`, `--oauth2-password`
+ : The password to use when authenticating with the OAuth2 server for the base
+ account. The default is the value, if any, of the PASSWORD environment
+ variable. If none of these are set, the user will be prompted via the
+ console when awsrun is invoked.
+
+ `role`, `--oauth2-role`
+ : The AWS role (not ARN) to assume in the base or source account when
+ federating via OIDC. This value **must** be provided via the user
+ configuration or as an awsrun command line argument. Note: this is not the
+ IAM role name used when performing the cross-account assume role (see
+ `x_acct: role` below).
+
+ `url`
+ : The URL to the OAuth2 server that will provide an access token upon
+ successful authentication for the base account. This value **must** be
+ provided via the user configuration file.
+
+ `http_headers`
+ : Additional HTTP headers to send in the request to the OAuth2 server. If
+ specified, it must be a dictionary of `key: value` pairs, where keys and
+ values are strings.
+
+ `oauth2_params`
+ : Additional OAuth2 parameters to send in the request to the OAuth2 server. If
+ specified, it must be a dictionary of `key: value` pairs, where keys and
+ values are strings. For example, this can be used to specify `client_id`
+ and `scope` parameters if required by the OAuth2 server.
+
+ `no_verify`, `--oauth2-no-verify`
+ : Disable HTTP certificate verification. This is not advisable and user will
+ be warned on the command line if verification has been disabled. The default
+ value is `false`.
+
+ `duration`, `--oauth2-duration`
+ : The amount of time, in seconds, that the AWS credentials for the role /
+ base account combination are cached in memory. The default value is `3600`
+ seconds (1 hour). Caching can be disabled by specifying `0` seconds. If AWS
+ credentials are expired sooner by a local IAM policy, then lower the value.
+ """
+
+ def __init__(self, parser, cfg):
+ super().__init__(parser, cfg)
+
+ # Define the arguments that we want to allow a user to override via the
+ # main CLI. Any CLI args added via add_argument will be commingled with
+ # the main awsrun args, so they are prefixed with '--oauth2-' to lessen
+ # chance of a name collision.
+ group = parser.add_argument_group("OAuth2 options")
+ group.add_argument(
+ "--oauth2-username",
+ metavar="USER",
+ default=self.cfg("username", type=Str, default=getpass.getuser()),
+ help="username for OAuth2 authentication",
+ )
+
+ group.add_argument(
+ "--oauth2-password",
+ metavar="PASS",
+ default=self.cfg(
+ "password", type=Str, default=os.environ.get("PASSWORD", None)
+ ),
+ help="password for OAuth2 authentication",
+ )
+
+ group.add_argument(
+ "--oauth2-role",
+ metavar="ROLE",
+ default=self.cfg("role", type=Str, must_exist=True),
+ help="base role name (not ARN) to assume via OAuth2",
+ )
+
+ group.add_argument(
+ "--oauth2-duration",
+ metavar="SECS",
+ type=int,
+ default=cfg("duration", type=Int, default=3600),
+ help="duration when requesting aws credentials in assume_role*",
+ )
+
+ group.add_argument(
+ "--oauth2-no-verify",
+ action="store_true",
+ default=cfg("no_verify", type=Bool, default=False),
+ help="disable cert verification for HTTP requests",
+ )
+
+ def instantiate(self, args):
+ cfg = self.cfg
+
+ # We don't prompt for this password above when checking the PASSWORD
+ # environment as we wouldn't have access to the username. which should
+ # be included in the prompt to the user to remind them of the username
+ # being used.
+ args.oauth2_password = args.oauth2_password or getpass.getpass(
+ f"Password for {args.oauth2_username}? "
+ )
+
+ # Build a session provider using the combination of options that have
+ # been specified in the user's configuration file or have been
+ # overridden on the command line.
+ session_provider = CredsViaOAuth2ROPC(
+ role=args.oauth2_role,
+ url=cfg("url", type=URL, must_exist=True),
+ username=args.oauth2_username,
+ password=args.oauth2_password,
+ headers=cfg("http_headers", type=Dict(Str, Str), default={}),
+ oauth2_params=cfg("oauth2_params", type=Dict(Str, Str), default={}),
+ duration=args.oauth2_duration,
+ no_verify=args.oauth2_no_verify,
+ )
+
+ # Test if password provided is correct. If the wrong password is used and
+ # one cannot be authenticated, then AccountRunner might inadvertently lock
+ # the user out as it will repeatedly attempt to get an OAuth2 token for
+ # each account. If this fails, an exception will be thrown and caught in
+ # the main CLI.
+ session_provider.token()
+
+ return session_provider
+
+
class AbstractCrossAccount(Plugin):
"""Abstract base class for cross-account access plug-ins.
@@ -571,3 +742,122 @@ def __init__(self, parser, cfg):
def _get_base_auth(self):
return self._base_auth
+
+
+class OAuth2CrossAccount(AbstractCrossAccount):
+ """CLI plug-in that uses a OAuth2 base account for cross-account access.
+
+ AWS supports cross-account access from one "base" or "source" account to
+ another account. This plug-in obtains credentials for the base account via
+ the `OAuth2` plug-in, and then uses those credentials to assume role to the
+ other account provided the proper IAM permissions have been setup. Cross
+ account access is typically used in large enterprises with a significant
+ number of AWS accounts to simply management of credentials. The benefit is
+ that a user does not need direct access to every account. Only access to the
+ base account is required from which the user can "hop off" to other
+ accounts.
+
+ Please refer to [Providing Access to an IAM User in Another AWS Account That
+ You
+ Own](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_aws-accounts.html)
+ for additional details on cross-account access.
+
+ ## Configuration
+
+ Options with an asterisk are mandatory and must be provided:
+
+ Credentials:
+ plugin: awsrun.plugins.creds.aws.OAuth2CrossAccount
+ options:
+ username: STRING
+ password: STRING
+ role_arn: STRING*
+ url: STRING*
+ http_headers:
+ STRING: STRING
+ oauth2_params:
+ STRING: STRING
+ no_verify: BOOLEAN
+ duration: INTEGER
+ x_acct:
+ base: STRING*
+ role: STRING*
+ duration: INTEGER
+
+ ## Plug-in Options
+
+ Some options can be overridden on the awsrun CLI via command line flags.
+ In those cases, the CLI flags are specified next to the option name below.
+
+ `username`, `--oauth2-username`
+ : The username to use when authenticating with the OAuth2 server for the base
+ account. If this value is not provided, the following environment variables
+ are checked in order: LOGNAME, USER, LNAME and USERNAME.
+
+ `password`, `--oauth2-password`
+ : The password to use when authenticating with the OAuth2 server for the base
+ account. The default is the value, if any, of the PASSWORD environment
+ variable. If none of these are set, the user will be prompted via the
+ console when awsrun is invoked.
+
+ `role_arn`, `--oauth2-role-arn`
+ : The AWS role ARN to assume in the base or source account when
+ federating via OIDC. This value **must** be provided via the user
+ configuration or as an awsrun command line argument. Note: this is not the
+ IAM role name used when performing the cross-account assume role (see
+ `x_acct: role` below).
+
+ `url`
+ : The URL to the OAuth2 server that will provide an access token upon
+ successful authentication for the base account. This value **must** be
+ provided via the user configuration file.
+
+ `http_headers`
+ : Additional HTTP headers to send in the request to the OAuth2 server. If
+ specified, it must be a dictionary of `key: value` pairs, where keys and
+ values are strings.
+
+ `oauth2_params`
+ : Additional OAuth2 parameters to send in the request to the OAuth2 server. If
+ specified, it must be a dictionary of `key: value` pairs, where keys and
+ values are strings. For example, this can be used to specify `client_id`
+ and `scope` parameters if required by the OAuth2 server.
+
+ `no_verify`, `--oauth2-no-verify`
+ : Disable HTTP certificate verification. This is not advisable and user will
+ be warned on the command line if verification has been disabled. The default
+ value is `false`.
+
+ `duration`, `--oauth2-duration`
+ : The amount of time, in seconds, that the AWS credentials for the role /
+ base account combination are cached in memory. The default value is `3600`
+ seconds (1 hour). Caching can be disabled by specifying `0` seconds. If AWS
+ credentials are expired sooner by a local IAM policy, then lower the value.
+
+ `x_acct: base`, `--x-acct-base`
+ : The base or source account from which to "hop off" to other accounts. This
+ value **must** be provided via the user configuration or as an awsrun
+ command line argument. OAuth2-based authentication will be used to obtain
+ credentials for this base account.
+
+ `x_acct: role`, `--x-acct-role`
+ : The IAM role name, not ARN, to assume from the base account to obtain
+ credentials for other accounts. It is not an ARN as the ARN is dynamically
+ created as each account is processed. This value **must** be provided via
+ the user configuration or as an awsrun command line argument. Note: this
+ value is not the role used for the OAuth2 authentication to the base
+ account (see the `role_arn` option above).
+
+ `x_acct: duration`, `--x-acct-duration`
+ : The amount of time, in seconds, that the AWS credentials for the role /
+ cross-account combination are cached in memory. The default value is `3600`
+ seconds (1 hour). Caching can be disabled by specifying `0` seconds. If AWS
+ credentials are expired sooner by a local IAM policy, then lower the value.
+ """
+
+ def __init__(self, parser, cfg):
+ self._base_auth = OAuth2(parser, cfg)
+ super().__init__(parser, cfg)
+
+ def _get_base_auth(self):
+ return self._base_auth
diff --git a/src/awsrun/runner.py b/src/awsrun/runner.py
index 51a8617..77b371b 100644
--- a/src/awsrun/runner.py
+++ b/src/awsrun/runner.py
@@ -237,7 +237,7 @@ class documents the contract between the awsrun framework and command
"""
@classmethod
- def from_cli(cls, parser, argv, cfg): # pylint: disable=unused-argument
+ def from_cli(cls, parser, argv, cfg):
"""Factory to build the command from CLI args and user configuration.
*This method is only required if the command is intended for use with
@@ -443,7 +443,7 @@ def collect_results(self, acct, get_result):
try:
print(get_result(), end="", flush=True)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s: error: %s", acct, e, exc_info=True)
print(f"{acct}: error: {e}", flush=True, file=sys.stderr)
@@ -509,7 +509,7 @@ def from_cli(cls, parser, argv, cfg):
return command
@classmethod
- def regional_from_cli(cls, parser, argv, cfg): # pylint: disable=unused-argument
+ def regional_from_cli(cls, parser, argv, cfg):
"""Factory to build a regional command from CLI args and user configuration.
*This method is only required if the command is intended for use with
@@ -617,7 +617,7 @@ def collect_results(self, acct, get_result):
for region, get_region_result in get_result():
self.regional_collect_results(acct, region, get_region_result)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s: error: %s", acct, e, exc_info=True)
print(f"{acct}: error: {e}", flush=True, file=sys.stderr)
@@ -647,7 +647,7 @@ def regional_collect_results(self, acct, region, get_result):
try:
print(get_result(), end="", flush=True)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
LOG.warning("%s/%s: error: %s", acct, region, e, exc_info=True)
print(f"{acct}/{region}: error: {e}", flush=True, file=sys.stderr)
@@ -678,7 +678,7 @@ def execute(self, session, acct):
def collect_results(self, acct, get_result):
try:
self.results[acct] = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
self.errors[acct] = e
@@ -733,7 +733,7 @@ def regional_execute(self, session, acct, region):
def regional_collect_results(self, acct, region, get_result):
try:
self.results[(acct, region)] = get_result()
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
self.errors[(acct, region)] = e
@@ -975,7 +975,7 @@ def worker_task(acct):
session = self.session_provider.session(acct_id)
return _wrap_result(cmd.execute, session, acct)
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
# NOTE: exceptions thrown by a Command's execute are not
# handled in this block, but in wrap_result above. This
# block handles exceptions that occur while obtaining a
@@ -1064,7 +1064,7 @@ def _wrap_result(fn, *args, **kwargs):
try:
result = fn(*args, **kwargs)
return lambda: result
- except Exception as e: # pylint: disable=broad-except
+ except Exception as e:
return _wrap_exception(e)
diff --git a/src/awsrun/session/aws.py b/src/awsrun/session/aws.py
index dde47f9..e3f2f69 100644
--- a/src/awsrun/session/aws.py
+++ b/src/awsrun/session/aws.py
@@ -7,16 +7,16 @@
## Overview
-This module provides a `SessionProvider` interface to obtain AWS credentials via
-one of several mechanisms: the standard AWS CLI configuration files, Single Sign
-On (SSO) via federated SAML authentication, or cross-account access initiated
-from a base account. Regardless of the mechanism, the session provider is
-responsible for returning a boto3 Session that contains the credentials for a
-requested account. In some cases, those credentials are cached to limit the
-number of API calls to AWS and/or Identity Providers (IdP) should a session be
-requested for the same account again.
-
-There are three concrete session provider implementations included in this
+This module provides a `SessionProvider` interface to obtain AWS credentials
+via one of several mechanisms: the standard AWS CLI configuration files,
+Single Sign On (SSO) via federated SAML/OIDC, or cross-account access
+initiated from a base account. Regardless of the mechanism, the session
+provider is responsible for returning a boto3 Session that contains the
+credentials for a requested account. In some cases, those credentials are
+cached to limit the number of API calls to AWS and/or Identity Providers (IdP)
+should a session be requested for the same account again.
+
+There are four concrete session provider implementations included in this
module:
`CredsViaProfile`
@@ -25,6 +25,9 @@
`CredsViaSAML`
: Credentials are obtained via a role assumed via SAML-based federation.
+`CredsViaOAuth2ROPC`
+: Credentials are obtained via a role assumed via OAuth2 ROPC authentication.
+
`CredsViaCrossAccount` respectively.
: Credentials are obtained via a role assumed from a base account.
@@ -119,6 +122,42 @@
AWS](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_saml.html)
for additional details on the use of federated SSO with SAML.
+### OIDC via OAuth2 ROPC
+
+AWS supports federated SSO access via OpenID Connect (OIDC). In this scenario,
+rather than defining AWS credentials in AWS configuration files (see first
+section), the user obtains an access token from an OAuth2 authorization server
+using the Resource Owner Password Credential (ROPC) flow. The access token is
+then sent to AWS to assume an AWS role within an account to obtain credentials
+for API usage.
+
+To create a `SessionProvider` that will obtain credentials via OAuth2
+ROPC-based federated access, create an instance of `CredsViaOAuth2ROPC`. In
+the example below, Microsoft Entra ID (formerly Azure AD) is used as the OAuth2
+authorization server:
+
+ # Instantiate a single session provider and reuse it
+ session_provider = CredsViaOAuth2ROPC(
+ role_arn='arn:aws:iam::111222333444:role/RoleName',
+ url='https://login.microsoftonline.com/TENANT_ID/oauth2/v2.0/token"',
+ username='example@example.com',
+ password='EXAMPLE_PASSWORD',
+ oauth2_params={
+ 'client_id': 'CLIENT_ID',
+ 'scope': 'api://CLIENT_ID/AssumeRoleWithWebIdentity'
+ })
+
+ # Obtain boto3 sessions for one or more accounts
+ session1 = session_provider.session('111111111111')
+ session2 = session_provider.session('222222222222')
+
+ # Use the sessions to interact with AWS
+ ec2 = session1.resource('ec2', region_name='us-east-1')
+ iam = session2.resource('iam', region_name='us-east-1')
+
+Please refer to AWS [OIDC federation guide](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_oidc.html)
+for additional details on the use of federated access with OIDC.
+
### Cross Account
In large companies with a significant number of AWS accounts, it is more common
@@ -152,8 +191,9 @@
ec2 = session1.resource('ec2', region_name='us-east-1')
iam = session2.resource('iam', region_name='us-east-1')
-In large companies, it's likely more common that SAML is used for direct access
-to the base account, and then cross-account access to the remainder of accounts:
+In large companies, it's likely more common that SAML or OIDC is used for
+direct access to the base account, and then cross-account access to the
+remainder of accounts:
# Instantiate a session provider for the base account
session_provider = CredsViaSAML(
@@ -173,14 +213,14 @@
need to have an assume role policy document that permits access from the base
account.
-Although, the `CredsViaProfile` session provider can support cross-account
-access, it does require that every account must be defined in the local AWS
-files. For a corporation with hundreds of accounts that change all the time,
-this may not be practical. `CredsViaCrossAccount` does not use the local AWS
-files. The only use of AWS files would be limited to the base account if
-`CredsViaProfile` is used. If, on the other hand, SAML authentication is used
-for the base account via `CredsViaSAML`, then the local AWS files aren't used at
-all.
+Although, the `CredsViaProfile` session provider could support direct access
+to multiple accounts, it would require that every account be defined in local
+AWS credential files. For a corporation with hundreds of accounts that change
+all the time, this may not be practical. `CredsViaCrossAccount` does not use
+the local AWS files. The only use of AWS files would be limited to the base
+account if `CredsViaProfile` is used. If, on the other hand, SAML or OIDC
+authentication is used for the base account via `CredsViaSAML` or
+`CredsViaOAuth2ROPC`, then the local AWS files aren't used at all.
Please refer to [Providing Access to an IAM User in Another AWS Account That You
Own](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_aws-accounts.html)
@@ -188,19 +228,24 @@
## Caching
-AWS credentials obtained via SAML-based federation (assume_role_with_saml) or
-cross account access (assume_role) are cached in memory by default for 1 hour
-(3600 seconds). This means subsequent calls to `SessionProvider.session` for the
-same account will return the cached set of credentials unless half of the cache
-duration has transpired, in which case a new set of credentials will be
-returned. If AWS credentials are expired sooner by a local policy, then lower
-the duration in the `CredsViaSAML` and `CredsViaCrossAccount` constructors.
+AWS credentials obtained via SAML (assume_role_with_saml), OIDC
+(assume_role_with_web_identity), or cross account access (assume_role) are
+cached in memory by default for 1 hour (3600 seconds). This means subsequent
+calls to `SessionProvider.session` for the same account will return the cached
+set of credentials unless half of the cache duration has transpired, in which
+case a new set of credentials will be returned. If AWS credentials are expired
+sooner by a local policy, then lower the duration in the `CredsViaSAML`,
+`CredsViaOAuth2ROPC` and `CredsViaCrossAccount` constructors.
Likewise, the SAML assertion obtained via the IdP by `CredsViaSAML` is cached
for 5 minutes (300 seconds). This, too, can be adjusted in the constructor. If
the IdP expires SAML assertions sooner, then the `saml_duration` value must be
set appropriately.
+If using `CredsViaOAuth2ROPC`, the access token obtained from the OAuth2
+server is cached based on the expiration provided in the token response. There
+is no need to adjust any caching parameters in this case.
+
Finally, it is important to note that caching in this context has no relevance
to refreshable credentials that boto3 can provide when assuming roles. This
module does not provide refreshable credentials at this time. At some point in
@@ -216,10 +261,10 @@
: Raised if the user could not be authenticated with IdP.
`IDPInvalidResponseException`
-: Raised if the SAML response cannot be found from IdP.
+: Raised if the SAML or OAuth2 response is not valid.
`IDPInvalidRoleException`
-: Raised if the user does not have access to the role.
+: Raised if the user does not have access to the role in a SAML flow.
`AWSAssumeRoleException`
: Raised if the assume_role* calls fail.
@@ -245,7 +290,7 @@
import requests
from bs4 import BeautifulSoup
-from awsrun.cache import ExpiringValue
+from awsrun.cache import ExpiringValue, ValueWithExpiry
from awsrun.session import SessionProvider
LOG = logging.getLogger(__name__)
@@ -257,7 +302,13 @@ class CachingSessionProvider(SessionProvider):
This class cannot be instantiated directly. Users of this class must provide
an implementation for `credentials` and must invoke its constructor. Caching
is based on the IAM `role` being assumed and the the account ID. Credentials
- are cached for `duration` seconds, which defaults to 1 hour.
+ are cached for the expiry returned from AWS assume_role calls.
+
+ For backwards compatibility, the `duration` parameter is provided to
+ specify the maximum cache duration. If the credentials returned from AWS
+ have an expiration, then that expiration is used to determine when to
+ refresh the credentials. If not, then the `duration` parameter is used to
+ determine when to refresh the credentials.
"""
def __init__(self, role, duration=3600):
@@ -287,7 +338,10 @@ def session(self, acct_id):
# the lock may seem redundant, but it's safer to make this explicit.
ev = self._creds.setdefault(
(acct_id, self._role),
- ExpiringValue(lambda: self.credentials(acct_id), self._duration / 2),
+ ExpiringValue(
+ lambda: self._expiry_value(self.credentials(acct_id)),
+ self._duration,
+ ),
)
creds = ev.value()
@@ -298,6 +352,22 @@ def session(self, acct_id):
aws_session_token=creds["SessionToken"],
)
+ # This function is used to wrap the credentials returned by the subclass's
+ # `credentials` method in a ValueWithExpiry object. If the credentials
+ # include an "Expiration" key, then the expiration time is used to set the
+ # expiration time of the ValueWithExpiry object. If not, then the
+ # credentials are returned as-is and the ExpiringValue will use the
+ # max_age specified in the constructor.
+ @staticmethod
+ def _expiry_value(creds):
+ if not creds.get("Expiration"):
+ return creds
+
+ # Subtract 5 mins from the expiration time to avoid edge cases where the
+ # credentials are expired by the time they are used.
+ expiration = creds["Expiration"].timestamp() - 300
+ return ValueWithExpiry(creds, expires_at=expiration)
+
def credentials(self, acct_id):
"""Returns a dict containing AWS credentials for the requested account.
@@ -351,6 +421,122 @@ def session(self, acct_id):
return boto3.Session()
+class CredsViaOAuth2ROPC(CachingSessionProvider):
+ """A session provider that uses OAuth2 ROPC authentication.
+
+ This class relies on an OAuth2 authorization server using the Resource
+ Owner Password Credential (ROPC) flow to provide an access token to an
+ authenticated user, which is then sent to AWS to obtain the credentials for
+ an AWS account.
+
+ The `role` is the IAM role name (not ARN) to assume in the AWS account.
+ This is used along with the account to construct the ARN when making the
+ underlying AWS `assume_role_with_web_identity` API call.
+
+ The `url` is the URL to the OAuth2 server that will provide a token to the
+ user once they have authenticated. The client will authenticate using the
+ `username` and `password` parameters. If additional OAuth2 parameters must
+ be provided to the server in the authentication request, specify a dict of
+ parameter/value pairs via the `oauth2_params` argument. If additional HTTP
+ headers must be provided, specify a dict of header/value pairs via the
+ `headers` argument. To disable certificate verification, which is strongly
+ discouraged, set `no_verify` to `True`.
+
+ The AWS credentials for the role and account are cached for `duration`
+ seconds, which defaults to 1 hour. Likewise, the token obtained from the
+ OAuth2 server is cached based on the expiration provided in the token.
+ """
+
+ def __init__(
+ self,
+ role,
+ url,
+ username,
+ password,
+ headers=None,
+ oauth2_params=None,
+ duration=3600,
+ no_verify=False,
+ ):
+ super().__init__(role, duration)
+ self._role = role
+ self._url = url
+ self._username = username
+ self._password = password
+ self._headers = {} if headers is None else headers
+ self._oauth2_params = {} if oauth2_params is None else oauth2_params
+ self._no_verify = no_verify
+
+ # `max_age` isn't used as we use the expiration that is provided in
+ # token response. While, `ExpriringValue` requires a max_age, it is
+ # only used if the return value from the `refresh_fn` isn't
+ # a `ValueWithExpiry`, which ours always is. For safety, we'll set
+ # it to 60 seconds just in case, so we never overload the server.
+ self._cached_token = ExpiringValue(self._request_token, max_age=60)
+
+ def token(self, refresh=False):
+ """Returns an OAuth2 access token from the OAuth2 server.
+
+ This value is cached by default. If refresh is True, the value is
+ refreshed first, then returned. See the module documentation for the
+ exceptions that may be raised.
+ """
+ return self._cached_token.value(refresh)
+
+ def _request_token(self):
+ """Returns a non-cached OAuth2 access token from the OAuth2 server.
+
+ See the module documentation for the exceptions that may be raised.
+ """
+ LOG.info("Fetching OAuth2 access token")
+ with requests.Session() as s:
+ s.headers.update(self._headers)
+ form = {
+ "userName": self._username,
+ "password": self._password,
+ "grant_type": "password",
+ }
+ form.update(self._oauth2_params)
+ resp = s.post(self._url, data=form, verify=not self._no_verify)
+
+ if resp.status_code == 400: # RFC6749 section 5.2
+ try:
+ desc = resp.json().get("error_description", "unknown error")
+ except ValueError:
+ desc = f"non-JSON response: {resp.text}"
+ raise IDPAccessDeniedException(f"Could not authenticate: {desc}")
+ if not 200 <= resp.status_code < 300:
+ raise IDPInvalidResponseException(
+ f"{resp.status_code} response from {self._url}"
+ )
+
+ token_data = resp.json()
+ access_token = token_data.get("access_token")
+ expires_in = token_data.get("expires_in")
+
+ if not access_token or not expires_in:
+ raise IDPInvalidResponseException(
+ "Cannot extract access token or expiration from response"
+ )
+
+ return ValueWithExpiry(access_token, ttl=expires_in)
+
+ def credentials(self, acct_id):
+ role_arn = f"arn:aws:iam::{acct_id}:role/{self._role}"
+ LOG.info("Assuming role with web identity for %s", role_arn)
+ assumed_role = boto3.client("sts").assume_role_with_web_identity(
+ RoleArn=role_arn,
+ RoleSessionName=f"AWSRunSession-{self._username}",
+ WebIdentityToken=self.token(),
+ DurationSeconds=self._duration,
+ )
+
+ if not assumed_role:
+ raise AWSAssumeRoleException(f"Cannot assume role: {role_arn}")
+
+ return assumed_role["Credentials"]
+
+
class CredsViaSAML(CachingSessionProvider):
"""A session provider that uses federated SAML authentication.
@@ -359,7 +545,7 @@ class CredsViaSAML(CachingSessionProvider):
credentials for an AWS account.
The `role` is the name of the IAM role (not the ARN) to assume in the AWS
- account. This is used when making the underlying AWS assume_role_with_saml
+ account. This is used when making the underlying AWS `assume_role_with_saml`
API call.
The `url` is the URL to the IdP web server that will provide a SAML
diff --git a/src/awsrun/session/azure.py b/src/awsrun/session/azure.py
index 6817995..5631a12 100644
--- a/src/awsrun/session/azure.py
+++ b/src/awsrun/session/azure.py
@@ -126,9 +126,6 @@ def wrapper(*scopes, **kwargs):
return wrapper
-# pylint: disable=too-few-public-methods
-
-
class CredsViaAzureDefault(SessionProvider):
"""A session provider that obtains credentials from a variety of sources.
diff --git a/tests/integration/test_acctload.py b/tests/integration/test_acctload.py
index a86912e..d631488 100644
--- a/tests/integration/test_acctload.py
+++ b/tests/integration/test_acctload.py
@@ -4,8 +4,6 @@
# SPDX-License-Identifier: Apache-2.0
#
-# pylint: disable=redefined-outer-name,missing-docstring
-
import json
from datetime import datetime, timedelta, timezone
from pathlib import Path
diff --git a/tests/unit/test_acctload.py b/tests/unit/test_acctload.py
index 751af03..76e8d98 100644
--- a/tests/unit/test_acctload.py
+++ b/tests/unit/test_acctload.py
@@ -4,8 +4,6 @@
# SPDX-License-Identifier: Apache-2.0
#
-# pylint: disable=redefined-outer-name,missing-docstring
-
import pytest
from awsrun import acctload
@@ -69,41 +67,41 @@ def many_acct_list():
@pytest.mark.parametrize(
"test_input, include, exclude, expected",
[
- (["100200300400"], {}, {}, set(["100200300400"])),
+ (["100200300400"], {}, {}, {"100200300400"}),
(["100200300400"], {"status": ["suspended"]}, {}, set()),
(
["300400100200", "100200300400"],
{},
{},
- set(["300400100200", "100200300400"]),
+ {"300400100200", "100200300400"},
),
(
["300400100200", "100200300400"],
{},
{"status": ["suspended"]},
- set(["100200300400"]),
+ {"100200300400"},
),
- (["300400100200", "100200300400"], {"env": ["dev"]}, {}, set(["300400100200"])),
- ([], {}, {}, set(["100200300400", "200300400100", "300400100200"])),
- ([], {"env": ["prod", "nonprod"]}, {}, set(["100200300400", "200300400100"])),
+ (["300400100200", "100200300400"], {"env": ["dev"]}, {}, {"300400100200"}),
+ ([], {}, {}, {"100200300400", "200300400100", "300400100200"}),
+ ([], {"env": ["prod", "nonprod"]}, {}, {"100200300400", "200300400100"}),
(
[],
{"status": ["active", "no_such_value"]},
{},
- set(["100200300400", "200300400100"]),
+ {"100200300400", "200300400100"},
),
(
[],
{"status": ["active"]},
{"env": ["nonprod", "dev"]},
- set(["100200300400"]),
+ {"100200300400"},
),
([], {"status": ["active"]}, {"env": ["nonprod", "dev", "prod"]}, set()),
(
[],
{"status": ["active", "suspended"]},
{"status": ["active", "no_such_value"]},
- set(["300400100200"]),
+ {"300400100200"},
),
],
)
@@ -183,10 +181,6 @@ def test_load_basic_with_list_of_accts(test_input, expected):
{"10": {"bu": "a"}, "20": {"bu": "b"}},
[{"id": "10", "bu": "a"}, {"id": "20", "bu": "b"}],
),
- (
- {"10": {"bu": "a"}, "20": {"bu": "b"}},
- [{"id": "10", "bu": "a"}, {"id": "20", "bu": "b"}],
- ),
],
)
def test_load_basic_with_dict_of_accts(test_input, expected):
@@ -367,7 +361,6 @@ def test_load_with_invalid_path(path, test_input, expected):
],
)
def test_convert_keys_to_valid_attribute_names(test_input, expected):
- # pylint: disable=protected-access
acctload._convert_keys_to_valid_attribute_names(test_input)
assert test_input == expected, "dict key names not rewritten"
@@ -396,5 +389,4 @@ def test_convert_keys_to_valid_attribute_names(test_input, expected):
],
)
def test_make_valid_attribute_name(test_input, expected):
- # pylint: disable=protected-access
assert acctload._make_valid_attribute_name(test_input) == expected
diff --git a/tests/unit/test_argparse.py b/tests/unit/test_argparse.py
index bfb1c0e..4a54b27 100644
--- a/tests/unit/test_argparse.py
+++ b/tests/unit/test_argparse.py
@@ -4,8 +4,6 @@
# SPDX-License-Identifier: Apache-2.0
#
-# pylint: disable=redefined-outer-name,missing-docstring
-
import argparse
import pytest
diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py
index ace9c36..94feaf6 100644
--- a/tests/unit/test_cache.py
+++ b/tests/unit/test_cache.py
@@ -4,91 +4,377 @@
# SPDX-License-Identifier: Apache-2.0
#
-# pylint: disable=redefined-outer-name,missing-docstring
-
+import json
import os
import random
import time
from datetime import timedelta
+import pytest
from freezegun import freeze_time
from awsrun import cache
-def test_expiring_value_caching():
- with freeze_time() as frozen_datetime:
- ev = cache.ExpiringValue(random.random, max_age=300)
- initial_value = ev.value()
+class TestExpiringValue:
+ """Tests for ExpiringValue."""
- frozen_datetime.tick(delta=timedelta(seconds=60))
- assert ev.value() == initial_value, (
- "value was different, should have been cached"
- )
+ def test_caching(self):
+ with freeze_time() as frozen_datetime:
+ ev = cache.ExpiringValue(random.random, max_age=300)
+ initial_value = ev.value()
+
+ frozen_datetime.tick(delta=timedelta(seconds=60))
+ assert ev.value() == initial_value, (
+ "value was different, should have been cached"
+ )
+
+ # Fast forward to 5 minutes after we first cached the value
+ frozen_datetime.tick(delta=timedelta(seconds=241))
+ second_value = ev.value()
+ assert second_value != initial_value, (
+ "value was the same, should have expired"
+ )
+
+ # Make sure the second value was cached
+ frozen_datetime.tick(delta=timedelta(seconds=60))
+ assert ev.value() == second_value, (
+ "value was different, should have been cached"
+ )
+
+ def test_no_caching(self):
+ ev = cache.ExpiringValue(random.random, max_age=0)
+ value1 = ev.value()
+ value2 = ev.value()
+ value3 = ev.value()
+ assert len({value1, value2, value3}) == 3, "values should not be cached"
+
+
+class TestValueWithExpiry:
+ """Tests for the ValueWithExpiry wrapper class."""
+
+ def test_ttl_sets_expires_at(self):
+ with freeze_time("2025-01-01 12:00:00"):
+ wrapper = cache.ValueWithExpiry("test_value", ttl=3600)
+ assert wrapper.value == "test_value"
+ # expires_at should be current time + ttl
+ expected_expiry = time.time() + 3600
+ assert wrapper.expires_at == expected_expiry
+
+ def test_expires_at_sets_directly(self):
+ wrapper = cache.ValueWithExpiry("test_value", expires_at=1234567890)
+ assert wrapper.value == "test_value"
+ assert wrapper.expires_at == 1234567890
- # Fast forward to 5 minutes after we first cached the value
- frozen_datetime.tick(delta=timedelta(seconds=241))
- second_value = ev.value()
- assert second_value != initial_value, "value was the same, should have expired"
+ def test_must_specify_ttl_or_expires_at(self):
+ with pytest.raises(ValueError, match="Must specify either ttl or expires_at"):
+ cache.ValueWithExpiry("test_value")
- # Make sure the second value was cached
- frozen_datetime.tick(delta=timedelta(seconds=60))
- assert ev.value() == second_value, (
- "value was different, should have been cached"
+ def test_cannot_specify_both_ttl_and_expires_at(self):
+ with pytest.raises(ValueError, match="Cannot specify both ttl and expires_at"):
+ cache.ValueWithExpiry("test_value", ttl=3600, expires_at=1234567890)
+
+ def test_repr(self):
+ wrapper = cache.ValueWithExpiry("test_value", expires_at=1234567890)
+ assert (
+ repr(wrapper)
+ == "ValueWithExpiry(value='test_value', expires_at=1234567890)"
)
-def test_expiring_value_no_caching():
- ev = cache.ExpiringValue(random.random, max_age=0)
- value1 = ev.value()
- value2 = ev.value()
- value3 = ev.value()
- assert len({value1, value2, value3}) == 3, "values should not be cached"
+class TestExpiringValueWithDynamicExpiry:
+ """Tests for ExpiringValue with ValueWithExpiry."""
+
+ def test_dynamic_expiry_with_ttl(self):
+ with freeze_time() as frozen_datetime:
+ call_count = 0
+
+ def refresh_fn():
+ nonlocal call_count
+ call_count += 1
+ # Return value with 60 second TTL
+ return cache.ValueWithExpiry({"token": f"token_{call_count}"}, ttl=60)
+
+ ev = cache.ExpiringValue(
+ refresh_fn, max_age=300
+ ) # max_age ignored when wrapper used
+ initial_value = ev.value()
+ assert initial_value == {"token": "token_1"}
+
+ # After 30 seconds, should still be cached
+ frozen_datetime.tick(delta=timedelta(seconds=30))
+ assert ev.value() == {"token": "token_1"}
+
+ # After 61 seconds total, should expire (using wrapper's 60s TTL, not 300s max_age)
+ frozen_datetime.tick(delta=timedelta(seconds=31))
+ assert ev.value() == {"token": "token_2"}
+
+ def test_dynamic_expiry_with_expires_at(self):
+ with freeze_time() as frozen_datetime:
+ call_count = 0
+
+ def refresh_fn():
+ nonlocal call_count
+ call_count += 1
+ # Return value with absolute expiry 120 seconds from now
+ return cache.ValueWithExpiry(
+ {"token": f"token_{call_count}"}, expires_at=time.time() + 120
+ )
+
+ ev = cache.ExpiringValue(refresh_fn, max_age=300)
+ initial_value = ev.value()
+ assert initial_value == {"token": "token_1"}
+
+ # After 100 seconds, should still be cached
+ frozen_datetime.tick(delta=timedelta(seconds=100))
+ assert ev.value() == {"token": "token_1"}
+
+ # After 121 seconds total, should expire
+ frozen_datetime.tick(delta=timedelta(seconds=21))
+ assert ev.value() == {"token": "token_2"}
+
+ def test_mixed_plain_and_wrapped_values(self):
+ """Test that switching between plain values and wrapped values works."""
+ with freeze_time() as frozen_datetime:
+ call_count = 0
+ use_dynamic_expiry = False
+
+ def refresh_fn():
+ nonlocal call_count
+ call_count += 1
+ if use_dynamic_expiry:
+ return cache.ValueWithExpiry(f"wrapped_{call_count}", ttl=30)
+ return f"plain_{call_count}"
+
+ ev = cache.ExpiringValue(refresh_fn, max_age=60)
+
+ # First call returns plain value
+ assert ev.value() == "plain_1"
+
+ # Force refresh with wrapped value
+ use_dynamic_expiry = True
+ assert ev.value(refresh=True) == "wrapped_2"
+
+ # Should expire after 30 seconds (wrapper TTL)
+ frozen_datetime.tick(delta=timedelta(seconds=31))
+ use_dynamic_expiry = False
+ assert ev.value() == "plain_3"
+
+
+class TestPersistentExpiringValueWithDynamicExpiry:
+ """Tests for PersistentExpiringValue with ValueWithExpiry."""
+
+ def test_dynamic_expiry_persisted(self, tmp_path):
+ with freeze_time() as frozen_datetime:
+ cache_file = tmp_path / "test.dat"
+ call_count = 0
+
+ def refresh_fn():
+ nonlocal call_count
+ call_count += 1
+ return cache.ValueWithExpiry({"token": f"token_{call_count}"}, ttl=60)
+
+ ev = cache.PersistentExpiringValue(refresh_fn, cache_file, max_age=300)
+ initial_value = ev.value()
+ assert initial_value == {"token": "token_1"}
+ assert cache_file.exists()
+
+ # After 30 seconds, should still be cached
+ frozen_datetime.tick(delta=timedelta(seconds=30))
+ assert ev.value() == {"token": "token_1"}
+
+ # After 61 seconds total, should expire (using wrapper's 60s TTL)
+ frozen_datetime.tick(delta=timedelta(seconds=31))
+ assert ev.value() == {"token": "token_2"}
+
+ def test_dynamic_expiry_survives_reload(self, tmp_path):
+ """Test that dynamic expiry is restored when loading from disk."""
+ with freeze_time() as frozen_datetime:
+ cache_file = tmp_path / "test.dat"
+
+ def refresh_fn():
+ return cache.ValueWithExpiry({"token": "original"}, ttl=120)
+
+ # Create first instance and cache the value
+ ev1 = cache.PersistentExpiringValue(refresh_fn, cache_file, max_age=300)
+ assert ev1.value() == {"token": "original"}
+
+ # Simulate time passing
+ frozen_datetime.tick(delta=timedelta(seconds=60))
+
+ # Create new instance (simulating process restart)
+ def refresh_fn_new():
+ return cache.ValueWithExpiry({"token": "refreshed"}, ttl=120)
+
+ ev2 = cache.PersistentExpiringValue(refresh_fn_new, cache_file, max_age=300)
+ # Should load cached value and restore dynamic expiry
+ assert ev2.value() == {"token": "original"}
-def test_persistent_expiring_value_caching(tmp_path):
- with freeze_time() as frozen_datetime:
+ # After another 61 seconds (121 total), should expire
+ frozen_datetime.tick(delta=timedelta(seconds=61))
+ assert ev2.value() == {"token": "refreshed"}
+
+ def test_backwards_compatible_with_plain_cache_files(self, tmp_path):
+ """Test that existing cache files without metadata still work."""
cache_file = tmp_path / "test.dat"
- assert not cache_file.exists()
- ev = cache.PersistentExpiringValue(random.random, cache_file, max_age=300)
- initial_value = ev.value()
+ # Manually create a cache file in the old format (no metadata)
+ with cache_file.open("w", encoding="utf-8") as f:
+ json.dump({"legacy": "data"}, f)
+
+ # Set file mtime to 400 seconds in the past so it's already expired
+ old_mtime = time.time() - 400
+ os.utime(str(cache_file), (old_mtime, old_mtime))
+
+ call_count = 0
+
+ def refresh_fn():
+ nonlocal call_count
+ call_count += 1
+ return {"new": f"data_{call_count}"}
+
+ ev = cache.PersistentExpiringValue(refresh_fn, cache_file, max_age=300)
+
+ # File is older than max_age, should refresh
+ assert ev.value() == {"new": "data_1"}
+ assert call_count == 1
+
+ # Now create a fresh file that's not expired
+ cache_file2 = tmp_path / "test2.dat"
+ with cache_file2.open("w", encoding="utf-8") as f:
+ json.dump({"legacy": "data"}, f)
+
+ # mtime is now (just created), so not expired
+ call_count = 0
+ ev2 = cache.PersistentExpiringValue(refresh_fn, cache_file2, max_age=300)
+
+ # File is fresh, should load legacy data
+ assert ev2.value() == {"legacy": "data"}
+ assert call_count == 0, "Should not have called refresh_fn"
+
+ def test_dynamic_expiry_with_zero_max_age_still_caches(self, tmp_path):
+ """Test that ValueWithExpiry causes caching even when max_age=0."""
+ with freeze_time() as frozen_datetime:
+ cache_file = tmp_path / "test.dat"
+ call_count = 0
+
+ def refresh_fn():
+ nonlocal call_count
+ call_count += 1
+ return cache.ValueWithExpiry({"token": f"token_{call_count}"}, ttl=60)
+
+ ev = cache.PersistentExpiringValue(refresh_fn, cache_file, max_age=0)
+ assert ev.value() == {"token": "token_1"}
+ assert cache_file.exists(), (
+ "File should be created when using ValueWithExpiry"
+ )
+
+ # Should still be cached after 30 seconds
+ frozen_datetime.tick(delta=timedelta(seconds=30))
+ assert ev.value() == {"token": "token_1"}
+ assert call_count == 1, "Should not have called refresh_fn again"
+
+ def test_expiry_file_created_and_removed(self, tmp_path):
+ """Test that expiry file is created when using dynamic expiry and removed otherwise."""
+ cache_file = tmp_path / "test.dat"
+ expiry_file = tmp_path / "test.dat.expiry"
+
+ call_count = 0
+ use_dynamic_expiry = True
+
+ def refresh_fn():
+ nonlocal call_count
+ call_count += 1
+ if use_dynamic_expiry:
+ return cache.ValueWithExpiry({"token": f"token_{call_count}"}, ttl=60)
+ return {"token": f"token_{call_count}"}
+
+ ev = cache.PersistentExpiringValue(refresh_fn, cache_file, max_age=300)
+
+ # First call with dynamic expiry - expiry file should be created
+ ev.value()
assert cache_file.exists()
+ assert expiry_file.exists(), (
+ "Expiry file should be created when using ValueWithExpiry"
+ )
- frozen_datetime.tick(delta=timedelta(seconds=60))
- assert ev.value() == initial_value, (
- "value was different, should have been cached"
+ # Force refresh without dynamic expiry - expiry file should be removed
+ use_dynamic_expiry = False
+ ev.value(refresh=True)
+ assert cache_file.exists()
+ assert not expiry_file.exists(), (
+ "Expiry file should be removed when not using ValueWithExpiry"
)
- # Fast forward to 5 minutes after we first cached the value
- frozen_datetime.tick(delta=timedelta(seconds=241))
- second_value = ev.value()
- assert second_value != initial_value, "value was the same, should have expired"
+ def test_corrupted_expiry_file_raises_error(self, tmp_path):
+ """Test that a corrupted expiry file raises an error instead of silently failing."""
+ cache_file = tmp_path / "test.dat"
+ expiry_file = tmp_path / "test.dat.expiry"
- # Because freezegun cannot adjust the time of the OS and timestamps
- # of files, we'll have to update the mtime of the cache file ourself.
- mtime = time.mktime(frozen_datetime().timetuple())
- os.utime(str(cache_file), (mtime, mtime))
+ # Create cache file with valid data
+ with cache_file.open("w", encoding="utf-8") as f:
+ json.dump({"token": "cached"}, f)
- # Make sure the second value was cached
- frozen_datetime.tick(delta=timedelta(seconds=60))
- assert ev.value() == second_value, (
- "value was different, should have been cached"
- )
+ # Create corrupted expiry file
+ expiry_file.write_text("not_a_valid_timestamp", encoding="utf-8")
+
+ def refresh_fn():
+ return {"token": "refreshed"}
+
+ ev = cache.PersistentExpiringValue(refresh_fn, cache_file, max_age=300)
+
+ with pytest.raises(ValueError):
+ ev.value()
+
+
+class TestPersistentExpiringValue:
+ """Tests for PersistentExpiringValue."""
+
+ def test_caching(self, tmp_path):
+ with freeze_time() as frozen_datetime:
+ cache_file = tmp_path / "test.dat"
+ assert not cache_file.exists()
+ ev = cache.PersistentExpiringValue(random.random, cache_file, max_age=300)
+ initial_value = ev.value()
+ assert cache_file.exists()
-def test_persistent_expiring_value_no_caching(tmp_path):
- cache_file = tmp_path / "test.dat"
- assert not cache_file.exists()
+ frozen_datetime.tick(delta=timedelta(seconds=60))
+ assert ev.value() == initial_value, (
+ "value was different, should have been cached"
+ )
- ev = cache.PersistentExpiringValue(random.random, cache_file, max_age=0)
- value1 = ev.value()
- assert not cache_file.exists()
+ # Fast forward to 5 minutes after we first cached the value
+ frozen_datetime.tick(delta=timedelta(seconds=241))
+ second_value = ev.value()
+ assert second_value != initial_value, (
+ "value was the same, should have expired"
+ )
- value2 = ev.value()
- assert not cache_file.exists()
+ # Because freezegun cannot adjust the time of the OS and timestamps
+ # of files, we'll have to update the mtime of the cache file ourself.
+ mtime = time.mktime(frozen_datetime().timetuple())
+ os.utime(str(cache_file), (mtime, mtime))
- value3 = ev.value()
- assert not cache_file.exists()
+ # Make sure the second value was cached
+ frozen_datetime.tick(delta=timedelta(seconds=60))
+ assert ev.value() == second_value, (
+ "value was different, should have been cached"
+ )
+
+ def test_no_caching(self, tmp_path):
+ cache_file = tmp_path / "test.dat"
+ assert not cache_file.exists()
+
+ ev = cache.PersistentExpiringValue(random.random, cache_file, max_age=0)
+ value1 = ev.value()
+ assert not cache_file.exists()
+
+ value2 = ev.value()
+ assert not cache_file.exists()
+
+ value3 = ev.value()
+ assert not cache_file.exists()
- assert len({value1, value2, value3}) == 3, "values should not be cached"
+ assert len({value1, value2, value3}) == 3, "values should not be cached"
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 5e385c0..efdd39f 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -4,8 +4,6 @@
# SPDX-License-Identifier: Apache-2.0
#
-# pylint: disable=redefined-outer-name,missing-docstring
-
import json
import pytest
@@ -199,7 +197,6 @@ def test_const_type(const_value, test_input, expected):
(["a", 10], "a", True),
(["a", 10], 10, True),
([], 30, False),
- ([], 30, False),
],
)
def test_choice_type(choices, test_input, expected):
diff --git a/tests/unit/test_runner.py b/tests/unit/test_runner.py
index fa732d8..c66bd5f 100644
--- a/tests/unit/test_runner.py
+++ b/tests/unit/test_runner.py
@@ -4,8 +4,6 @@
# SPDX-License-Identifier: Apache-2.0
#
-# pylint: disable=redefined-outer-name,missing-docstring
-
import pytest
from awsrun.runner import AccountRunner, Command