diff --git a/CMakeLists.txt b/CMakeLists.txt index ac852cec..44a829e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,8 +22,9 @@ set(LUNCHBOX_LICENSE LGPL) set(LUNCHBOX_DEB_DEPENDS libboost-regex-dev libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-test-dev libboost-thread-dev libhwloc-dev avahi-daemon libavahi-client-dev - libleveldb-dev libopenmpi-dev openmpi-bin) -set(LUNCHBOX_PORT_DEPENDS boost) + libleveldb-dev libopenmpi-dev openmpi-bin libmemcached-dev libmemcached-tools + memcached) +set(LUNCHBOX_PORT_DEPENDS boost libmemcached memcached) set(COMMON_PROJECT_DOMAIN ch.eyescale) include(Common) @@ -34,8 +35,8 @@ else() list(APPEND COMMON_FIND_PACKAGE_DEFINES LUNCHBOX_USE_V1_API) endif() -common_find_package(Boost REQUIRED COMPONENTS regex serialization filesystem system - thread unit_test_framework) +common_find_package(Boost REQUIRED COMPONENTS + filesystem regex serialization system thread unit_test_framework) common_find_package(hwloc) common_find_package(leveldb) if(LUNCHBOX_USE_MPI) @@ -43,6 +44,7 @@ if(LUNCHBOX_USE_MPI) endif() common_find_package(OpenMP) common_find_package(Servus REQUIRED) +common_find_package(libmemcached) common_find_package(skv) common_find_package_post() diff --git a/doc/Changelog.md b/doc/Changelog.md index 7a4bdebf..3e72293e 100644 --- a/doc/Changelog.md +++ b/doc/Changelog.md @@ -2,6 +2,9 @@ # git master +* [263](https://github.com/Eyescale/Lunchbox/pull/263): + Add memcached PersistentMap backend, add + PersistentMap::createCache * [252](https://github.com/Eyescale/Lunchbox/pull/252): Monitor::set() returns old value diff --git a/lunchbox/CMakeLists.txt b/lunchbox/CMakeLists.txt index c1dd47f4..16ec2aec 100644 --- a/lunchbox/CMakeLists.txt +++ b/lunchbox/CMakeLists.txt @@ -138,6 +138,9 @@ endif() if(LEVELDB_FOUND) list(APPEND LUNCHBOX_LINK_LIBRARIES ${LEVELDB_LIBRARIES}) endif() +if(LIBMEMCACHED_FOUND) + list(APPEND LUNCHBOX_LINK_LIBRARIES ${libmemcached_LIBRARIES}) +endif() if(SKV_FOUND) list(APPEND LUNCHBOX_LINK_LIBRARIES skv_client fxlogger) endif() diff --git a/lunchbox/leveldb/persistentMap.h b/lunchbox/leveldb/persistentMap.h index 7bc5c91e..8607edd5 100644 --- a/lunchbox/leveldb/persistentMap.h +++ b/lunchbox/leveldb/persistentMap.h @@ -1,5 +1,5 @@ -/* Copyright (c) 2014, Stefan.Eilemann@epfl.ch +/* Copyright (c) 2014-2016, Stefan.Eilemann@epfl.ch * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License version 2.1 as published @@ -68,12 +68,6 @@ class PersistentMap : public detail::PersistentMap return std::string(); } - bool contains( const std::string& key ) const final - { - std::string value; - return _db->Get( db::ReadOptions(), key, &value ).ok(); - } - bool flush() final { /*NOP?*/ return true; } private: diff --git a/lunchbox/memcached/persistentMap.h b/lunchbox/memcached/persistentMap.h new file mode 100644 index 00000000..d873ccdc --- /dev/null +++ b/lunchbox/memcached/persistentMap.h @@ -0,0 +1,136 @@ + +/* Copyright (c) 2016, Stefan.Eilemann@epfl.ch + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef LUNCHBOX_USE_LIBMEMCACHED +#include + +namespace lunchbox +{ +namespace memcached +{ +namespace +{ +memcached_st* _getInstance( const servus::URI& uri ) +{ + const std::string& host = uri.getHost(); + const int16_t port = uri.getPort() ? uri.getPort() : 11211; + memcached_st* instance = memcached_create( 0 ); + + if( uri.getHost().empty( )) + { + const char* servers = ::getenv( "MEMCACHED_SERVERS" ); + if( servers ) + { + instance->server_failure_limit = 10; + std::string data = servers; + while( !data.empty( )) + { + const size_t comma = data.find( ',' ); + const std::string& server = data.substr( 0, comma ); + + const size_t colon = server.find( ':' ); + if( colon == std::string::npos ) + memcached_server_add( instance, server.c_str(), port ); + else + memcached_server_add( instance, + server.substr( 0, colon ).c_str(), + std::stoi( server.substr( colon+1 ))); + + if( comma == std::string::npos ) + data.clear(); + else + data = data.substr( comma + 1 ); + } + + } + else + memcached_server_add( instance, "127.0.0.1", port ); + + } + else + memcached_server_add( instance, host.c_str(), port ); + + return instance; +} +} + +// memcached has relative strict requirements on keys (no whitespace or control +// characters, max length). We therefore hash incoming keys and use their string +// representation. + +class PersistentMap : public detail::PersistentMap +{ +public: + explicit PersistentMap( const servus::URI& uri ) + : _instance( _getInstance( uri )) + , _lastError( MEMCACHED_SUCCESS ) + { + if( !_instance ) + throw std::runtime_error( std::string( "Open of " ) + + std::to_string( uri ) + " failed" ); + } + + virtual ~PersistentMap() { memcached_free( _instance ); } + + static bool handles( const servus::URI& uri ) + { return uri.getScheme() == "memcached"; } + + bool insert( const std::string& key, const void* data, const size_t size ) + final + { + const std::string& hash = servus::make_uint128( key ).getString(); + const memcached_return_t ret = + memcached_set( _instance, hash.c_str(), hash.length(), + (const char*)data, size, (time_t)0, (uint32_t)0 ); + + if( ret != MEMCACHED_SUCCESS && _lastError != ret ) + { + LBWARN << "memcached_set failed: " + << memcached_strerror( _instance, ret ) << std::endl; + _lastError = ret; + } + + return ret == MEMCACHED_SUCCESS; + } + + std::string operator [] ( const std::string& key ) const final + { + const std::string& hash = servus::make_uint128( key ).getString(); + size_t size = 0; + uint32_t flags = 0; + memcached_return_t ret = MEMCACHED_SUCCESS; + char* data = memcached_get( _instance, hash.c_str(), hash.length(), + &size, &flags, &ret ); + std::string value; + if( ret == MEMCACHED_SUCCESS ) + { + value.assign( data, data + size ); + free( data ); + } + return value; + } + + bool flush() final { /*NOP?*/ return true; } + +private: + memcached_st* const _instance; + memcached_return_t _lastError; +}; +} +} + +#endif diff --git a/lunchbox/persistentMap.cpp b/lunchbox/persistentMap.cpp index 4252e515..da3d6fed 100644 --- a/lunchbox/persistentMap.cpp +++ b/lunchbox/persistentMap.cpp @@ -1,5 +1,5 @@ -/* Copyright (c) 2014-2015, Stefan.Eilemann@epfl.ch +/* Copyright (c) 2014-2016, Stefan.Eilemann@epfl.ch * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License version 2.1 as published @@ -35,7 +35,6 @@ class PersistentMap virtual std::string operator [] ( const std::string& key ) const = 0; virtual bool fetch( const std::string&, const size_t ) const { return true; } - virtual bool contains( const std::string& key ) const = 0; virtual bool flush() = 0; bool swap; @@ -49,6 +48,7 @@ class PersistentMap // Impls - need detail::PersistentMap interface above #include "leveldb/persistentMap.h" +#include "memcached/persistentMap.h" #include "skv/persistentMap.h" namespace @@ -60,6 +60,10 @@ lunchbox::detail::PersistentMap* _newImpl( const servus::URI& uri ) if( lunchbox::leveldb::PersistentMap::handles( uri )) return new lunchbox::leveldb::PersistentMap( uri ); #endif +#ifdef LUNCHBOX_USE_LIBMEMCACHED + if( lunchbox::memcached::PersistentMap::handles( uri )) + return new lunchbox::memcached::PersistentMap( uri ); +#endif #ifdef LUNCHBOX_USE_SKV if( lunchbox::skv::PersistentMap::handles( uri )) return new lunchbox::skv::PersistentMap( uri ); @@ -102,12 +106,32 @@ PersistentMap::~PersistentMap() delete _impl; } +PersistentMapPtr PersistentMap::createCache() +{ +#ifdef LUNCHBOX_USE_LIBMEMCACHED + if( ::getenv( "MEMCACHED_SERVERS" )) + return PersistentMapPtr( new PersistentMap( "memcached://" )); +#endif +#ifdef LUNCHBOX_USE_LEVELDB + const char* leveldb = ::getenv( "LEVELDB_CACHE" ); + if( leveldb ) + return PersistentMapPtr( new PersistentMap( + std::string( "leveldb://" ) + leveldb )); +#endif + + return PersistentMapPtr(); +} + bool PersistentMap::handles( const servus::URI& uri ) { #ifdef LUNCHBOX_USE_LEVELDB if( lunchbox::leveldb::PersistentMap::handles( uri )) return true; #endif +#ifdef LUNCHBOX_USE_LIBMEMCACHED + if( lunchbox::memcached::PersistentMap::handles( uri )) + return true; +#endif #ifdef LUNCHBOX_USE_SKV if( lunchbox::skv::PersistentMap::handles( uri )) return true; @@ -127,8 +151,8 @@ size_t PersistentMap::setQueueDepth( const size_t depth ) return _impl->setQueueDepth( depth ); } -bool PersistentMap::_insert( const std::string& key, const void* data, - const size_t size ) +bool PersistentMap::insert( const std::string& key, const void* data, + const size_t size ) { #ifdef HISTOGRAM ++_impl->keys[ key.size() ]; @@ -147,11 +171,6 @@ bool PersistentMap::fetch( const std::string& key, const size_t sizeHint ) const return _impl->fetch( key, sizeHint ); } -bool PersistentMap::contains( const std::string& key ) const -{ - return _impl->contains( key ); -} - bool PersistentMap::flush() { return _impl->flush(); diff --git a/lunchbox/persistentMap.h b/lunchbox/persistentMap.h index c0b8a284..721feca6 100644 --- a/lunchbox/persistentMap.h +++ b/lunchbox/persistentMap.h @@ -1,5 +1,5 @@ -/* Copyright (c) 2014-2015, Stefan.Eilemann@epfl.ch +/* Copyright (c) 2014-2016, Stefan.Eilemann@epfl.ch * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License version 2.1 as published @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -40,6 +41,9 @@ namespace lunchbox { namespace detail { class PersistentMap; } +class PersistentMap; +typedef boost::shared_ptr< PersistentMap > PersistentMapPtr; + /** * Unified interface to save key-value pairs in a persistent store. * @@ -54,8 +58,18 @@ class PersistentMap : public boost::noncopyable * Depending on the URI scheme an implementation backend is chosen. If no * URI is given, a default one is selected. Available implementations are: * * leveldb://path (if LUNCHBOX_USE_LEVELDB is defined) + * * memcached://[server] (if LUNCHBOX_USE_LIBMEMCACHED is defined) * * skv://path_to_config\#pdsname (if LUNCHBOX_USE_SKV is defined) * + * If no path is given for leveldb, the implementation uses + * persistentMap.leveldb in the current working directory. + * + * If no servers are given for memcached, the implementation uses all + * servers in the MEMCACHED_SERVERS environment variable, or + * 127.0.0.1. MEMCACHED_SERVERS contains a comma-separated list of + * servers. Each server contains the address, and optionally a + * colon-separated port number. + * * @param uri the storage backend and destination. * @throw std::runtime_error if no suitable implementation is found. * @throw std::runtime_error if opening the leveldb failed. @@ -79,6 +93,24 @@ class PersistentMap : public boost::noncopyable LUNCHBOX_API static bool handles( const servus::URI& uri ); /** + * Create a map which can be used for caching IO on the local system. + * + * The concrete implementation used depends on the system setup and + * available backend implementations. If no suitable implementation is + * found, a null pointer is returned. + * + * The current implementation returns: + * * A memcached-backed cache if libmemcached is available and the + * environment variable MEMCACHED_SERVERS is set (see constructor + * documentation for details). + * * A leveldb-backed cache if leveldb is available and LEVELDB_CACHE is set + * to the path for the leveldb storage. + * + * @return a PersistentMap for caching IO, or 0. + */ + LUNCHBOX_API static PersistentMapPtr createCache(); + + /** * Set the maximum number of asynchronous outstanding write operations. * * Some backend implementations support asynchronous writes, which can be @@ -104,6 +136,8 @@ class PersistentMap : public boost::noncopyable */ template< class V > bool insert( const std::string& key, const V& value ) { return _insert( key, value, boost::has_trivial_assign< V >( )); } + LUNCHBOX_API bool insert( const std::string& key, const void* data, + size_t size ); /** * Insert or update a vector of values in the database. @@ -179,10 +213,7 @@ class PersistentMap : public boost::noncopyable * @return false on error, true otherwise. * @version 1.11 */ - LUNCHBOX_API bool fetch( const std::string& key, size_t sizeHint = 0 ) const; - - /** @return true if the key exists. @version 1.9.2 */ - LUNCHBOX_API bool contains( const std::string& key ) const; + LUNCHBOX_API bool fetch( const std::string& key, size_t sizeHint=0 ) const; /** Flush outstanding operations to the backend storage. @version 1.11 */ LUNCHBOX_API bool flush(); @@ -193,8 +224,6 @@ class PersistentMap : public boost::noncopyable private: detail::PersistentMap* const _impl; - LUNCHBOX_API bool _insert( const std::string& key, const void* data, - const size_t size ); LUNCHBOX_API bool _swap() const; @@ -204,7 +233,7 @@ class PersistentMap : public boost::noncopyable template< size_t N > bool _insert( const std::string& k, char const (& v)[N], const boost::true_type&) { - return _insert( k, (void*)v, N - 1 ); // strip '0' + return insert( k, (void*)v, N - 1 ); // strip '0' } template< class V > @@ -212,7 +241,7 @@ class PersistentMap : public boost::noncopyable { if( boost::is_pointer< V >::value ) LBTHROW( std::runtime_error( "Can't insert pointers" )); - return _insert( k, &v, sizeof( V )); + return insert( k, &v, sizeof( V )); } template< class V > @@ -222,7 +251,7 @@ class PersistentMap : public boost::noncopyable template< class V > bool _insert( const std::string& key, const std::vector< V >& values, const boost::true_type& ) - { return _insert( key, values.data(), values.size() * sizeof( V )); } + { return insert( key, values.data(), values.size() * sizeof( V )); } template< class V > V _get( const std::string& k ) const { @@ -249,7 +278,7 @@ template<> inline bool PersistentMap::_insert( const std::string& k, const std::string& v, const boost::false_type& ) { - return _insert( k, v.data(), v.length( )); + return insert( k, v.data(), v.length( )); } template< class V > inline diff --git a/lunchbox/skv/persistentMap.h b/lunchbox/skv/persistentMap.h index e3dda679..978f4279 100644 --- a/lunchbox/skv/persistentMap.h +++ b/lunchbox/skv/persistentMap.h @@ -1,5 +1,5 @@ -/* Copyright (c) 2014-2015, Stefan.Eilemann@epfl.ch +/* Copyright (c) 2014-2016, Stefan.Eilemann@epfl.ch * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License version 2.1 as published @@ -151,12 +151,6 @@ class PersistentMap : public detail::PersistentMap return false; } - bool contains( const std::string& key ) const final - { - std::string value; - return _retrieve( key, value ) == SKV_SUCCESS; - } - bool flush() final { return _flush( 0 ); } private: diff --git a/lunchbox/types.h b/lunchbox/types.h index bca24af0..b9fd01e7 100644 --- a/lunchbox/types.h +++ b/lunchbox/types.h @@ -219,6 +219,7 @@ typedef Strings::iterator StringsIter; class Clock; class DSO; class Lock; +class PersistentMap; class Referenced; class RequestHandler; class SpinLock; @@ -244,7 +245,7 @@ typedef Future< uint32_t > f_uint32_t; //!< A future 32 bit unsigned promise typedef Future< ssize_t > f_ssize_t; //!< A future signed size promise typedef Future< void > f_void_t; //!< A future signed size promise -typedef std::vector< DSO* > DSOs; /** A vector of DSO @version 1.11.0 */ +typedef std::vector< DSO* > DSOs; //!< A vector of DSO @version 1.11.0 #ifdef LUNCHBOX_USE_V1_API using servus::Servus; diff --git a/tests/persistentMap.cpp b/tests/persistentMap.cpp index d01dc679..2153f9d8 100644 --- a/tests/persistentMap.cpp +++ b/tests/persistentMap.cpp @@ -1,5 +1,5 @@ -/* Copyright (c) 2014-2015, Stefan.Eilemann@epfl.ch +/* Copyright (c) 2014-2016, Stefan.Eilemann@epfl.ch * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License version 2.1 as published @@ -100,11 +100,26 @@ void read( const std::string& uri ) read( map ); } +bool testAvailable( const std::string& uri ) +{ + try + { + PersistentMap map( uri ); + if( !map.insert( "foo", "bar" )) + return false; + return map[ "foo" ] == "bar"; + } + catch( ... ) + { + return false; + } + return true; +} + void setup( const std::string& uri ) { PersistentMap map( uri ); TEST( map.insert( "foo", "bar" )); - TEST( map.contains( "foo" )); TESTINFO( map[ "foo" ] == "bar", map[ "foo" ] << " length " << map[ "foo" ].length( )); TEST( map[ "bar" ].empty( )); @@ -286,6 +301,16 @@ int main( int, char* argv[] ) for( size_t i=1; i <= 65536; i = i<<2 ) benchmark( "leveldb://", 0, i ); #endif +#ifdef LUNCHBOX_USE_LIBMEMCACHED + if( testAvailable( "memcached://" )) + { + setup( "memcached://" ); + read( "memcached://" ); + if( perfTest ) + for( size_t i=1; i <= 65536; i = i<<2 ) + benchmark( "memcached://", 0, i ); + } +#endif #ifdef LUNCHBOX_USE_SKV FxLogger_Init( argv[0] ); setup( "skv://" );