diff --git a/include/caffe/test/test_caffe_main.hpp b/include/caffe/test/test_caffe_main.hpp
index bd5f31e063f..fc156091476 100644
--- a/include/caffe/test/test_caffe_main.hpp
+++ b/include/caffe/test/test_caffe_main.hpp
@@ -40,34 +40,36 @@ class MultiDeviceTest : public ::testing::Test {
 
 typedef ::testing::Types<float, double> TestDtypes;
 
-struct FloatCPU {
-  typedef float Dtype;
+template <typename TypeParam>
+struct CPUDevice {
+  typedef TypeParam Dtype;
   static const Caffe::Brew device = Caffe::CPU;
 };
 
-struct DoubleCPU {
-  typedef double Dtype;
-  static const Caffe::Brew device = Caffe::CPU;
+template <typename Dtype>
+class CPUDeviceTest : public MultiDeviceTest<CPUDevice<Dtype> > {
 };
 
 #ifdef CPU_ONLY
 
-typedef ::testing::Types<FloatCPU, DoubleCPU> TestDtypesAndDevices;
+typedef ::testing::Types<CPUDevice<float>,
+                         CPUDevice<double> > TestDtypesAndDevices;
 
 #else
 
-struct FloatGPU {
-  typedef float Dtype;
+template <typename TypeParam>
+struct GPUDevice {
+  typedef TypeParam Dtype;
   static const Caffe::Brew device = Caffe::GPU;
 };
 
-struct DoubleGPU {
-  typedef double Dtype;
-  static const Caffe::Brew device = Caffe::GPU;
+template <typename Dtype>
+class GPUDeviceTest : public MultiDeviceTest<GPUDevice<Dtype> > {
 };
 
-typedef ::testing::Types<FloatCPU, DoubleCPU, FloatGPU, DoubleGPU>
-    TestDtypesAndDevices;
+typedef ::testing::Types<CPUDevice<float>, CPUDevice<double>,
+                         GPUDevice<float>, GPUDevice<double> >
+                         TestDtypesAndDevices;
 
 #endif
 
diff --git a/src/caffe/test/test_accuracy_layer.cpp b/src/caffe/test/test_accuracy_layer.cpp
index 6cbf51df45e..c14b67cc0e9 100644
--- a/src/caffe/test/test_accuracy_layer.cpp
+++ b/src/caffe/test/test_accuracy_layer.cpp
@@ -16,7 +16,7 @@
 namespace caffe {
 
 template <typename Dtype>
-class AccuracyLayerTest : public ::testing::Test {
+class AccuracyLayerTest : public CPUDeviceTest<Dtype> {
  protected:
   AccuracyLayerTest()
       : blob_bottom_data_(new Blob<Dtype>()),
@@ -92,7 +92,6 @@ TYPED_TEST(AccuracyLayerTest, TestSetupTopK) {
 
 TYPED_TEST(AccuracyLayerTest, TestForwardCPU) {
   LayerParameter layer_param;
-  Caffe::set_mode(Caffe::CPU);
   AccuracyLayer<TypeParam> layer(layer_param);
   layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
   layer.Forward(this->blob_bottom_vec_, this->blob_top_vec_);
@@ -118,7 +117,6 @@ TYPED_TEST(AccuracyLayerTest, TestForwardCPU) {
 }
 
 TYPED_TEST(AccuracyLayerTest, TestForwardWithSpatialAxes) {
-  Caffe::set_mode(Caffe::CPU);
   this->blob_bottom_data_->Reshape(2, 10, 4, 5);
   vector<int> label_shape(3);
   label_shape[0] = 2; label_shape[1] = 4; label_shape[2] = 5;
@@ -162,7 +160,6 @@ TYPED_TEST(AccuracyLayerTest, TestForwardWithSpatialAxes) {
 }
 
 TYPED_TEST(AccuracyLayerTest, TestForwardIgnoreLabel) {
-  Caffe::set_mode(Caffe::CPU);
   LayerParameter layer_param;
   const TypeParam kIgnoreLabelValue = -1;
   layer_param.mutable_accuracy_param()->set_ignore_label(kIgnoreLabelValue);
diff --git a/src/caffe/test/test_argmax_layer.cpp b/src/caffe/test/test_argmax_layer.cpp
index 3487d42f21e..895c3d372ff 100644
--- a/src/caffe/test/test_argmax_layer.cpp
+++ b/src/caffe/test/test_argmax_layer.cpp
@@ -13,13 +13,12 @@
 namespace caffe {
 
 template <typename Dtype>
-class ArgMaxLayerTest : public ::testing::Test {
+class ArgMaxLayerTest : public CPUDeviceTest<Dtype> {
  protected:
   ArgMaxLayerTest()
       : blob_bottom_(new Blob<Dtype>(10, 20, 1, 1)),
         blob_top_(new Blob<Dtype>()),
         top_k_(5) {
-    Caffe::set_mode(Caffe::CPU);
     Caffe::set_random_seed(1701);
     // fill the values
     FillerParameter filler_param;
diff --git a/src/caffe/test/test_convolution_layer.cpp b/src/caffe/test/test_convolution_layer.cpp
index c1fe3b58c58..67d41fff844 100644
--- a/src/caffe/test/test_convolution_layer.cpp
+++ b/src/caffe/test/test_convolution_layer.cpp
@@ -424,7 +424,7 @@ TYPED_TEST(ConvolutionLayerTest, TestGradientGroup) {
 #ifdef USE_CUDNN
 
 template <typename Dtype>
-class CuDNNConvolutionLayerTest : public ::testing::Test {
+class CuDNNConvolutionLayerTest : public GPUDeviceTest<Dtype> {
  protected:
   CuDNNConvolutionLayerTest()
       : blob_bottom_(new Blob<Dtype>(2, 3, 6, 4)),
@@ -467,7 +467,6 @@ class CuDNNConvolutionLayerTest : public ::testing::Test {
 TYPED_TEST_CASE(CuDNNConvolutionLayerTest, TestDtypes);
 
 TYPED_TEST(CuDNNConvolutionLayerTest, TestSetupCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   this->blob_bottom_vec_.push_back(this->blob_bottom_2_);
   this->blob_top_vec_.push_back(this->blob_top_2_);
   LayerParameter layer_param;
@@ -505,7 +504,6 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestSetupCuDNN) {
 }
 
 TYPED_TEST(CuDNNConvolutionLayerTest, TestSimpleConvolutionCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   this->blob_bottom_vec_.push_back(this->blob_bottom_2_);
   this->blob_top_vec_.push_back(this->blob_top_2_);
   LayerParameter layer_param;
@@ -541,7 +539,6 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestSimpleConvolutionCuDNN) {
 }
 
 TYPED_TEST(CuDNNConvolutionLayerTest, TestSimpleConvolutionGroupCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   ConvolutionParameter* convolution_param =
       layer_param.mutable_convolution_param();
@@ -572,7 +569,7 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestSobelConvolutionCuDNN) {
   // Test separable convolution by computing the Sobel operator
   // as a single filter then comparing the result
   // as the convolution of two rectangular filters.
-  Caffe::set_mode(Caffe::GPU);
+
   // Fill bottoms with identical Gaussian noise.
   shared_ptr<GaussianFiller<TypeParam> > filler;
   FillerParameter filler_param;
@@ -665,7 +662,6 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestSobelConvolutionCuDNN) {
 }
 
 TYPED_TEST(CuDNNConvolutionLayerTest, TestGradientCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   ConvolutionParameter* convolution_param =
       layer_param.mutable_convolution_param();
@@ -683,7 +679,6 @@ TYPED_TEST(CuDNNConvolutionLayerTest, TestGradientCuDNN) {
 }
 
 TYPED_TEST(CuDNNConvolutionLayerTest, TestGradientGroupCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   ConvolutionParameter* convolution_param =
       layer_param.mutable_convolution_param();
diff --git a/src/caffe/test/test_dummy_data_layer.cpp b/src/caffe/test/test_dummy_data_layer.cpp
index 99548352746..c9ed38db3a5 100644
--- a/src/caffe/test/test_dummy_data_layer.cpp
+++ b/src/caffe/test/test_dummy_data_layer.cpp
@@ -13,7 +13,7 @@
 namespace caffe {
 
 template <typename Dtype>
-class DummyDataLayerTest : public ::testing::Test {
+class DummyDataLayerTest : public CPUDeviceTest<Dtype> {
  protected:
   DummyDataLayerTest()
       : blob_top_a_(new Blob<Dtype>()),
@@ -44,7 +44,6 @@ class DummyDataLayerTest : public ::testing::Test {
 TYPED_TEST_CASE(DummyDataLayerTest, TestDtypes);
 
 TYPED_TEST(DummyDataLayerTest, TestOneTopConstant) {
-  Caffe::set_mode(Caffe::CPU);
   LayerParameter param;
   DummyDataParameter* dummy_data_param = param.mutable_dummy_data_param();
   dummy_data_param->add_num(5);
@@ -74,7 +73,6 @@ TYPED_TEST(DummyDataLayerTest, TestOneTopConstant) {
 }
 
 TYPED_TEST(DummyDataLayerTest, TestTwoTopConstant) {
-  Caffe::set_mode(Caffe::CPU);
   LayerParameter param;
   DummyDataParameter* dummy_data_param = param.mutable_dummy_data_param();
   dummy_data_param->add_num(5);
@@ -113,7 +111,6 @@ TYPED_TEST(DummyDataLayerTest, TestTwoTopConstant) {
 }
 
 TYPED_TEST(DummyDataLayerTest, TestThreeTopConstantGaussianConstant) {
-  Caffe::set_mode(Caffe::CPU);
   LayerParameter param;
   DummyDataParameter* dummy_data_param = param.mutable_dummy_data_param();
   dummy_data_param->add_num(5);
diff --git a/src/caffe/test/test_im2col_kernel.cu b/src/caffe/test/test_im2col_kernel.cu
index ee684c00255..0017ac23e69 100644
--- a/src/caffe/test/test_im2col_kernel.cu
+++ b/src/caffe/test/test_im2col_kernel.cu
@@ -25,7 +25,7 @@ __global__ void im2col_gpu_kernel(const int n, const Dtype* data_im,
 extern cudaDeviceProp CAFFE_TEST_CUDA_PROP;
 
 template <typename Dtype>
-class Im2colKernelTest : public ::testing::Test {
+class Im2colKernelTest : public GPUDeviceTest<Dtype> {
  protected:
   Im2colKernelTest()
         // big so launches > 1024 threads
@@ -68,8 +68,6 @@ class Im2colKernelTest : public ::testing::Test {
 TYPED_TEST_CASE(Im2colKernelTest, TestDtypes);
 
 TYPED_TEST(Im2colKernelTest, TestGPU) {
-  Caffe::set_mode(Caffe::GPU);
-
   // Reshape the blobs to correct size for im2col output
   this->blob_top_->Reshape(this->blob_bottom_->num(),
           this->channels_ * this->kernel_size_ * this->kernel_size_,
diff --git a/src/caffe/test/test_math_functions.cpp b/src/caffe/test/test_math_functions.cpp
index 667f744bdd7..a095b544e17 100644
--- a/src/caffe/test/test_math_functions.cpp
+++ b/src/caffe/test/test_math_functions.cpp
@@ -15,8 +15,10 @@
 
 namespace caffe {
 
-template<typename Dtype>
-class MathFunctionsTest : public ::testing::Test {
+template <typename TypeParam>
+class MathFunctionsTest : public MultiDeviceTest<TypeParam> {
+  typedef typename TypeParam::Dtype Dtype;
+
  protected:
   MathFunctionsTest()
       : blob_bottom_(new Blob<Dtype>()),
@@ -64,14 +66,19 @@ class MathFunctionsTest : public ::testing::Test {
   Blob<Dtype>* const blob_top_;
 };
 
-TYPED_TEST_CASE(MathFunctionsTest, TestDtypes);
+template <typename Dtype>
+class CPUMathFunctionsTest
+  : public MathFunctionsTest<CPUDevice<Dtype> > {
+};
+
+TYPED_TEST_CASE(CPUMathFunctionsTest, TestDtypes);
 
-TYPED_TEST(MathFunctionsTest, TestNothing) {
+TYPED_TEST(CPUMathFunctionsTest, TestNothing) {
   // The first test case of a test suite takes the longest time
   //   due to the set up overhead.
 }
 
-TYPED_TEST(MathFunctionsTest, TestHammingDistanceCPU) {
+TYPED_TEST(CPUMathFunctionsTest, TestHammingDistance) {
   int n = this->blob_bottom_->count();
   const TypeParam* x = this->blob_bottom_->cpu_data();
   const TypeParam* y = this->blob_top_->cpu_data();
@@ -79,7 +86,7 @@ TYPED_TEST(MathFunctionsTest, TestHammingDistanceCPU) {
             caffe_cpu_hamming_distance<TypeParam>(n, x, y));
 }
 
-TYPED_TEST(MathFunctionsTest, TestAsumCPU) {
+TYPED_TEST(CPUMathFunctionsTest, TestAsum) {
   int n = this->blob_bottom_->count();
   const TypeParam* x = this->blob_bottom_->cpu_data();
   TypeParam std_asum = 0;
@@ -90,7 +97,7 @@ TYPED_TEST(MathFunctionsTest, TestAsumCPU) {
   EXPECT_LT((cpu_asum - std_asum) / std_asum, 1e-2);
 }
 
-TYPED_TEST(MathFunctionsTest, TestSignCPU) {
+TYPED_TEST(CPUMathFunctionsTest, TestSign) {
   int n = this->blob_bottom_->count();
   const TypeParam* x = this->blob_bottom_->cpu_data();
   caffe_cpu_sign<TypeParam>(n, x, this->blob_bottom_->mutable_cpu_diff());
@@ -100,7 +107,7 @@ TYPED_TEST(MathFunctionsTest, TestSignCPU) {
   }
 }
 
-TYPED_TEST(MathFunctionsTest, TestSgnbitCPU) {
+TYPED_TEST(CPUMathFunctionsTest, TestSgnbit) {
   int n = this->blob_bottom_->count();
   const TypeParam* x = this->blob_bottom_->cpu_data();
   caffe_cpu_sgnbit<TypeParam>(n, x, this->blob_bottom_->mutable_cpu_diff());
@@ -110,7 +117,7 @@ TYPED_TEST(MathFunctionsTest, TestSgnbitCPU) {
   }
 }
 
-TYPED_TEST(MathFunctionsTest, TestFabsCPU) {
+TYPED_TEST(CPUMathFunctionsTest, TestFabs) {
   int n = this->blob_bottom_->count();
   const TypeParam* x = this->blob_bottom_->cpu_data();
   caffe_abs<TypeParam>(n, x, this->blob_bottom_->mutable_cpu_diff());
@@ -120,7 +127,7 @@ TYPED_TEST(MathFunctionsTest, TestFabsCPU) {
   }
 }
 
-TYPED_TEST(MathFunctionsTest, TestScaleCPU) {
+TYPED_TEST(CPUMathFunctionsTest, TestScale) {
   int n = this->blob_bottom_->count();
   TypeParam alpha = this->blob_bottom_->cpu_diff()[caffe_rng_rand() %
                                                    this->blob_bottom_->count()];
@@ -133,11 +140,10 @@ TYPED_TEST(MathFunctionsTest, TestScaleCPU) {
   }
 }
 
-TYPED_TEST(MathFunctionsTest, TestCopyCPU) {
+TYPED_TEST(CPUMathFunctionsTest, TestCopy) {
   const int n = this->blob_bottom_->count();
   const TypeParam* bottom_data = this->blob_bottom_->cpu_data();
   TypeParam* top_data = this->blob_top_->mutable_cpu_data();
-  Caffe::set_mode(Caffe::CPU);
   caffe_copy(n, bottom_data, top_data);
   for (int i = 0; i < n; ++i) {
     EXPECT_EQ(bottom_data[i], top_data[i]);
@@ -146,8 +152,14 @@ TYPED_TEST(MathFunctionsTest, TestCopyCPU) {
 
 #ifndef CPU_ONLY
 
+template <typename Dtype>
+class GPUMathFunctionsTest : public MathFunctionsTest<GPUDevice<Dtype> > {
+};
+
+TYPED_TEST_CASE(GPUMathFunctionsTest, TestDtypes);
+
 // TODO: Fix caffe_gpu_hamming_distance and re-enable this test.
-TYPED_TEST(MathFunctionsTest, DISABLED_TestHammingDistanceGPU) {
+TYPED_TEST(GPUMathFunctionsTest, DISABLED_TestHammingDistance) {
   int n = this->blob_bottom_->count();
   const TypeParam* x = this->blob_bottom_->cpu_data();
   const TypeParam* y = this->blob_top_->cpu_data();
@@ -158,7 +170,7 @@ TYPED_TEST(MathFunctionsTest, DISABLED_TestHammingDistanceGPU) {
   EXPECT_EQ(reference_distance, computed_distance);
 }
 
-TYPED_TEST(MathFunctionsTest, TestAsumGPU) {
+TYPED_TEST(GPUMathFunctionsTest, TestAsum) {
   int n = this->blob_bottom_->count();
   const TypeParam* x = this->blob_bottom_->cpu_data();
   TypeParam std_asum = 0;
@@ -170,7 +182,7 @@ TYPED_TEST(MathFunctionsTest, TestAsumGPU) {
   EXPECT_LT((gpu_asum - std_asum) / std_asum, 1e-2);
 }
 
-TYPED_TEST(MathFunctionsTest, TestSignGPU) {
+TYPED_TEST(GPUMathFunctionsTest, TestSign) {
   int n = this->blob_bottom_->count();
   caffe_gpu_sign<TypeParam>(n, this->blob_bottom_->gpu_data(),
                             this->blob_bottom_->mutable_gpu_diff());
@@ -181,7 +193,7 @@ TYPED_TEST(MathFunctionsTest, TestSignGPU) {
   }
 }
 
-TYPED_TEST(MathFunctionsTest, TestSgnbitGPU) {
+TYPED_TEST(GPUMathFunctionsTest, TestSgnbit) {
   int n = this->blob_bottom_->count();
   caffe_gpu_sgnbit<TypeParam>(n, this->blob_bottom_->gpu_data(),
                             this->blob_bottom_->mutable_gpu_diff());
@@ -192,7 +204,7 @@ TYPED_TEST(MathFunctionsTest, TestSgnbitGPU) {
   }
 }
 
-TYPED_TEST(MathFunctionsTest, TestFabsGPU) {
+TYPED_TEST(GPUMathFunctionsTest, TestFabs) {
   int n = this->blob_bottom_->count();
   caffe_gpu_abs<TypeParam>(n, this->blob_bottom_->gpu_data(),
                             this->blob_bottom_->mutable_gpu_diff());
@@ -203,7 +215,7 @@ TYPED_TEST(MathFunctionsTest, TestFabsGPU) {
   }
 }
 
-TYPED_TEST(MathFunctionsTest, TestScaleGPU) {
+TYPED_TEST(GPUMathFunctionsTest, TestScale) {
   int n = this->blob_bottom_->count();
   TypeParam alpha = this->blob_bottom_->cpu_diff()[caffe_rng_rand() %
                                                    this->blob_bottom_->count()];
@@ -216,11 +228,10 @@ TYPED_TEST(MathFunctionsTest, TestScaleGPU) {
   }
 }
 
-TYPED_TEST(MathFunctionsTest, TestCopyGPU) {
+TYPED_TEST(GPUMathFunctionsTest, TestCopy) {
   const int n = this->blob_bottom_->count();
   const TypeParam* bottom_data = this->blob_bottom_->gpu_data();
   TypeParam* top_data = this->blob_top_->mutable_gpu_data();
-  Caffe::set_mode(Caffe::GPU);
   caffe_copy(n, bottom_data, top_data);
   bottom_data = this->blob_bottom_->cpu_data();
   top_data = this->blob_top_->mutable_cpu_data();
diff --git a/src/caffe/test/test_multinomial_logistic_loss_layer.cpp b/src/caffe/test/test_multinomial_logistic_loss_layer.cpp
index 9038017e3e2..b2db984feb1 100644
--- a/src/caffe/test/test_multinomial_logistic_loss_layer.cpp
+++ b/src/caffe/test/test_multinomial_logistic_loss_layer.cpp
@@ -16,7 +16,7 @@
 namespace caffe {
 
 template <typename Dtype>
-class MultinomialLogisticLossLayerTest : public ::testing::Test {
+class MultinomialLogisticLossLayerTest : public CPUDeviceTest<Dtype> {
  protected:
   MultinomialLogisticLossLayerTest()
       : blob_bottom_data_(new Blob<Dtype>(10, 5, 1, 1)),
@@ -51,7 +51,6 @@ TYPED_TEST_CASE(MultinomialLogisticLossLayerTest, TestDtypes);
 
 TYPED_TEST(MultinomialLogisticLossLayerTest, TestGradientCPU) {
   LayerParameter layer_param;
-  Caffe::set_mode(Caffe::CPU);
   MultinomialLogisticLossLayer<TypeParam> layer(layer_param);
   layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
   GradientChecker<TypeParam> checker(1e-2, 2*1e-2, 1701, 0, 0.05);
diff --git a/src/caffe/test/test_neuron_layer.cpp b/src/caffe/test/test_neuron_layer.cpp
index 030f4bbae7f..37b54713b46 100644
--- a/src/caffe/test/test_neuron_layer.cpp
+++ b/src/caffe/test/test_neuron_layer.cpp
@@ -586,7 +586,7 @@ TYPED_TEST(NeuronLayerTest, TestPReLUInPlace) {
 
 #ifdef USE_CUDNN
 template <typename Dtype>
-class CuDNNNeuronLayerTest : public ::testing::Test {
+class CuDNNNeuronLayerTest : public GPUDeviceTest<Dtype> {
  protected:
   CuDNNNeuronLayerTest()
       : blob_bottom_(new Blob<Dtype>(2, 3, 4, 5)),
@@ -609,7 +609,6 @@ class CuDNNNeuronLayerTest : public ::testing::Test {
 TYPED_TEST_CASE(CuDNNNeuronLayerTest, TestDtypes);
 
 TYPED_TEST(CuDNNNeuronLayerTest, TestReLUCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CuDNNReLULayer<TypeParam> layer(layer_param);
   layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
@@ -624,7 +623,6 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestReLUCuDNN) {
 }
 
 TYPED_TEST(CuDNNNeuronLayerTest, TestReLUGradientCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CuDNNReLULayer<TypeParam> layer(layer_param);
   GradientChecker<TypeParam> checker(1e-2, 1e-3, 1701, 0., 0.01);
@@ -633,7 +631,6 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestReLUGradientCuDNN) {
 }
 
 TYPED_TEST(CuDNNNeuronLayerTest, TestReLUWithNegativeSlopeCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CHECK(google::protobuf::TextFormat::ParseFromString(
       "relu_param { negative_slope: 0.01 }", &layer_param));
@@ -653,7 +650,6 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestReLUWithNegativeSlopeCuDNN) {
 }
 
 TYPED_TEST(CuDNNNeuronLayerTest, TestReLUGradientWithNegativeSlopeCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CHECK(google::protobuf::TextFormat::ParseFromString(
       "relu_param { negative_slope: 0.01 }", &layer_param));
@@ -664,7 +660,6 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestReLUGradientWithNegativeSlopeCuDNN) {
 }
 
 TYPED_TEST(CuDNNNeuronLayerTest, TestSigmoidCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CuDNNSigmoidLayer<TypeParam> layer(layer_param);
   layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
@@ -681,7 +676,6 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestSigmoidCuDNN) {
 }
 
 TYPED_TEST(CuDNNNeuronLayerTest, TestSigmoidGradientCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CuDNNSigmoidLayer<TypeParam> layer(layer_param);
   GradientChecker<TypeParam> checker(1e-2, 1e-3, 1701, 0., 0.01);
@@ -690,7 +684,6 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestSigmoidGradientCuDNN) {
 }
 
 TYPED_TEST(CuDNNNeuronLayerTest, TestTanHCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CuDNNTanHLayer<TypeParam> layer(layer_param);
   layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
@@ -713,7 +706,6 @@ TYPED_TEST(CuDNNNeuronLayerTest, TestTanHCuDNN) {
 }
 
 TYPED_TEST(CuDNNNeuronLayerTest, TestTanHGradientCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CuDNNTanHLayer<TypeParam> layer(layer_param);
   GradientChecker<TypeParam> checker(1e-2, 1e-3);
diff --git a/src/caffe/test/test_pooling_layer.cpp b/src/caffe/test/test_pooling_layer.cpp
index e9964e7f0b7..69f2d5c1135 100644
--- a/src/caffe/test/test_pooling_layer.cpp
+++ b/src/caffe/test/test_pooling_layer.cpp
@@ -608,7 +608,7 @@ TYPED_TEST(PoolingLayerTest, TestGradientAvePadded) {
 
 #ifdef USE_CUDNN
 template <typename Dtype>
-class CuDNNPoolingLayerTest : public ::testing::Test {
+class CuDNNPoolingLayerTest : public GPUDeviceTest<Dtype> {
  protected:
   CuDNNPoolingLayerTest()
       : blob_bottom_(new Blob<Dtype>()),
@@ -963,7 +963,6 @@ class CuDNNPoolingLayerTest : public ::testing::Test {
 TYPED_TEST_CASE(CuDNNPoolingLayerTest, TestDtypes);
 
 TYPED_TEST(CuDNNPoolingLayerTest, TestSetupCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   PoolingParameter* pooling_param = layer_param.mutable_pooling_param();
   pooling_param->set_kernel_size(3);
@@ -977,7 +976,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestSetupCuDNN) {
 }
 
 TYPED_TEST(CuDNNPoolingLayerTest, TestSetupPaddedCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   PoolingParameter* pooling_param = layer_param.mutable_pooling_param();
   pooling_param->set_kernel_size(3);
@@ -994,7 +992,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestSetupPaddedCuDNN) {
 
 /*
 TYPED_TEST(CuDNNPoolingLayerTest, PrintBackwardCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   layer_param.set_kernelsize(3);
   layer_param.set_stride(2);
@@ -1020,7 +1017,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, PrintBackwardCuDNN) {
 */
 
 TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   this->TestForwardSquare();
   this->TestForwardRectHigh();
   this->TestForwardRectWide();
@@ -1030,7 +1026,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxCuDNN) {
 // the corresponding backward test.
 /*
 TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxTopMaskCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   this->blob_top_vec_.push_back(this->blob_top_mask_);
   this->TestForwardSquare();
   this->TestForwardRectHigh();
@@ -1039,7 +1034,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxTopMaskCuDNN) {
 */
 
 TYPED_TEST(CuDNNPoolingLayerTest, TestGradientMaxCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   for (int kernel_h = 3; kernel_h <= 4; kernel_h++) {
     for (int kernel_w = 3; kernel_w <= 4; kernel_w++) {
       LayerParameter layer_param;
@@ -1059,7 +1053,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestGradientMaxCuDNN) {
 }
 
 TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxPaddedCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   PoolingParameter* pooling_param = layer_param.mutable_pooling_param();
   pooling_param->set_kernel_size(3);
@@ -1105,7 +1098,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardMaxPaddedCuDNN) {
 
 /*
 TYPED_TEST(CuDNNPoolingLayerTest, TestGradientMaxTopMaskCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   for (int kernel_h = 3; kernel_h <= 4; kernel_h++) {
     for (int kernel_w = 3; kernel_w <= 4; kernel_w++) {
       LayerParameter layer_param;
@@ -1126,7 +1118,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestGradientMaxTopMaskCuDNN) {
 */
 
 TYPED_TEST(CuDNNPoolingLayerTest, TestForwardAveCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   PoolingParameter* pooling_param = layer_param.mutable_pooling_param();
   pooling_param->set_kernel_size(3);
@@ -1152,7 +1143,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestForwardAveCuDNN) {
 }
 
 TYPED_TEST(CuDNNPoolingLayerTest, TestGradientAveCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   for (int kernel_h = 3; kernel_h <= 4; kernel_h++) {
     for (int kernel_w = 3; kernel_w <= 4; kernel_w++) {
       LayerParameter layer_param;
@@ -1170,7 +1160,6 @@ TYPED_TEST(CuDNNPoolingLayerTest, TestGradientAveCuDNN) {
 }
 
 TYPED_TEST(CuDNNPoolingLayerTest, TestGradientAvePaddedCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   for (int kernel_h = 3; kernel_h <= 4; kernel_h++) {
     for (int kernel_w = 3; kernel_w <= 4; kernel_w++) {
       LayerParameter layer_param;
diff --git a/src/caffe/test/test_softmax_layer.cpp b/src/caffe/test/test_softmax_layer.cpp
index f6674422e56..996da4b8f7c 100644
--- a/src/caffe/test/test_softmax_layer.cpp
+++ b/src/caffe/test/test_softmax_layer.cpp
@@ -82,7 +82,7 @@ TYPED_TEST(SoftmaxLayerTest, TestGradient) {
 
 #ifdef USE_CUDNN
 template <typename Dtype>
-class CuDNNSoftmaxLayerTest : public ::testing::Test {
+class CuDNNSoftmaxLayerTest : public GPUDeviceTest<Dtype> {
  protected:
   CuDNNSoftmaxLayerTest()
       : blob_bottom_(new Blob<Dtype>(2, 10, 2, 3)),
@@ -104,7 +104,6 @@ class CuDNNSoftmaxLayerTest : public ::testing::Test {
 TYPED_TEST_CASE(CuDNNSoftmaxLayerTest, TestDtypes);
 
 TYPED_TEST(CuDNNSoftmaxLayerTest, TestForwardCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CuDNNSoftmaxLayer<TypeParam> layer(layer_param);
   layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_);
@@ -138,7 +137,6 @@ TYPED_TEST(CuDNNSoftmaxLayerTest, TestForwardCuDNN) {
 }
 
 TYPED_TEST(CuDNNSoftmaxLayerTest, TestGradientCuDNN) {
-  Caffe::set_mode(Caffe::GPU);
   LayerParameter layer_param;
   CuDNNSoftmaxLayer<TypeParam> layer(layer_param);
   GradientChecker<TypeParam> checker(1e-2, 1e-3);
diff --git a/src/caffe/test/test_stochastic_pooling.cpp b/src/caffe/test/test_stochastic_pooling.cpp
index 12962c65d85..f84464c322c 100644
--- a/src/caffe/test/test_stochastic_pooling.cpp
+++ b/src/caffe/test/test_stochastic_pooling.cpp
@@ -16,8 +16,10 @@ using std::min;
 
 namespace caffe {
 
-template <typename Dtype>
-class StochasticPoolingLayerTest : public ::testing::Test {
+template <typename TypeParam>
+class StochasticPoolingLayerTest : public MultiDeviceTest<TypeParam> {
+  typedef typename TypeParam::Dtype Dtype;
+
  protected:
   StochasticPoolingLayerTest()
       : blob_bottom_(new Blob<Dtype>()),
@@ -45,9 +47,14 @@ class StochasticPoolingLayerTest : public ::testing::Test {
   vector<Blob<Dtype>*> blob_top_vec_;
 };
 
-TYPED_TEST_CASE(StochasticPoolingLayerTest, TestDtypes);
+template <typename Dtype>
+class CPUStochasticPoolingLayerTest
+  : public StochasticPoolingLayerTest<CPUDevice<Dtype> > {
+};
+
+TYPED_TEST_CASE(CPUStochasticPoolingLayerTest, TestDtypes);
 
-TYPED_TEST(StochasticPoolingLayerTest, TestSetup) {
+TYPED_TEST(CPUStochasticPoolingLayerTest, TestSetup) {
   LayerParameter layer_param;
   PoolingParameter* pooling_param = layer_param.mutable_pooling_param();
   pooling_param->set_kernel_size(3);
@@ -60,8 +67,16 @@ TYPED_TEST(StochasticPoolingLayerTest, TestSetup) {
   EXPECT_EQ(this->blob_top_->width(), 2);
 }
 
-TYPED_TEST(StochasticPoolingLayerTest, TestStochasticGPU) {
-  Caffe::set_mode(Caffe::GPU);
+#ifndef CPU_ONLY
+
+template <typename Dtype>
+class GPUStochasticPoolingLayerTest
+  : public StochasticPoolingLayerTest<GPUDevice<Dtype> > {
+};
+
+TYPED_TEST_CASE(GPUStochasticPoolingLayerTest, TestDtypes);
+
+TYPED_TEST(GPUStochasticPoolingLayerTest, TestStochastic) {
   LayerParameter layer_param;
   layer_param.set_phase(TRAIN);
   PoolingParameter* pooling_param = layer_param.mutable_pooling_param();
@@ -104,8 +119,7 @@ TYPED_TEST(StochasticPoolingLayerTest, TestStochasticGPU) {
   EXPECT_GE(total / this->blob_top_->count(), 0.55);
 }
 
-TYPED_TEST(StochasticPoolingLayerTest, TestStochasticGPUTestPhase) {
-  Caffe::set_mode(Caffe::GPU);
+TYPED_TEST(GPUStochasticPoolingLayerTest, TestStochasticTestPhase) {
   LayerParameter layer_param;
   layer_param.set_phase(TEST);
   PoolingParameter* pooling_param = layer_param.mutable_pooling_param();
@@ -142,8 +156,7 @@ TYPED_TEST(StochasticPoolingLayerTest, TestStochasticGPUTestPhase) {
   }
 }
 
-TYPED_TEST(StochasticPoolingLayerTest, TestGradientGPU) {
-  Caffe::set_mode(Caffe::GPU);
+TYPED_TEST(GPUStochasticPoolingLayerTest, TestGradient) {
   LayerParameter layer_param;
   layer_param.set_phase(TRAIN);
   PoolingParameter* pooling_param = layer_param.mutable_pooling_param();
@@ -158,6 +171,6 @@ TYPED_TEST(StochasticPoolingLayerTest, TestGradientGPU) {
       this->blob_top_vec_);
 }
 
-
+#endif
 
 }  // namespace caffe