From 45cb97380ffbe8f71a5f769615aa4e51cfcf7275 Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 20:20:57 -0800 Subject: [PATCH 01/12] Update README.md --- README.md | 46 ++++++++++++++-------------------------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 44b9e62c157..4359272c065 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,19 @@ -# Caffe +# Deep Metric Learning via Lifted Structured Feature Embedding -[![Build Status](https://travis-ci.org/BVLC/caffe.svg?branch=master)](https://travis-ci.org/BVLC/caffe) -[![License](https://img.shields.io/badge/license-BSD-blue.svg)](LICENSE) +## Citing this work +If you find this work useful in your research, please consider citing: -Caffe is a deep learning framework made with expression, speed, and modularity in mind. -It is developed by the Berkeley Vision and Learning Center ([BVLC](http://bvlc.eecs.berkeley.edu)) and community contributors. - -Check out the [project site](http://caffe.berkeleyvision.org) for all the details like - -- [DIY Deep Learning for Vision with Caffe](https://docs.google.com/presentation/d/1UeKXVgRvvxg9OUdh_UiC5G71UMscNPlvArsWER41PsU/edit#slide=id.p) -- [Tutorial Documentation](http://caffe.berkeleyvision.org/tutorial/) -- [BVLC reference models](http://caffe.berkeleyvision.org/model_zoo.html) and the [community model zoo](https://github.com/BVLC/caffe/wiki/Model-Zoo) -- [Installation instructions](http://caffe.berkeleyvision.org/installation.html) - -and step-by-step examples. - -[![Join the chat at https://gitter.im/BVLC/caffe](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/BVLC/caffe?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) - -Please join the [caffe-users group](https://groups.google.com/forum/#!forum/caffe-users) or [gitter chat](https://gitter.im/BVLC/caffe) to ask questions and talk about methods and models. -Framework development discussions and thorough bug reports are collected on [Issues](https://github.com/BVLC/caffe/issues). - -Happy brewing! - -## License and Citation + @inproceedings{songCVPR16, + Author = {Hyun Oh Song and Yu Xiang and Stefanie Jegelka and Silvio Savarese}, + Title = {Deep Metric Learning via Lifted Structured Feature Embedding}, + Booktitle = {CVPR}, + Year = {2016} + } -Caffe is released under the [BSD 2-Clause license](https://github.com/BVLC/caffe/blob/master/LICENSE). -The BVLC reference models are released for unrestricted use. +## Download prerequsites +Install prerequsites for `Caffe` (see: [Caffe installation instructions](http://caffe.berkeleyvision.org/installation.html)) -Please cite Caffe in your publications if it helps your research: +## - @article{jia2014caffe, - Author = {Jia, Yangqing and Shelhamer, Evan and Donahue, Jeff and Karayev, Sergey and Long, Jonathan and Girshick, Ross and Guadarrama, Sergio and Darrell, Trevor}, - Journal = {arXiv preprint arXiv:1408.5093}, - Title = {Caffe: Convolutional Architecture for Fast Feature Embedding}, - Year = {2014} - } +## Licence +MIT Licence From 338798f9adf34b08d2e2c92429a5812385331bc1 Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 20:24:52 -0800 Subject: [PATCH 02/12] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4359272c065..988e3b7aed4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Deep Metric Learning via Lifted Structured Feature Embedding +This repository is an extension of Caffe for the paper "Deep Metric Learning via Lifted Structured Feature Embedding" (CVPR16). ## Citing this work If you find this work useful in your research, please consider citing: From a41f80b405d0c5d357293ee143702d9e7e6aec85 Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 20:26:59 -0800 Subject: [PATCH 03/12] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 988e3b7aed4..4682c5c2d9b 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ If you find this work useful in your research, please consider citing: Year = {2016} } -## Download prerequsites -Install prerequsites for `Caffe` (see: [Caffe installation instructions](http://caffe.berkeleyvision.org/installation.html)) +## Download prerequisites +Install prerequisites for `Caffe` (see: [Caffe installation instructions](http://caffe.berkeleyvision.org/installation.html)) ## From 5b3a46b101e265d7234f46f926b4f62a57cadfa6 Mon Sep 17 00:00:00 2001 From: rksltnl Date: Thu, 10 Mar 2016 21:20:18 -0800 Subject: [PATCH 04/12] hos --- .../lifted_struct_similarity_softmax_layer.hpp | 47 +++++ .../lifted_struct_similarity_softmax_layer.cpp | 219 +++++++++++++++++++++ src/caffe/proto/caffe.proto | 7 +- ..._lifted_structsimilarity_softmax_loss_layer.cpp | 64 ++++++ 4 files changed, 336 insertions(+), 1 deletion(-) create mode 100644 include/caffe/layers/lifted_struct_similarity_softmax_layer.hpp create mode 100644 src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp create mode 100644 src/caffe/test/test_lifted_structsimilarity_softmax_loss_layer.cpp diff --git a/include/caffe/layers/lifted_struct_similarity_softmax_layer.hpp b/include/caffe/layers/lifted_struct_similarity_softmax_layer.hpp new file mode 100644 index 00000000000..ef82d938f8b --- /dev/null +++ b/include/caffe/layers/lifted_struct_similarity_softmax_layer.hpp @@ -0,0 +1,47 @@ +#ifndef CAFFE_LIFTED_STRUCT_SIMILARITY_LOSS_LAYER_HPP_ +#define CAFFE_LIFTED_STRUCT_SIMILARITY_LOSS_LAYER_HPP_ + +#include + +#include "caffe/blob.hpp" +#include "caffe/layer.hpp" +#include "caffe/proto/caffe.pb.h" + +#include "caffe/layers/loss_layer.hpp" + +namespace caffe { + +template +class LiftedStructSimilaritySoftmaxLossLayer : public LossLayer { + public: + explicit LiftedStructSimilaritySoftmaxLossLayer(const LayerParameter& param) + : LossLayer(param) {} + virtual void LayerSetUp(const vector*>& bottom, + const vector*>& top); + + virtual inline int ExactNumBottomBlobs() const { return 2; } + virtual inline const char* type() const { return "LiftedStructSimilaritySoftmaxLoss"; } + virtual inline bool AllowForceBackward(const int bottom_index) const { + return bottom_index != 1; + } + + protected: + virtual void Forward_cpu(const vector*>& bottom, + const vector*>& top); + + virtual void Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom); + + Blob dist_sq_; // cached for backward pass + Blob dot_; + Blob ones_; + Blob blob_pos_diff_; + Blob blob_neg_diff_; + Blob loss_aug_inference_; + Blob summer_vec_; + Dtype num_constraints; +}; + +} // namespace caffe + +#endif // CAFFE_LIFTED_STRUCT_SIMILARITY_LOSS_LAYER_HPP_ diff --git a/src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp b/src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp new file mode 100644 index 00000000000..b8216a6d58d --- /dev/null +++ b/src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp @@ -0,0 +1,219 @@ +#include +#include + +#include "caffe/layers/lifted_struct_similarity_softmax_layer.hpp" +#include "caffe/util/math_functions.hpp" + +namespace caffe { + +template +void LiftedStructSimilaritySoftmaxLossLayer::LayerSetUp(const vector*>& bottom, const vector*>& top) { + LossLayer::LayerSetUp(bottom, top); + CHECK_EQ(bottom[0]->height(), 1); + CHECK_EQ(bottom[0]->width(), 1); + CHECK_EQ(bottom[1]->channels(), 1); + CHECK_EQ(bottom[1]->height(), 1); + CHECK_EQ(bottom[1]->width(), 1); + // List of member variables defined in /include/caffe/loss_layers.hpp; + // diff_, dist_sq_, summer_vec_, loss_aug_inference_; + dist_sq_.Reshape(bottom[0]->num(), 1, 1, 1); + dot_.Reshape(bottom[0]->num(), bottom[0]->num(), 1, 1); + ones_.Reshape(bottom[0]->num(), 1, 1, 1); // n by 1 vector of ones. + for (int i=0; i < bottom[0]->num(); ++i){ + ones_.mutable_cpu_data()[i] = Dtype(1); + } + blob_pos_diff_.Reshape(bottom[0]->channels(), 1, 1, 1); + blob_neg_diff_.Reshape(bottom[0]->channels(), 1, 1, 1); +} + +template +void LiftedStructSimilaritySoftmaxLossLayer::Forward_cpu(const vector*>& bottom, const vector*>& top) { + + const int channels = bottom[0]->channels(); + for (int i = 0; i < bottom[0]->num(); i++){ + dist_sq_.mutable_cpu_data()[i] = caffe_cpu_dot(channels, bottom[0]->cpu_data() + (i*channels), bottom[0]->cpu_data() + (i*channels)); + } + + int M_ = bottom[0]->num(); + int N_ = bottom[0]->num(); + int K_ = bottom[0]->channels(); + + const Dtype* bottom_data1 = bottom[0]->cpu_data(); + const Dtype* bottom_data2 = bottom[0]->cpu_data(); + + Dtype dot_scaler(-2.0); + caffe_cpu_gemm(CblasNoTrans, CblasTrans, M_, N_, K_, dot_scaler, bottom_data1, bottom_data2, (Dtype)0., dot_.mutable_cpu_data()); + + // add ||x_i||^2 to all elements in row i + for (int i=0; i > label_mat(N_, vector(N_, false)); + for (int i=0; icpu_data()[i] == bottom[1]->cpu_data()[j]); + } + } + + Dtype margin = this->layer_param_.lifted_struct_sim_softmax_loss_param().margin(); + Dtype loss(0.0); + num_constraints = Dtype(0.0); + const Dtype* bin = bottom[0]->cpu_data(); + Dtype* bout = bottom[0]->mutable_cpu_diff(); + + // zero initialize bottom[0]->mutable_cpu_diff(); + for (int i=0; imutable_cpu_data()[0] = loss; +} + +template +void LiftedStructSimilaritySoftmaxLossLayer::Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom) { + + const Dtype alpha = top[0]->cpu_diff()[0] / num_constraints / Dtype(2.0); + + int num = bottom[0]->num(); + int channels = bottom[0]->channels(); + for (int i = 0; i < num; i++){ + Dtype* bout = bottom[0]->mutable_cpu_diff(); + caffe_scal(channels, alpha, bout + (i*channels)); + } +} + +#ifdef CPU_ONLY +STUB_GPU(LiftedStructSimilaritySoftmaxLossLayer); +#endif + +INSTANTIATE_CLASS(LiftedStructSimilaritySoftmaxLossLayer); +REGISTER_LAYER_CLASS(LiftedStructSimilaritySoftmaxLoss); + +} // namespace caffe + + + diff --git a/src/caffe/proto/caffe.proto b/src/caffe/proto/caffe.proto index 6900bb71482..a8c608d614a 100644 --- a/src/caffe/proto/caffe.proto +++ b/src/caffe/proto/caffe.proto @@ -306,7 +306,7 @@ message ParamSpec { // NOTE // Update the next available ID when you add a new LayerParameter field. // -// LayerParameter next available layer-specific ID: 145 (last added: crop_param) +// LayerParameter next available layer-specific ID: 146 (last added: lifted_struct_sim) message LayerParameter { optional string name = 1; // the layer name optional string type = 2; // the layer type @@ -376,6 +376,7 @@ message LayerParameter { optional InfogainLossParameter infogain_loss_param = 116; optional InnerProductParameter inner_product_param = 117; optional InputParameter input_param = 143; + optional LiftedStructSimilaritySoftmaxLossParameter lifted_struct_sim_softmax_loss_param = 145; optional LogParameter log_param = 134; optional LRNParameter lrn_param = 118; optional MemoryDataParameter memory_data_param = 119; @@ -760,6 +761,10 @@ message HingeLossParameter { optional Norm norm = 1 [default = L1]; } +message LiftedStructSimilaritySoftmaxLossParameter { + optional float margin = 1 [default = 1.0]; +} + message ImageDataParameter { // Specify the data source. optional string source = 1; diff --git a/src/caffe/test/test_lifted_structsimilarity_softmax_loss_layer.cpp b/src/caffe/test/test_lifted_structsimilarity_softmax_loss_layer.cpp new file mode 100644 index 00000000000..a2b68caa593 --- /dev/null +++ b/src/caffe/test/test_lifted_structsimilarity_softmax_loss_layer.cpp @@ -0,0 +1,64 @@ +#include +#include + +#include "gtest/gtest.h" + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/layers/lifted_struct_similarity_softmax_layer.hpp" + +#include "caffe/test/test_caffe_main.hpp" +#include "caffe/test/test_gradient_check_util.hpp" + +namespace caffe { + +template +class LiftedStructSimilaritySoftmaxLossLayerTest : public MultiDeviceTest { + typedef typename TypeParam::Dtype Dtype; + + protected: + LiftedStructSimilaritySoftmaxLossLayerTest() : blob_bottom_data(new Blob(128, 64, 1, 1)), + blob_bottom_y_(new Blob(128, 1, 1, 1)), + blob_top_loss_(new Blob()) { + // fill the values + FillerParameter filler_param; + filler_param.set_min(-1.0); + filler_param.set_max(1.0); // distances~=1.0 to test both sides of margin + UniformFiller filler(filler_param); + filler.Fill(this->blob_bottom_data); + blob_bottom_vec_.push_back(blob_bottom_data); + for (int i = 0; i < blob_bottom_y_->count(); ++i) { + //blob_bottom_y_->mutable_cpu_data()[i] = caffe_rng_rand() % 2; // 0 or 1 + blob_bottom_y_->mutable_cpu_data()[i] = i/2; + } + blob_bottom_vec_.push_back(blob_bottom_y_); + blob_top_vec_.push_back(blob_top_loss_); + } + virtual ~LiftedStructSimilaritySoftmaxLossLayerTest() { + delete blob_bottom_data; + delete blob_bottom_y_; + delete blob_top_loss_; + } + + Blob* const blob_bottom_data; + Blob* const blob_bottom_y_; + Blob* const blob_top_loss_; + vector*> blob_bottom_vec_; + vector*> blob_top_vec_; +}; + +TYPED_TEST_CASE(LiftedStructSimilaritySoftmaxLossLayerTest, TestDtypesAndDevices); + +TYPED_TEST(LiftedStructSimilaritySoftmaxLossLayerTest, TestGradient) { + typedef typename TypeParam::Dtype Dtype; + LayerParameter layer_param; + LiftedStructSimilaritySoftmaxLossLayer layer(layer_param); + layer.SetUp(this->blob_bottom_vec_, this->blob_top_vec_); + GradientChecker checker(1e-2, 1e-2, 1701); + // check the gradient for the first two bottom layers + checker.CheckGradientExhaustive(&layer, this->blob_bottom_vec_, + this->blob_top_vec_, 0); +} + +} // namespace caffe From d7a1629a6d2ed1ae58f14bba692a43b1eb3f9fe6 Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 21:36:37 -0800 Subject: [PATCH 05/12] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4682c5c2d9b..4f5af827373 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ If you find this work useful in your research, please consider citing: Year = {2016} } -## Download prerequisites -Install prerequisites for `Caffe` (see: [Caffe installation instructions](http://caffe.berkeleyvision.org/installation.html)) +## Installation +1. Install prerequisites for `Caffe` (see: [Caffe installation instructions](http://caffe.berkeleyvision.org/installation.html)) -## +2. Compile this caffe extension. `make all` ## Licence MIT Licence From 7ef605868e776bf85f26cb6ca819a86683e17f5d Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 21:36:59 -0800 Subject: [PATCH 06/12] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4f5af827373..f5fb39e1d38 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ If you find this work useful in your research, please consider citing: ## Installation 1. Install prerequisites for `Caffe` (see: [Caffe installation instructions](http://caffe.berkeleyvision.org/installation.html)) -2. Compile this caffe extension. `make all` +2. Compile this caffe extension. +`make all` +`make test` ## Licence MIT Licence From 14bc2f4033ea890a27a05b990acf64ad471fa4f1 Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 21:37:21 -0800 Subject: [PATCH 07/12] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index f5fb39e1d38..570bd9a70e5 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,7 @@ If you find this work useful in your research, please consider citing: 1. Install prerequisites for `Caffe` (see: [Caffe installation instructions](http://caffe.berkeleyvision.org/installation.html)) 2. Compile this caffe extension. -`make all` -`make test` +`make all`, `make test` ## Licence MIT Licence From 4d5a89823b8b2127ec8be792180346c785b71391 Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 21:37:32 -0800 Subject: [PATCH 08/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 570bd9a70e5..dc49b150a96 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ If you find this work useful in your research, please consider citing: 1. Install prerequisites for `Caffe` (see: [Caffe installation instructions](http://caffe.berkeleyvision.org/installation.html)) 2. Compile this caffe extension. -`make all`, `make test` +Run `make all`, `make test` ## Licence MIT Licence From 7f1a71ed082035e2db986e90715cd82bd61dec62 Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 21:42:12 -0800 Subject: [PATCH 09/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dc49b150a96..13aaeb0a3a1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Deep Metric Learning via Lifted Structured Feature Embedding -This repository is an extension of Caffe for the paper "Deep Metric Learning via Lifted Structured Feature Embedding" (CVPR16). +This repository is an extension of [Caffe](https://github.com/bvlc/caffe) for the paper "Deep Metric Learning via Lifted Structured Feature Embedding" (CVPR16). ## Citing this work If you find this work useful in your research, please consider citing: From 0efd7544a9846f58df923c8b992198ba5c355454 Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 21:48:04 -0800 Subject: [PATCH 10/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 13aaeb0a3a1..6eed1aac92c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Deep Metric Learning via Lifted Structured Feature Embedding -This repository is an extension of [Caffe](https://github.com/bvlc/caffe) for the paper "Deep Metric Learning via Lifted Structured Feature Embedding" (CVPR16). +This repository is an extension of [Caffe](https://github.com/bvlc/caffe) for the paper "Deep Metric Learning via Lifted Structured Feature Embedding" (CVPR16). Also, this is a Github submodule for the main repository at [Deep-Metric-Learning-CVPR16](https://github.com/rksltnl/Deep-Metric-Learning-CVPR16). ## Citing this work If you find this work useful in your research, please consider citing: From 993325902b1af18f1c0240d36bbd86d48ef2b252 Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Thu, 10 Mar 2016 22:46:26 -0800 Subject: [PATCH 11/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6eed1aac92c..2a038ba0a6f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ If you find this work useful in your research, please consider citing: @inproceedings{songCVPR16, Author = {Hyun Oh Song and Yu Xiang and Stefanie Jegelka and Silvio Savarese}, Title = {Deep Metric Learning via Lifted Structured Feature Embedding}, - Booktitle = {CVPR}, + Booktitle = {Computer Vision and Pattern Recognition (CVPR)}, Year = {2016} } From 7e416f667d4d7aec167655b2d5863f7f03d5885d Mon Sep 17 00:00:00 2001 From: Hyun Oh Song Date: Fri, 29 Apr 2016 10:24:32 -0700 Subject: [PATCH 12/12] Update lifted_struct_similarity_softmax_layer.cpp --- src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp b/src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp index b8216a6d58d..85c40581b14 100644 --- a/src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp +++ b/src/caffe/layers/lifted_struct_similarity_softmax_layer.cpp @@ -206,9 +206,9 @@ void LiftedStructSimilaritySoftmaxLossLayer::Backward_cpu(const vector