From 830d484093ab6e1b72a00f770348703943f55485 Mon Sep 17 00:00:00 2001 From: Saurabh Gupta Date: Fri, 29 May 2015 14:51:50 -0700 Subject: [PATCH 1/8] function to upsample the image to the appropriate size --- include/caffe/data_transformer.hpp | 13 +++++++++++++ src/caffe/data_transformer.cpp | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/include/caffe/data_transformer.hpp b/include/caffe/data_transformer.hpp index 880356601a4..78ba7f6495d 100644 --- a/include/caffe/data_transformer.hpp +++ b/include/caffe/data_transformer.hpp @@ -6,6 +6,7 @@ #include "caffe/blob.hpp" #include "caffe/common.hpp" #include "caffe/proto/caffe.pb.h" +#include "opencv2/opencv.hpp" namespace caffe { @@ -87,6 +88,18 @@ class DataTransformer { */ void Transform(Blob* input_blob, Blob* transformed_blob); + /** + * @brief Imresizes the image so that the longer side is of length img_size + * and flip it if do_mirror is on + * @param cv_img + * cv::Mat for the image to transform + * @param img_size + * int, final length of the longer side + * @param do_mirror + * bool whether to flip or not + */ + cv::Mat Transform_IDL(cv::Mat cv_img, int img_size, bool do_mirror); + protected: /** * @brief Generates a random integer from Uniform({0, 1, ..., n-1}). diff --git a/src/caffe/data_transformer.cpp b/src/caffe/data_transformer.cpp index b0b98e478c1..60f52a1b0a3 100644 --- a/src/caffe/data_transformer.cpp +++ b/src/caffe/data_transformer.cpp @@ -11,6 +11,30 @@ namespace caffe { template +cv::Mat DataTransformer::Transform_IDL(cv::Mat cv_img, int img_size, bool do_mirror){ + cv::Size cv_size; //(width, height) + if(cv_img.cols > cv_img.rows){ + cv::Size tmp(img_size, round(cv_img.rows*img_size/cv_img.cols)); + cv_size = tmp; + } + else{ + cv::Size tmp(round(cv_img.cols*img_size/cv_img.rows), img_size); + cv_size = tmp; + } + + cv::Mat cv_resized_img; + cv::resize(cv_img, cv_resized_img, + cv_size, 0, 0, cv::INTER_LINEAR); + + // horizontal flip at random + if (do_mirror) { + cv::flip(cv_resized_img, cv_resized_img, 1); + } + return cv_resized_img; +} + + +template DataTransformer::DataTransformer(const TransformationParameter& param, Phase phase) : param_(param), phase_(phase) { From a7d38507359f7ce744e0ac35bc3e1bc2bd6dd060 Mon Sep 17 00:00:00 2001 From: Saurabh Gupta Date: Fri, 29 May 2015 15:01:52 -0700 Subject: [PATCH 2/8] layer for loading images and resizing them to a canonical size, loading labels from a hdf5 file --- include/caffe/data_layers.hpp | 36 +++++++ src/caffe/layers/mil_data_layer.cpp | 188 ++++++++++++++++++++++++++++++++++++ src/caffe/proto/caffe.proto | 32 ++++++ 3 files changed, 256 insertions(+) create mode 100644 src/caffe/layers/mil_data_layer.cpp diff --git a/include/caffe/data_layers.hpp b/include/caffe/data_layers.hpp index 3958cb7ecb0..9618afe03b1 100644 --- a/include/caffe/data_layers.hpp +++ b/include/caffe/data_layers.hpp @@ -322,6 +322,42 @@ class WindowDataLayer : public BasePrefetchingDataLayer { vector > image_database_cache_; }; + +/** + * @brief Uses a text file which specifies the image names, and a hdf5 file for the labels. + * Note that each image can have multiple positive labels. + * + * TODO(dox): thorough documentation for Forward and proto params. + */ +template +class MILDataLayer : public BasePrefetchingDataLayer { + public: + explicit MILDataLayer(const LayerParameter& param) + : BasePrefetchingDataLayer(param) {} + virtual ~MILDataLayer(); + virtual void DataLayerSetUp(const vector*>& bottom, + vector*>& top); + + virtual inline const char* type() const { return "MILData"; } + virtual inline int ExactNumBottomBlobs() const { return 0; } + virtual inline int ExactNumTopBlobs() const { return 2; } + + protected: + virtual unsigned int PrefetchRand(); + virtual void InternalThreadEntry(); + int num_images_; + unsigned int counter_; + shared_ptr prefetch_rng_; + vector< std::pair > image_database_; + hid_t label_file_id_; + + vector mean_value_; + Blob label_blob_; +}; + + + + } // namespace caffe #endif // CAFFE_DATA_LAYERS_HPP_ diff --git a/src/caffe/layers/mil_data_layer.cpp b/src/caffe/layers/mil_data_layer.cpp new file mode 100644 index 00000000000..2f00dbca522 --- /dev/null +++ b/src/caffe/layers/mil_data_layer.cpp @@ -0,0 +1,188 @@ +#include + +#include +#include +#include +#include +#include + +#include "opencv2/core/core.hpp" +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/imgproc/imgproc.hpp" + +#include "caffe/common.hpp" +#include "caffe/data_layers.hpp" +#include "caffe/layer.hpp" +#include "caffe/util/io.hpp" +#include "caffe/util/math_functions.hpp" +#include "caffe/util/rng.hpp" + +namespace caffe { + +template +MILDataLayer::~MILDataLayer() { + this->JoinPrefetchThread(); +} + +template +void MILDataLayer::DataLayerSetUp(const vector*>& bottom, + vector*>& top) { + // LayerSetUp runs through the image name file and stores them. + + string label_file = this->layer_param_.mil_data_param().label_file().c_str(); + + LOG(INFO) << "Loading labels from: "<< label_file; + label_file_id_ = H5Fopen(label_file.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); + + LOG(INFO) << "MIL Data layer:" << std::endl; + std::ifstream infile(this->layer_param_.mil_data_param().source().c_str()); + CHECK(infile.good()) << "Failed to open window file " + << this->layer_param_.mil_data_param().source() << std::endl; + + const int channels = this->layer_param_.mil_data_param().channels(); + const int img_size = this->transform_param_.crop_size(); + const int scale = this->transform_param_.scale(); + const int images_per_batch = this->layer_param_.mil_data_param().images_per_batch(); + const int n_classes = this->layer_param_.mil_data_param().n_classes(); + const int num_scales = this->layer_param_.mil_data_param().num_scales(); + const float scale_factor = this->layer_param_.mil_data_param().scale_factor(); + mean_value_.clear(); + std::copy(this->transform_param_.mean_value().begin(), + this->transform_param_.mean_value().end(), + std::back_inserter(mean_value_)); + + if(mean_value_.size() == 0) + for(int i = 0; i < channels; i++) + mean_value_.push_back((float)128.0); + + CHECK_EQ(mean_value_.size(), channels); + + LOG(INFO) << "MIL Data Layer: "<< "channels: " << channels; + LOG(INFO) << "MIL Data Layer: "<< "img_size: " << img_size; + LOG(INFO) << "MIL Data Layer: "<< "scale: " << scale; + LOG(INFO) << "MIL Data Layer: "<< "n_classes: " << n_classes; + LOG(INFO) << "MIL Data Layer: "<< "num_scales: " << num_scales; + LOG(INFO) << "MIL Data Layer: "<< "scale_factor: " << scale_factor; + LOG(INFO) << "MIL Data Layer: "<< "images_per_batch: " << images_per_batch; + for(int i = 0; i < mean_value_.size(); i++) + LOG(INFO) << "MIL Data Layer: "<< "mean_value[" << i << "]: " << mean_value_[i]; + + const bool prefetch_needs_rand = + this->transform_param_.mirror() || + this->transform_param_.crop_size(); + if (prefetch_needs_rand) { + const unsigned int prefetch_rng_seed = caffe_rng_rand(); + prefetch_rng_.reset(new Caffe::RNG(prefetch_rng_seed)); + } else { + prefetch_rng_.reset(); + } + + string im_name; + int count = 0; + while (infile >> im_name) { + string full_im_name = string(this->layer_param_.mil_data_param().root_dir() + "/" + im_name + "." + this->layer_param_.mil_data_param().ext()); + image_database_.push_back(std::make_pair(im_name, full_im_name)); + count = count+1; + if(count % 1000 == 0) + LOG(INFO) << "num: " << count << ", image_name: " << im_name; + } + num_images_ = count; + LOG(INFO) << "Number of images: " << count; + + top[0]->Reshape(images_per_batch*num_scales, channels, img_size, img_size); + this->prefetch_data_.Reshape(images_per_batch*num_scales, channels, img_size, img_size); + + LOG(INFO) << "output data size: " << top[0]->num() << "," + << top[0]->channels() << "," << top[0]->height() << "," + << top[0]->width(); + + // label + top[1]->Reshape(images_per_batch, n_classes, 1, 1); + this->prefetch_label_.Reshape(images_per_batch, n_classes, 1, 1); + + this->counter_ = 0; +} + +template +unsigned int MILDataLayer::PrefetchRand() { + CHECK(prefetch_rng_); + caffe::rng_t* prefetch_rng = + static_cast(prefetch_rng_->generator()); + return (*prefetch_rng)(); +} + +// Thread fetching the data +template +void MILDataLayer::InternalThreadEntry() { + Dtype* top_data = this->prefetch_data_.mutable_cpu_data(); + Dtype* top_label = this->prefetch_label_.mutable_cpu_data(); + const int img_size = this->transform_param_.crop_size(); + const int channels = this->layer_param_.mil_data_param().channels(); + const int scale = this->transform_param_.scale(); + const bool mirror = this->transform_param_.mirror(); + + const int images_per_batch = this->layer_param_.mil_data_param().images_per_batch(); + const int n_classes = this->layer_param_.mil_data_param().n_classes(); + const int num_scales = this->layer_param_.mil_data_param().num_scales(); + const float scale_factor = this->layer_param_.mil_data_param().scale_factor(); + + // zero out batch + caffe_set(this->prefetch_data_.count(), Dtype(0), top_data); + int item_id; + for(int i_image = 0; i_image < images_per_batch; i_image++){ + // Sample which image to read + unsigned int index = counter_; counter_ = counter_ + 1; + const unsigned int rand_index = this->PrefetchRand(); + if(this->layer_param_.mil_data_param().randomize()) + index = rand_index; + + // LOG(INFO) << index % this->num_images_ << ", " << this->num_images_; + pair p = this->image_database_[index % this->num_images_]; + string im_name = p.first; + string full_im_name = p.second; + + cv::Mat cv_img = cv::imread(full_im_name, CV_LOAD_IMAGE_COLOR); + if (!cv_img.data) { + LOG(ERROR) << "Could not open or find file " << full_im_name; + return; + } + + // hdf5_load_nd_dataset(this->label_file_id_, string("/label-"+im_name).c_str(), 2, 2, label_blob_); + hdf5_load_nd_dataset(this->label_file_id_, string("/labels-"+im_name).c_str(), 2, 2, &this->label_blob_); + const Dtype* label = label_blob_.mutable_cpu_data(); + // LOG(INFO) << "[width, height, channels, num] = " << label_blob_.width() << + // ", " << label_blob_.height() << ", " << label_blob_.channels() << ", " << label_blob_.num(); + CHECK_EQ(label_blob_.width(), 1) << "Expected width of label to be 1." ; + CHECK_EQ(label_blob_.height(), n_classes) << "Expected height of label to be " << n_classes; + + float img_size_i = img_size; + for(int i_scales = 0; i_scales < num_scales; i_scales++){ + // Resize such that the image is of size img_size, img_size + item_id = i_image*num_scales + i_scales; + // LOG(INFO) << "MIL Data Layer: scale: " << (int) round(img_size_i); + cv::Mat cv_cropped_img = this->data_transformer_->Transform_IDL(cv_img, (int) round(img_size_i), mirror); + for (int c = 0; c < channels; ++c) { + for (int h = 0; h < cv_cropped_img.rows; ++h) { + for (int w = 0; w < cv_cropped_img.cols; ++w) { + Dtype pixel = + static_cast(cv_cropped_img.at(h, w)[c]); + top_data[((item_id * channels + c) * img_size + h) + * img_size + w] + = (pixel - (Dtype)mean_value_[c])*scale; + } + } + } + img_size_i = std::max((float)1., img_size_i*scale_factor); + } + + for(int i_label = 0; i_label < n_classes; i_label++){ + top_label[i_image*n_classes + i_label] = + label[i_label]; + } + } +} + +INSTANTIATE_CLASS(MILDataLayer); +REGISTER_LAYER_CLASS(MILData); + +} // namespace caffe diff --git a/src/caffe/proto/caffe.proto b/src/caffe/proto/caffe.proto index c471fa0a93e..5877b402c0d 100644 --- a/src/caffe/proto/caffe.proto +++ b/src/caffe/proto/caffe.proto @@ -346,6 +346,7 @@ message LayerParameter { optional TanHParameter tanh_param = 127; optional ThresholdParameter threshold_param = 128; optional WindowDataParameter window_data_param = 129; + optional MILDataParameter mil_data_param = 10001; } // Message that stores parameters used to apply transformation @@ -832,6 +833,37 @@ message WindowDataParameter { optional string root_folder = 13 [default = ""]; } +message MILDataParameter { + // Specify the data source. + optional string source = 1; + + // Number of scales for each image + optional uint32 num_scales = 2 [default = 1]; + + // Side length ratio between neighbouring scales + optional float scale_factor = 6 [default = 1]; + + // Number of channels in the image + optional uint32 channels = 4 [default = 3]; + + // Specify the number of images per batch + optional uint32 images_per_batch = 3; + // Specify the number of classes + optional uint32 n_classes = 5; + // specify the box_dir and label_dir + optional string label_file = 7; + + // Root directory which contains all the images + optional string root_dir = 11; + // Extention for the file + optional string ext = 12; + + // To randomize or not + optional bool randomize = 13 [default = true]; +} + + + message SPPParameter { enum PoolMethod { MAX = 0; From 3c582433fb9d23925530e2465f3ce2abf47313cf Mon Sep 17 00:00:00 2001 From: Saurabh Gupta Date: Fri, 29 May 2015 15:44:42 -0700 Subject: [PATCH 3/8] layers for mil --- include/caffe/common_layers.hpp | 28 ++++++++++ src/caffe/layers/mil_layer.cpp | 119 ++++++++++++++++++++++++++++++++++++++++ src/caffe/proto/caffe.proto | 11 ++++ 3 files changed, 158 insertions(+) create mode 100644 src/caffe/layers/mil_layer.cpp diff --git a/include/caffe/common_layers.hpp b/include/caffe/common_layers.hpp index e6b42c14587..577f3ef66bc 100644 --- a/include/caffe/common_layers.hpp +++ b/include/caffe/common_layers.hpp @@ -498,6 +498,34 @@ class SliceLayer : public Layer { vector slice_point_; }; +/* MILLayer +*/ +template +class MILLayer : public Layer { + public: + explicit MILLayer(const LayerParameter& param) + : Layer(param) {} + virtual void LayerSetUp(const vector*>& bottom, + const vector*>& top); + virtual void Reshape(const vector*>& bottom, + const vector*>& top); + virtual inline const char* type() const { return "MIL"; } + virtual inline int ExactNumBottomBlobs() const { return 1; } + virtual inline int ExactNumTopBlobs() const { return 1; } + protected: + virtual void Forward_cpu(const vector*>& bottom, + const vector*>& top); + // virtual Dtype Forward_gpu(const vector*>& bottom, + // vector*>* top); + virtual void Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom); + // virtual void Backward_gpu(const vector*>& top, + // const vector& propagate_down, vector*>* bottom); + + int channels_, height_, width_, num_images_; +}; + + } // namespace caffe #endif // CAFFE_COMMON_LAYERS_HPP_ diff --git a/src/caffe/layers/mil_layer.cpp b/src/caffe/layers/mil_layer.cpp new file mode 100644 index 00000000000..643297a337c --- /dev/null +++ b/src/caffe/layers/mil_layer.cpp @@ -0,0 +1,119 @@ +// Copyright 2014 BVLC and contributors. + +#include +#include +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/layer.hpp" +#include "caffe/vision_layers.hpp" +#include "caffe/util/math_functions.hpp" + +using std::max; +using std::min; + +namespace caffe { + +template +void MILLayer::LayerSetUp(const vector*>& bottom, + const vector*>& top) { + CHECK_EQ(bottom.size(), 1) << "MIL Layer takes a single blob as input."; + CHECK_EQ(top.size(), 1) << "MIL Layer takes a single blob as output."; +} +template +void MILLayer::Reshape(const vector*>& bottom, + const vector*>& top){ + top[0]->Reshape(bottom[0]->num(), bottom[0]->channels(), 1, 1); +} + +template +void MILLayer::Forward_cpu(const vector*>& bottom, + const vector*>& top) { + const Dtype* bottom_data = bottom[0]->cpu_data(); + Dtype* top_data = top[0]->mutable_cpu_data(); + int offset; + channels_ = bottom[0]->channels(); + num_images_ = bottom[0]->num(); + height_ = bottom[0]->height(); + width_ = bottom[0]->width(); + + for(int j = 0; j < channels_; j++){ + for(int i = 0; i < num_images_; i++){ + Dtype prob, max_prob; + + switch (this->layer_param_.mil_param().type()) { + case MILParameter_MILType_MAX: + prob = -FLT_MAX; + offset = bottom[0]->offset(i, j); + for(int k = 0; k < height_; k++){ + for(int l = 0; l < width_; l++){ + prob = max(prob, bottom_data[offset]); + offset = offset + 1; + } + } + top_data[i*channels_ + j] = prob; + break; + + case MILParameter_MILType_NOR: + prob = 1.; max_prob = -FLT_MAX; + offset = bottom[0]->offset(i, j); + for(int k = 0; k < height_; k++){ + for(int l = 0; l < width_; l++){ + prob = prob*(1. - bottom_data[offset]); + max_prob = max(max_prob, bottom_data[offset]); + CHECK_LE(bottom_data[offset], 1.) << "input mil_prob not <= 1"; + CHECK_GE(bottom_data[offset], 0.) << "input mil_prob not >= 0"; + offset = offset + 1; + } + } + top_data[i*channels_ + j] = max(Dtype(1.) - prob, max_prob); + CHECK_LE(top_data[i*channels_ + j], 1.) << "mil_prob not <= 1"; + CHECK_GE(top_data[i*channels_ + j], 0.) << "mil_prob not >= 0"; + break; + } + } + } +} + +template +void MILLayer::Backward_cpu(const vector*>& top, + const vector& propagate_down, + const vector*>& bottom) { + + const Dtype* top_diff = top[0]->cpu_diff(); + const Dtype* top_data = top[0]->cpu_data(); + const Dtype* bottom_data = bottom[0]->cpu_data(); + Dtype* bottom_diff = bottom[0]->mutable_cpu_diff(); + int offset; + + if(propagate_down[0]){ + for(int j = 0; j < channels_; j++){ + for(int i = 0; i < num_images_; i++){ + offset = bottom[0]->offset(i, j); + + for(int k = 0; k < height_; k++){ + for(int l = 0; l < width_; l++){ + switch (this->layer_param_.mil_param().type()) { + case MILParameter_MILType_MAX: + bottom_diff[offset] = + top_diff[i*channels_ + j] * (top_data[i*channels_ + j] == bottom_data[offset]); + break; + case MILParameter_MILType_NOR: + bottom_diff[offset] = top_diff[i*channels_ + j] * + min(Dtype(1.),((1-top_data[i*channels_ + j])/(1-bottom_data[offset]))); + break; + } + offset = offset + 1; + } + } + + } + } + } +} + +INSTANTIATE_CLASS(MILLayer); +REGISTER_LAYER_CLASS(MIL); +} // namespace caffe diff --git a/src/caffe/proto/caffe.proto b/src/caffe/proto/caffe.proto index 5877b402c0d..93c8272310a 100644 --- a/src/caffe/proto/caffe.proto +++ b/src/caffe/proto/caffe.proto @@ -347,6 +347,7 @@ message LayerParameter { optional ThresholdParameter threshold_param = 128; optional WindowDataParameter window_data_param = 129; optional MILDataParameter mil_data_param = 10001; + optional MILParameter mil_param = 10002; } // Message that stores parameters used to apply transformation @@ -800,6 +801,16 @@ message ThresholdParameter { optional float threshold = 1 [default = 0]; // Strictly positive values } +// Message that stores parameters used by MILLayer +message MILParameter { + enum MILType { + MAX = 0; + NOR = 1; + } + optional MILType type = 1 [default = MAX]; // The MIL method +} + + message WindowDataParameter { // Specify the data source. optional string source = 1; From 45de6be0b526b1379a6bd6463f0a7eb67a399be1 Mon Sep 17 00:00:00 2001 From: Saurabh Gupta Date: Sun, 31 May 2015 11:45:13 -0700 Subject: [PATCH 4/8] Cross Entropy Loss Layer --- include/caffe/loss_layers.hpp | 19 ++++++ src/caffe/layers/cross_entropy_loss_layer.cpp | 83 +++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 src/caffe/layers/cross_entropy_loss_layer.cpp diff --git a/include/caffe/loss_layers.hpp b/include/caffe/loss_layers.hpp index 86c34241168..559da830fae 100644 --- a/include/caffe/loss_layers.hpp +++ b/include/caffe/loss_layers.hpp @@ -651,6 +651,25 @@ class SigmoidCrossEntropyLossLayer : public LossLayer { vector*> sigmoid_top_vec_; }; +/* CrossEntropyLossLayer +*/ +template +class CrossEntropyLossLayer : public LossLayer { + public: + explicit CrossEntropyLossLayer(const LayerParameter& param) + : LossLayer(param) {} + virtual void LayerSetUp(const vector*>& bottom, + const vector*>& top); + + protected: + virtual void Forward_cpu(const vector*>& bottom, + const vector*>& top); + virtual void Backward_cpu(const vector*>& top, + const vector& propagate_down, const vector*>& bottom); +}; + + + // Forward declare SoftmaxLayer for use in SoftmaxWithLossLayer. template class SoftmaxLayer; diff --git a/src/caffe/layers/cross_entropy_loss_layer.cpp b/src/caffe/layers/cross_entropy_loss_layer.cpp new file mode 100644 index 00000000000..e2cb551b941 --- /dev/null +++ b/src/caffe/layers/cross_entropy_loss_layer.cpp @@ -0,0 +1,83 @@ +// Copyright 2014 BVLC and contributors. + +#include +#include +#include +#include + +#include "caffe/layer.hpp" +#include "caffe/vision_layers.hpp" +#include "caffe/util/math_functions.hpp" +#include "caffe/util/io.hpp" + +using std::max; + +namespace caffe { + +template +void CrossEntropyLossLayer::LayerSetUp( + const vector*>& bottom, const vector*>& top) { +} + +template +void CrossEntropyLossLayer::Forward_cpu( + const vector*>& bottom, const vector*>& top) { + const Dtype* bottom_data = bottom[0]->cpu_data(); + const Dtype* bottom_label = bottom[1]->cpu_data(); + int count = bottom[0]->count(); + int num = bottom[0]->num(); + Dtype loss = 0; Dtype l; + for (int i = 0; i < count; ++i) { + if(bottom_label[i] != 0){ + l = -bottom_label[i]*log(max(bottom_data[i],Dtype(0.00001))); + loss += l; + } + + if(bottom_label[i] != 1){ + l = -(1-bottom_label[i])*log(max((1-bottom_data[i]),Dtype(0.00001))); + loss += l; + } + CHECK_GE(l, 0.) << "loss is not >= 0, loss: " << l << " bottom_label: " << bottom_label[i] << " bottom_data: " << bottom_data[i]; + } + top[0]->mutable_cpu_data()[0] = loss / count; + // LOG(INFO) << "CrossEntropyLossLayer: " << loss / count; +} + +template +void CrossEntropyLossLayer::Backward_cpu( + const vector*>& top, const vector& propagate_down, + const vector*>& bottom) { + + if (propagate_down[1]) { + LOG(FATAL) << this->type() + << " Layer cannot backpropagate to label inputs."; + } + if (propagate_down[0]) { + const Dtype* bottom_data = bottom[0]->cpu_data(); + const Dtype* bottom_label = bottom[1]->cpu_data(); + Dtype* bottom_diff = bottom[0]->mutable_cpu_diff(); + int count = bottom[0]->count(); + int num = bottom[0]->num(); + memset(bottom_diff, 0, sizeof(Dtype) * bottom[0]->count()); + for (int i = 0; i < count; ++i) { + Dtype val = 0; + if(bottom_label[i] != 0) + val = val - bottom_label[i]/max(bottom_data[i], Dtype(0.00001))/count; + if(bottom_label[i] != 1) + val = val - (-(1-bottom_label[i])/max(1-bottom_data[i], Dtype(0.00001))/count); + + bottom_diff[i] = val; + } + const Dtype loss_weight = top[0]->cpu_diff()[0]; + caffe_scal(count, loss_weight, bottom_diff); + } +} + +#ifdef CPU_ONLY +STUB_GPU(CrossEntropyLossLayer); +#endif + +INSTANTIATE_CLASS(CrossEntropyLossLayer); +REGISTER_LAYER_CLASS(CrossEntropyLoss); + +} // namespace caffe From df4b1d911b270c1217577d979931b324365b9f6d Mon Sep 17 00:00:00 2001 From: Saurabh Gupta Date: Sun, 31 May 2015 17:34:44 -0700 Subject: [PATCH 5/8] bug fix in funsction header, changes in checks for label blob size --- include/caffe/data_layers.hpp | 2 +- src/caffe/layers/mil_data_layer.cpp | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/include/caffe/data_layers.hpp b/include/caffe/data_layers.hpp index 9618afe03b1..d5702688c37 100644 --- a/include/caffe/data_layers.hpp +++ b/include/caffe/data_layers.hpp @@ -336,7 +336,7 @@ class MILDataLayer : public BasePrefetchingDataLayer { : BasePrefetchingDataLayer(param) {} virtual ~MILDataLayer(); virtual void DataLayerSetUp(const vector*>& bottom, - vector*>& top); + const vector*>& top); virtual inline const char* type() const { return "MILData"; } virtual inline int ExactNumBottomBlobs() const { return 0; } diff --git a/src/caffe/layers/mil_data_layer.cpp b/src/caffe/layers/mil_data_layer.cpp index 2f00dbca522..70e2307c574 100644 --- a/src/caffe/layers/mil_data_layer.cpp +++ b/src/caffe/layers/mil_data_layer.cpp @@ -26,7 +26,7 @@ MILDataLayer::~MILDataLayer() { template void MILDataLayer::DataLayerSetUp(const vector*>& bottom, - vector*>& top) { + const vector*>& top) { // LayerSetUp runs through the image name file and stores them. string label_file = this->layer_param_.mil_data_param().label_file().c_str(); @@ -148,13 +148,16 @@ void MILDataLayer::InternalThreadEntry() { } // hdf5_load_nd_dataset(this->label_file_id_, string("/label-"+im_name).c_str(), 2, 2, label_blob_); - hdf5_load_nd_dataset(this->label_file_id_, string("/labels-"+im_name).c_str(), 2, 2, &this->label_blob_); + hdf5_load_nd_dataset(this->label_file_id_, string("/labels-"+im_name).c_str(), 4, 4, &this->label_blob_); const Dtype* label = label_blob_.mutable_cpu_data(); // LOG(INFO) << "[width, height, channels, num] = " << label_blob_.width() << - // ", " << label_blob_.height() << ", " << label_blob_.channels() << ", " << label_blob_.num(); - CHECK_EQ(label_blob_.width(), 1) << "Expected width of label to be 1." ; - CHECK_EQ(label_blob_.height(), n_classes) << "Expected height of label to be " << n_classes; + // ", " << label_blob_.height() << ", " << label_blob_.channels() << ", " << label_blob_.num(); + CHECK_EQ(label_blob_.width(), 1) << "Expected width of label to be 1." ; + CHECK_EQ(label_blob_.height(), n_classes) << "Expected height of label to be " << n_classes; + CHECK_EQ(label_blob_.channels(), 1) << "Expected channels of label to be 1." ; + CHECK_EQ(label_blob_.num(), 1) << "Expected num of label to be 1." ; + float img_size_i = img_size; for(int i_scales = 0; i_scales < num_scales; i_scales++){ // Resize such that the image is of size img_size, img_size From a0787631a27ca6478f70341462aafdcf35dabb19 Mon Sep 17 00:00:00 2001 From: Saurabh Gupta Date: Sun, 31 May 2015 17:35:10 -0700 Subject: [PATCH 6/8] removed check which fails for deep datasets in hdf5 files --- src/caffe/util/io.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/caffe/util/io.cpp b/src/caffe/util/io.cpp index 77ef7f257f4..a524cf3714a 100644 --- a/src/caffe/util/io.cpp +++ b/src/caffe/util/io.cpp @@ -234,8 +234,8 @@ void hdf5_load_nd_dataset_helper( hid_t file_id, const char* dataset_name_, int min_dim, int max_dim, Blob* blob) { // Verify that the dataset exists. - CHECK(H5LTfind_dataset(file_id, dataset_name_)) - << "Failed to find HDF5 dataset " << dataset_name_; + // CHECK(H5LTfind_dataset(file_id, dataset_name_)) + // << "Failed to find HDF5 dataset " << dataset_name_; // Verify that the number of dimensions is in the accepted range. herr_t status; int ndims; From 4c49424172a1e334ccd5f944337fae7fd6c1a705 Mon Sep 17 00:00:00 2001 From: Saurabh Gupta Date: Mon, 1 Jun 2015 16:52:17 -0700 Subject: [PATCH 7/8] fix to making pycaffe --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e4e66dfd138..414b2a6b2dd 100644 --- a/Makefile +++ b/Makefile @@ -439,11 +439,11 @@ py$(PROJECT): py py: $(PY$(PROJECT)_SO) $(PROTO_GEN_PY) -$(PY$(PROJECT)_SO): $(PY$(PROJECT)_SRC) $(PY$(PROJECT)_HXX) | $(DYNAMIC_NAME) +$(PY$(PROJECT)_SO): $(PY$(PROJECT)_SRC) $(PY$(PROJECT)_HXX) $(DYNAMIC_NAME) @ echo CXX/LD -o $@ $< $(Q)$(CXX) -shared -o $@ $(PY$(PROJECT)_SRC) \ -o $@ $(LINKFLAGS) -l$(PROJECT) $(PYTHON_LDFLAGS) \ - -Wl,-rpath,$(ORIGIN)/../../build/lib + -Wl,-rpath,$(ORIGIN)/../../$(BUILD_DIR)/lib mat$(PROJECT): mat From d65cdb9f44f86b32e965dcc6e623bfed48b24180 Mon Sep 17 00:00:00 2001 From: Saurabh Gupta Date: Tue, 11 Aug 2015 11:58:38 -0700 Subject: [PATCH 8/8] removed NO_GPU stub in cross entropy layer --- src/caffe/layers/cross_entropy_loss_layer.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/caffe/layers/cross_entropy_loss_layer.cpp b/src/caffe/layers/cross_entropy_loss_layer.cpp index e2cb551b941..0cd7e9c5c68 100644 --- a/src/caffe/layers/cross_entropy_loss_layer.cpp +++ b/src/caffe/layers/cross_entropy_loss_layer.cpp @@ -73,10 +73,6 @@ void CrossEntropyLossLayer::Backward_cpu( } } -#ifdef CPU_ONLY -STUB_GPU(CrossEntropyLossLayer); -#endif - INSTANTIATE_CLASS(CrossEntropyLossLayer); REGISTER_LAYER_CLASS(CrossEntropyLoss);