#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/highgui/highgui_c.h>
#include <opencv2/imgproc/imgproc.hpp>
#include <boost/lexical_cast.hpp>

#include <fstream>  // NOLINT(readability/streams)
#include <iostream>  // NOLINT(readability/streams)
#include <string>
#include <utility>
#include <vector>
#include <cstdlib>
#include <ctime>
#include <algorithm>

#include "caffe/data_layers.hpp"
#include "caffe/layer.hpp"
#include "caffe/util/io.hpp"
#include "caffe/util/math_functions.hpp"
#include "caffe/util/rng.hpp"
#include "caffe/util/Util.hpp"

namespace caffe {

template <typename Dtype>
ImageDataPairLayer<Dtype>::~ImageDataPairLayer<Dtype>() {
  this->JoinPrefetchThread();
}

template <typename Dtype>
void ImageDataPairLayer<Dtype>::getImgData(const int& id, const string& rootfolder, const string& filename,
		const int cropsize, const int channels, const int height,
		const int width, const bool crop_center, const bool mirror,
		Dtype* top_data, const Dtype* mean, const Dtype scale, const int size) {
	cv::Mat img = cv::imread(rootfolder + "/" + filename, CV_LOAD_IMAGE_COLOR);
	//LOG(INFO) << rootfolder + "/" + filename;
	//LOG(INFO) << img.rows << "," << img.cols;
	int h_off = 0, w_off = 0;
	int perturb=12;
	//wang:如果需要裁剪：
	if (cropsize) {


		// We only do random crop when we do training.
		//LOG(INFO) << height << "," << width << "," << cropsize_h << "," << cropsize_w;
		//LOG(INFO) << top_data;
		if (Caffe::phase() == Caffe::TRAIN && true/*!crop_center*/) {
			h_off = rand() %(height - cropsize_h);
			w_off = rand() % (width - cropsize_w);
			h_off = (height-cropsize_h)/2+perturb/2-rand()%perturb;
			w_off = (width-cropsize_w)/2+perturb/2-rand() % perturb;
			//LOG(INFO) << h_off << "," << w_off;
			//LOG(INFO) << "(" << (height-cropsize_h)/2+perturb/2 << "," << (height-cropsize_h)/2+perturb/2-perturb << "), (" << (width-cropsize_w)/2+perturb/2 << "," << (width-cropsize_w)/2+perturb/2-perturb << ")";
			//LOG(INFO) << rand() << "," << rand()%perturb;
			//h_off = (height - cropsize_h) / 2;
			//w_off = (width - cropsize_w) / 2;
		} else {
			h_off = (height - cropsize_h) / 2;
			w_off = (width - cropsize_w) / 2;
		}
		bool mirror2=false;
		//LOG(INFO) << "1";
		//wang:如果需要求镜像，复制镜像图片
		if (Caffe::phase() == Caffe::TRAIN&&mirror2 && rand() % 2) {
			// Copy mirrored version
			LOG(INFO) << "1";
			for (int c = 0; c < channels; ++c) {
				for (int h = 0; h < cropsize_h; ++h) {
					for (int w = 0; w < cropsize_w; ++w) {
						top_data[((id * channels + c) * cropsize_h + h) * cropsize_w
								+ cropsize_w - 1 - w] =128*scale-
								(static_cast<uint8_t>(img.at<cv::Vec3b>(
										h + h_off, w + w_off)[c])
										- mean[(c * height + h + h_off) * width
												+ w + w_off]) * scale;
						//if (w+w_off<0) {LOG(INFO) << "w+w_off:" << w+w_off << "," << (int)(img.at<cv::Vec3b>(h + h_off, w + w_off)[c]) << "," <<(int)(img.at<cv::Vec3b>(h + h_off, -w - w_off)[c]);}
        				        //LOG(INFO) << 128;
//						top_data[((id * channels + c) * cropsize + h) * cropsize
//								+ cropsize - 1 - w] = (img.data[(c * height + h
//								+ h_off) * width + w + w_off]
//								- mean[(c * height + h + h_off) * width + w
//										+ w_off]) * scale;
					}
				}
			}
			LOG(INFO) << "1";
		}

		else {
			// Normal copy
			//LOG(INFO) << "1";
			/*for (int i=0;i<3456000;i=i+100)
				{
					LOG(INFO) << i << "***";
					top_data[i]=i;
				}*/
			for (int c = 0; c < channels; ++c) {

				for (int h = 0; h < cropsize_h; ++h) {
					int w=cropsize_w-1;
					//LOG(INFO) << c << "," << h << "," << (c * height + h + h_off) * width + w + w_off << "," << mean[(c * height + h + h_off) * width + w + w_off];
					for (int w = 0; w < cropsize_w; ++w) {
						//LOG(INFO) << "(" << h+h_off << "," << w+w_off << "," << c << "):" << (int)(img.at<cv::Vec3b>(
								//h + h_off, w + w_off)[c]);
						top_data[((id * channels + c) * cropsize_h + h) * cropsize_w
														+ w] = 128*scale-(static_cast<uint8_t>(img.at<cv::Vec3b>(
																h + h_off, w + w_off)[c])) * scale;
						/***top_data[((id * channels + c) * cropsize_h + h) * cropsize_w
								+ w] = 128*scale-(static_cast<uint8_t>(img.at<cv::Vec3b>(
								h + h_off, w + w_off)[c])
								- mean[(c * height + h + h_off) * width + w
										+ w_off]) * scale;*/
						/*if (id>118) {
						LOG(INFO) << c << "," << h << "," << w;
						}*/
						//if (w+w_off<0) {LOG(INFO) << "w+w_off:" << w+w_off << "," << (int)(img.at<cv::Vec3b>(h + h_off, w + w_off)[c]) << "," <<(int)(img.at<cv::Vec3b>(h + h_off-1, w + w_off+width)[c]) << "," << (int)(img.at<cv::Vec3b>(h + h_off-1, w + w_off+width-1)[c]) << "," << (int)(img.at<cv::Vec3b>(h + h_off-1, w + w_off+width+1)[c]);}
        				        //LOG(INFO) << 128;
//						top_data[((id * channels + c) * cropsize + h) * cropsize
//								+ w] = (img.data[(c * height + h + h_off)
//								* width + w + w_off]
//								- mean[(c * height + h + h_off) * width + w
//										+ w_off]) * scale;
					}
				}
			}
			//LOG(INFO) << "1";
		}
		//LOG(INFO) << "img:" << (int)(img.at<cv::Vec3b>(1,1)[0]);height
	}
	//wang:如果不需要裁剪
	else {
		// we will prefer to use data() first, and then try float_data()
//		std::cout<<"img.data[j] access mode";
//		for (int j = 0; j < size; ++j) {
//			top_data[id * size + j] = (img.data[j] - mean[j]) * scale;
//		}
		for (int c = 0; c < channels; ++c) {
			for (int h = 0; h < img.rows; ++h) {
				for (int w = 0; w < img.cols; ++w) {
					top_data[((id * channels + c) * cropsize + h) * cropsize + w] =128*scale-
							(static_cast<uint8_t>(img.at<cv::Vec3b>(h + h_off,
									w + w_off)[c])
									- mean[(c * height + h + h_off) * width + w
											+ w_off]) * scale;
				}
			}
		}
	}
}



template <typename Dtype>
void ImageDataPairLayer<Dtype>::addIfNotExist(std::map<string, int>& name2id, const string& rootfolder,
		const vector<string>& filenames, vector<int>& ids, const int cropsize,
		const int channels, const int height, const int width,
		const bool crop_center, const bool mirror, Dtype* top_data,
		const Dtype* mean, const Dtype scale, const int size) {
	for (int file_id = 0; file_id < filenames.size(); file_id++) {
		std::map<string, int>::iterator name_iter = name2id.find(
				filenames[file_id]);
		if (name_iter == name2id.end()) {
			//printf("\nbegin to load:%s%s",filenames[file_id].c_str() , ".jpg");
			//LOG(INFO)<<("\nbegin toload");
			getImgData(name2id.size(), rootfolder, filenames[file_id] + ".jpg",
					cropsize, channels, height, width, crop_center, mirror,
					top_data, mean, scale, size);
			//LOG(INFO)<<("\n load suc");
			name2id.insert(name_iter,
					std::make_pair(filenames[file_id], name2id.size()));
			ids[file_id] = name2id.size()-1;
		} else {
			ids[file_id] = name_iter->second;
		}
	}
}

//wang:最重要的预取了。
template <typename Dtype>
void* ImageDataPairLayer<Dtype>::DataLayerPrefetch(void* layer_pointer) {

	CHECK(layer_pointer);
	ImageDataPairLayer<Dtype>* layer = reinterpret_cast<ImageDataPairLayer<Dtype>*>(layer_pointer);
	CHECK(layer);
	//wang:如果批量iter没完，接就减1；
	if (Caffe::phase() == Caffe::TRAIN && layer->batch_iter_ != 0) {
//LOG(INFO) << "Reusing previous batch";
		layer->batch_iter_--;
		return (void*) NULL;
	}
//LOG(INFO) << "Generating new batch";
	layer->batch_iter_ = layer->layer_param().image_data_pair_param().batch_iter() - 1;
	//LOG(INFO) << &(this->prefetch_data_);
	Datum datum;
	//CHECK(layer->prefetch_data_);
	//wang:从cpu取数据。
	Dtype* top_data = layer->prefetch_data_.mutable_cpu_data();
	//LOG(INFO) << top_data;
	/*for (int i=0;i<layer->prefetch_data_.count();i=i+100)
	{
		LOG(INFO) << i;
		top_data[i]=i;
	}*/
	std::vector<double>& top_label = layer->prefetch_labels_;
	const Dtype scale = layer->layer_param().image_data_pair_param().scale();
	const int batchsize = layer->layer_param().image_data_pair_param().batchsize();
	const int cropsize = layer->layer_param().image_data_pair_param().cropsize();
	const bool mirror = layer->layer_param().image_data_pair_param().mirror();
	const int class_per_iter = layer->layer_param().image_data_pair_param().class_per_iter();
	const int triplet_per_class = layer->layer_param().image_data_pair_param().triplet_per_class();
	const int img_counts_per_class_per_iter =
			layer->layer_param().image_data_pair_param().img_counts_per_class_per_iter();
	//for (int i=0;i<100000;i++){
	//LOG(INFO) << batchsize << "," << cropsize << "," << class_per_iter << "," << triplet_per_class << "," << img_counts_per_class_per_iter;
	//}
	if (mirror && cropsize == 0) {
		LOG(FATAL)<< "Current implementation requires mirror and cropsize to be "
		<< "set at the same time.";
	}
	// datum scales
	const int channels = layer->datum_channels();
	const int height = layer->datum_height();
	const int width = layer->datum_width();
	const int size = layer->datum_size();
	const Dtype* mean = layer->data_mean_.cpu_data();

//	const std::map<string, int>& class2id = layer->class2id_;
	// 测试的时候不需要生成triplet

	//LOG(INFO) << Caffe::phase() << "," <<Caffe::TEST << "," << Caffe::TRAIN;
	if (Caffe::phase() == Caffe::TEST) {
		//LOG(INFO) << "TEST";
		//LOG(INFO) << batchsize << "," << layer->curIndex << "," << layer->filenames_.size();
		//layer->prefetch_data_.Reshape(batchsize, 3, cropsize_h, cropsize_w);
		//top_data = layer->prefetch_data_.mutable_cpu_data();
		for (int i = 0;
				i < batchsize && layer->curIndex < layer->filenames_.size();
				i++, layer->curIndex++) {
			LOG(INFO) << i << "," << layer->curIndex << "," << layer->filenames_[layer->curIndex];
			getImgData(i, layer->layer_param().image_data_pair_param().source(),
					layer->filenames_[layer->curIndex], cropsize, channels,
					height, width, layer->layer_param().image_data_pair_param().crop_center(), mirror,
					top_data, mean, scale, size);
			string line=layer->filenames_[layer->curIndex];
			LOG(INFO) << "line:" << line;
			string class_name_tmp = line.substr(0, line.find_last_of('_'));
			LOG(INFO) << "class_name_tmp:" << class_name_tmp;
			std::map<std::string, int>::iterator name_tmp;
			name_tmp = class2id_.find(class_name_tmp);
			LOG(INFO) << "name_tmp:" << name_tmp->first << "," << name_tmp->second;
			top_label[i] = name_tmp->second;
		}
		return (void*) NULL;
	}

	const vector<string>& class_names = layer->class_names_;
	const vector<int>& img_counts_per_class = layer->img_counts_per_class_;
	vector<vector<string> >& triplets = prefetch_triplets_;
	vector<double> & labels = prefetch_labels_;
	vector<vector<int> >& triplets_id = prefetch_triplets_id_;
	std::map<string, int>& name2id = prefetch_name2id_;
	name2id.clear();
	//wang:先选出class_per_iter那么img_counts_per_class_per_iter多类,假设每次迭代40个类
	//wang:class_names是什么？
	vector<int> candidate_classes(class_names.size(), 0);
	//wang:class_names所有类的类名组成的字符串数组。
	for (int i = 0; i < candidate_classes.size(); i++) {
		candidate_classes[i] = i;
	}
	//wang:把所有的类转成1，2，3，类似于我们的labels
	std::srand(unsigned(std::time(0)));
//	std::random_shuffle(selected_index.begin(), selected_index.end());
	random_unique(candidate_classes.begin(), candidate_classes.end(),
			MIN(candidate_classes.size(), class_per_iter));
	// 接着每类随机选img_counts_per_class_per_iter张图片
	vector<vector<int> > candidate_imgs(class_per_iter, vector<int>());
	//wang:候选图片为每次迭代需要的类数，和每类的哪几张。

	for (int i = 0; i < class_per_iter; i++) {
		vector<int> tmp_indexs(
				img_counts_per_class[candidate_classes[i
						% candidate_classes.size()]], 0);
		//创建一个这样tmp_indexs,它只有一行，元素个数：i,i是当前迭代中，循环到第i类
		//比如现在是第35类，i=35
		//wang:  i<=40; trainlabel(i%316)
		for (int j = 0; j < tmp_indexs.size(); j++) {
			tmp_indexs[j] = j;
		}
		//wang:tmp_indexs初始化，
		random_unique(tmp_indexs.begin(), tmp_indexs.end(),
				MIN(tmp_indexs.size(), img_counts_per_class_per_iter));
		for (int j = 0; j < img_counts_per_class_per_iter; j++) {
			candidate_imgs[i].push_back( //candidate_imgs代表40类（行），每类若干张（列）
					tmp_indexs[j % tmp_indexs.size()] + layer->beg_index_);
		}
		//wang:假设每次迭代40类，每类80张图，现在是i=35类，j=78张图，j%35，
		//反正这里是为了找到第i类有哪几张图
	}
	//LOG(INFO) << "*******";
	//LOG(INFO)<<"beg_index:"<<layer->beg_index_;
	//LOG(INFO) << layer->prefetch_data_.count();
	memset(top_data, 0, sizeof(Dtype) * layer->prefetch_data_.count());
	//LOG(INFO) << "*******";
	//wang:这里不懂，先忽略
	for (int index_i = 0; index_i < class_per_iter; index_i++) {
		//LOG(INFO) << index_i;
		int class_index = candidate_classes[index_i % candidate_classes.size()];
		int img_counts = img_counts_per_class[class_index];
		if(img_counts==1){
			LOG(INFO)<<"only one image,skip";
			//index_i-=1;
                        continue;
		}
		//wang:现在处于某一次迭代的40个类的index_i类时刻

		for (int triplet_i = 0; triplet_i < triplet_per_class; triplet_i++) {
			//LOG(INFO) << triplet_i;
//			int o1_index = rand() % img_counts + layer->beg_index_;
//			int o2_index = rand() % img_counts + layer->beg_index_;
			int rand1=rand()% candidate_imgs[index_i].size();
			//wang：从本次的第index_i类中随机选一张
			int rand2=rand()% candidate_imgs[index_i].size();
			//再从本次的第index_i类中随机选一张
			if(rand2==rand1){
				rand2=(rand1+1)%candidate_imgs[index_i].size();
			}
			//wang:避免选到同一张
			int o1_index = candidate_imgs[index_i][rand1];
			int o2_index = candidate_imgs[index_i][rand2];

//			int o3_class_index = rand() % class_names.size();
//			if (o3_class_index == class_index) {
//				o3_class_index = (o3_class_index + 1) % class_names.size();
//			}


			//wang:随机选一个类，感觉% class_per_iter无意义。
			int o3_tmp_index = (rand() % class_per_iter)
					% candidate_classes.size();
			int o3_class_index = candidate_classes[o3_tmp_index];
			//wang:o3_tmp_index是本次
			//wang:避免是和前面是同一个类
			if (o3_class_index == class_index) {
				o3_tmp_index = ((o3_tmp_index + 1) % class_per_iter)
						% candidate_classes.size();
				o3_class_index = candidate_classes[o3_tmp_index];
			}

//			int o3_index = rand() % img_counts_per_class[o3_class_index] + 1;
			int o3_index = candidate_imgs[o3_tmp_index][rand()
					% candidate_imgs[o3_tmp_index].size()];
			//wang:从该类中随机选一张图
			string o1 = class_names[class_index] + "_"
					+ boost::lexical_cast<string>(o1_index);
			string o2 = class_names[class_index] + "_"
					+ boost::lexical_cast<string>(o2_index);
			string o3 = class_names[o3_class_index] + "_"
					+ boost::lexical_cast<string>(o3_index);
		//LOG(INFO)<< o1 << ", " << o2 << ", " << o3;
			//LOG(INFO)<<"class index:"<<class_index<<","<<o3_class_index;
			//LOG(INFO)<<"class size:"<<candidate_imgs[index_i].size()<<","<<candidate_imgs[o3_tmp_index].size();

			triplets[index_i * triplet_per_class + triplet_i][0] = o1;
			//wang:现在是第index_i个类,每个类有triplet_per_class,现在是
			//第



			if (rand()%2==1)
			{

				triplets[index_i * triplet_per_class + triplet_i][1] = o2;

				labels[index_i * triplet_per_class + triplet_i] = 1.0;

			}else
			{

				triplets[index_i * triplet_per_class + triplet_i][1] = o3;

				labels[index_i * triplet_per_class + triplet_i] = 0;//-1.0;


			}




			//LOG(INFO)<<("\nbegin to load triplet data");
			addIfNotExist(name2id, layer->layer_param().image_data_pair_param().source(),
					triplets[index_i * triplet_per_class + triplet_i],
					triplets_id[index_i * triplet_per_class + triplet_i],
					cropsize, channels, height, width,
					layer->layer_param().image_data_pair_param().crop_center(), mirror, top_data, mean,
					scale, size);

			//LOG(INFO)<<("\no problem");
		}
	}

	return (void*) NULL;
}

template <typename Dtype>
void ImageDataPairLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
      vector<Blob<Dtype>*>* top) {
	CHECK_EQ(bottom.size(), 0)<< "Data Layer takes no input blobs.";
//	CHECK_EQ(top->size(), 2) << "Data Layer takes two blobs as output.";
	CHECK_EQ(top->size(), 4) << "Image Data Pair Layer takes four blobs as output.";

	CHECK(this->layer_param_.image_data_pair_param().has_img_files()) << "the file contain all the "
	<< "image names should be specified via attribute 'img_files' in prototxt";
	LOG(INFO) << &(this->prefetch_data_);
	int batchsize = this->layer_param_.image_data_pair_param().batchsize();
	std::ifstream img_files_if(this->layer_param_.image_data_pair_param().img_files().c_str());
	CHECK(img_files_if) << "Failed to open " << this->layer_param_.image_data_pair_param().img_files();
	string line;
	counts_ = 0;
	std::map<std::string, int>::iterator name_iter_tmp;
	int name_id_tmp;
	class2id_.clear();
	class_names_.clear();
	img_counts_per_class_.clear();
	LOG(INFO) << "phaze:" << Caffe::phase();
	while(img_files_if >> line) {
		//LOG(INFO) << "see next";
		//LOG(INFO) << line;
		if (line.empty()) {
			continue;
		}
		counts_++;
		//LOG(INFO) << Caffe::phase() << "," <<Caffe::TEST << "," << Caffe::TRAIN;
		if (Caffe::phase() == Caffe::TEST) {
			counts_--;
			// 根据id的上下限筛选图片
			int index1 = line.find_last_of('_') + 1;
			int index2 = line.find_last_of('.');
			//LOG(INFO) << line;
			//LOG(INFO) << this->layer_param_.image_data_pair_param().has_id_upper_bound() << "," << this->layer_param_.image_data_pair_param().has_id_lower_bound();
			int id = atoi(line.substr(index1, index2 - index1).c_str());
			if (this->layer_param_.image_data_pair_param().has_id_upper_bound() && id > this->layer_param_.image_data_pair_param().id_upper_bound()) continue;
			if (this->layer_param_.image_data_pair_param().has_id_lower_bound() && id < this->layer_param_.image_data_pair_param().id_lower_bound()) continue;

			counts_++;
			filenames_.push_back(line);


		}

		string class_name_tmp = line.substr(0, line.find_last_of('_'));
		name_iter_tmp = class2id_.find(class_name_tmp);
		if (name_iter_tmp == class2id_.end()) {
			name_id_tmp = class2id_.size();
			class2id_.insert(name_iter_tmp, std::make_pair(class_name_tmp, name_id_tmp));
			class_names_.push_back(class_name_tmp);
			img_counts_per_class_.push_back(1);
		} else {
			name_id_tmp = name_iter_tmp->second;
			img_counts_per_class_[name_id_tmp]++;
		}
	}
	img_files_if.close();

	int total_img_count = counts_;
	if (Caffe::phase() == Caffe::TRAIN) {
		batch_iter_ = 0;
		beg_index_ = 1;

		// 限制选择图片的数量
		if (this->layer_param_.image_data_pair_param().has_id_upper_bound()) {
			total_img_count = 0;
			for (int i = 0; i < img_counts_per_class_.size(); i++) {
				img_counts_per_class_[i] = MIN(this->layer_param_.image_data_pair_param().id_upper_bound(), img_counts_per_class_[i]);
				total_img_count += img_counts_per_class_[i];
			}
		}

		if (this->layer_param_.image_data_pair_param().has_id_lower_bound()) {
			beg_index_ = this->layer_param_.image_data_pair_param().id_lower_bound();
			total_img_count = 0;
			for (int i = 0; i < img_counts_per_class_.size(); i++) {
				img_counts_per_class_[i] = img_counts_per_class_[i] - beg_index_ + 1;
				CHECK_GT(img_counts_per_class_[i], 0);
				total_img_count += img_counts_per_class_[i];
				LOG(INFO) << class_names_[i] << ", img counts: " << img_counts_per_class_[i]
				<< "from " << beg_index_ << " to " << (beg_index_ + img_counts_per_class_[i])-1;
			}
		}

	} else {
		curIndex = 0;
	}
	LOG(INFO) << &(this->prefetch_data_);
	cv::Mat img;
	if (Caffe::phase() == Caffe::TRAIN) {
		img = cv::imread(
				this->layer_param_.image_data_pair_param().source() + "/" + class_names_[0] + "_1.jpg",
				CV_LOAD_IMAGE_COLOR);
			LOG(INFO)<<"begin to load img"<<this->layer_param_.image_data_pair_param().source() + "/" + class_names_[0] + "_1.jpg";
	} else {
		img = cv::imread(
				this->layer_param_.image_data_pair_param().source() + "/" + filenames_[0],
				CV_LOAD_IMAGE_COLOR);
	}
// image
	int cropsize = this->layer_param_.image_data_pair_param().cropsize();
	if(this->layer_param_.image_data_pair_param().has_cropsize_h()){
	cropsize_h=this->layer_param_.image_data_pair_param().cropsize_h();
	}
	else
		cropsize_h=cropsize;
	if(this->layer_param_.image_data_pair_param().has_cropsize_w()){
	cropsize_w=this->layer_param_.image_data_pair_param().cropsize_w();
	}
	else
		cropsize_w=cropsize;

//wang:到这里了
// 测试的时候，不需要生成triplet，只需要计算所有图片的特征
	LOG(INFO) << "phaze:" << Caffe::phase();

	if (Caffe::phase() == Caffe::TRAIN) {
		batchsize = this->layer_param_.image_data_pair_param().class_per_iter() * this->layer_param_.image_data_pair_param().triplet_per_class();
//		Caffe::mutable_triplets() = vector<vector<string> >(batchsize, vector<string>(3, string()));
//		Caffe::mutable_triplets_id() = vector<vector<int> >(batchsize, vector<int>(3, 0));

		/***Caffe::mutable_prefetch_triplets() = vector<vector<string> >(batchsize, vector<string>(2, string()));
		Caffe::mutable_prefetch_labels() = vector<double>(batchsize);
		Caffe::mutable_prefetch_triplets_id() = vector<vector<int> >(batchsize, vector<int>(2, 0));*/
		prefetch_triplets_ = vector<vector<string> >(batchsize, vector<string>(2, string()));
			prefetch_labels_ = vector<double>(batchsize);
			prefetch_triplets_id_ = vector<vector<int> >(batchsize, vector<int>(2, 0));

		batchsize *= 3;
		LOG(INFO) << "total_img_count: " << total_img_count
		<< ", batchsize: " << batchsize;
		batchsize = MIN(total_img_count, batchsize);
		if (this->layer_param_.image_data_pair_param().has_id_upper_bound()) {
			int img_counts_bound = this->layer_param_.image_data_pair_param().id_upper_bound() * this->layer_param_.image_data_pair_param().class_per_iter();
			if (this->layer_param_.image_data_pair_param().has_id_lower_bound()) {
				img_counts_bound -= this->layer_param_.image_data_pair_param().id_lower_bound() * this->layer_param_.image_data_pair_param().class_per_iter();
				img_counts_bound+= this->layer_param_.image_data_pair_param().class_per_iter();
			}
			LOG(INFO) << "img_counst_bound: " << img_counts_bound;
			batchsize = MIN(img_counts_bound, batchsize);
		}
		img_counts_per_class_per_iter_ = this->layer_param_.image_data_pair_param().img_counts_per_class_per_iter();
		if (img_counts_per_class_per_iter_ == 0) {
			img_counts_per_class_per_iter_ = std::numeric_limits<int>::max();
		} else {
			batchsize = MIN(img_counts_per_class_per_iter_ * this->layer_param_.image_data_pair_param().class_per_iter(), batchsize);
		}
	}
	LOG(INFO) << &(this->prefetch_data_);
	LOG(INFO) << "Max Batchsize: " << batchsize;
	const int class_per_iter = this->layer_param_.image_data_pair_param().class_per_iter();
		const int triplet_per_class = this->layer_param_.image_data_pair_param().triplet_per_class();
	if (cropsize > 0) {
		(*top)[0]->Reshape(
				batchsize, 3, cropsize_h, cropsize_w);
		this->prefetch_data_.Reshape(batchsize, 3, cropsize_h, cropsize_w);
	} else {
		(*top)[0]->Reshape(
				batchsize, 3, img.rows,
				img.cols);
		this->prefetch_data_.Reshape(batchsize, 3, img.rows, img.cols);
	}
	this->prefetch_label_.Reshape(batchsize, 3, 1, 1);
	(*top)[1]->Reshape(class_per_iter*triplet_per_class, 1, 1, 1);  //pair1
	(*top)[2]->Reshape(class_per_iter*triplet_per_class, 1, 1, 1);  //pair2
	(*top)[3]->Reshape(class_per_iter*triplet_per_class, 1, 1, 1);  //pair label
	LOG(INFO) << "output data size: " << (*top)[0]->num() << ","
	<< (*top)[0]->channels() << "," << (*top)[0]->height() << ","
	<< (*top)[0]->width();

	this->datum_channels_ = 3;
	this->datum_height_ = img.rows;
	this->datum_width_ = img.cols;
	this->datum_size_ = 3 * img.rows * img.cols;
	CHECK_GE(this->datum_height_, cropsize);
	CHECK_GE(this->datum_width_, cropsize);
// check if we want to have mean
	if (this->layer_param_.image_data_pair_param().has_meanfile()) {
		BlobProto blob_proto;
		LOG(INFO)<< "Loading mean file from" << this->layer_param_.image_data_pair_param().meanfile();
		ReadProtoFromBinaryFile(this->layer_param_.image_data_pair_param().meanfile().c_str(), &blob_proto);
		data_mean_.FromProto(blob_proto);
		CHECK_EQ(data_mean_.num(), 1);
		CHECK_EQ(data_mean_.channels(), this->datum_channels_);
		CHECK_EQ(data_mean_.height(), this->datum_height_);
		CHECK_EQ(data_mean_.width(), this->datum_width_);
	} else {
// Simply initialize an all-empty mean.
		data_mean_.Reshape(1, this->datum_channels_, this->datum_height_, this->datum_width_);
	}
// Now, start the prefetch thread. Before calling prefetch, we make two
// cpu_data calls so that the prefetch thread does not accidentally make
// simultaneous cudaMalloc calls when the main thread is running. In some
// GPUs this seems to cause failures if we do not so.
	this->prefetch_data_.mutable_cpu_data();
	LOG(INFO) << &(this->prefetch_data_);
//	prefetch_label_->mutable_cpu_data();
	data_mean_.cpu_data();
	LOG(INFO) << "Initializing prefetch";
	LOG(INFO) << &(this->prefetch_data_);
	/***CHECK(!pthread_create(&thread_, NULL, ImageDataPairLayer<Dtype>::DataLayerPrefetch,
					reinterpret_cast<void*>(this))) << "Pthread execution failed.";*/
	DataLayerPrefetch(this);
	LOG(INFO) << "Prefetch initialized.";
	LOG(INFO) << &(this->prefetch_data_);
// output info
//    LOG(INFO) << "Total class: " << class_names_.size() << ", Image counts per class as follow: ";
//    for (int i = 0; i < img_counts_per_class_.size(); i++) {
//        LOG(INFO) << class_names_[i] << ": " << img_counts_per_class_[i];
//	}
}


template<typename Dtype>
void ImageDataPairLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
		vector<Blob<Dtype>*>* top) {
	//CHECK(!pthread_join(thread_, NULL)) << "Pthread joining failed.";

// Copy the data
	DataLayerPrefetch(this);
	memcpy((*top)[0]->mutable_cpu_data(), this->prefetch_data_.cpu_data(),
			sizeof(Dtype) * this->prefetch_data_.count());
//	memcpy((*top)[1]->mutable_cpu_data(), prefetch_label_->cpu_data(),
//			sizeof(Dtype) * prefetch_label_->count());

	/***Caffe::mutable_triplets() = Caffe::mutable_prefetch_triplets();
	Caffe::mutable_labels() = Caffe::mutable_prefetch_labels();
	Caffe::mutable_triplets_id() = Caffe::mutable_prefetch_triplets_id();
	Caffe::mutable_name2id() = Caffe::mutable_prefetch_name2id();*/

//	if (Caffe::phase() == Caffe::TEST) {
// zhu
//	filenames_ = *prefetch_filenames_;
//	offset_ = *prefetch_offset_;
//	memcpy(labels_->mutable_cpu_data(), prefetch_label_->cpu_data(),
//			sizeof(Dtype) * prefetch_label_->count());
//	}

// Start a new prefetch thread
	/***CHECK(!pthread_create(&thread_, NULL, ImageDataPairLayer<Dtype>::DataLayerPrefetch,
					reinterpret_cast<void*>(this)))
															<< "Pthread execution failed.";*/
	vector<vector<int> >& triplets_id = prefetch_triplets_id_;
	vector<double> & labels = prefetch_labels_;
	const int class_per_iter = this->layer_param_.image_data_pair_param().class_per_iter();
			const int triplet_per_class = this->layer_param_.image_data_pair_param().triplet_per_class();
			LOG(INFO) << class_per_iter*triplet_per_class << "," << triplets_id.size();
			Dtype* top_id1 =(*top)[1]->mutable_cpu_data();
			Dtype* top_id2 =(*top)[2]->mutable_cpu_data();
			Dtype* top_label =(*top)[3]->mutable_cpu_data();
	for (int triplet_i = 0; triplet_i < triplets_id.size(); triplet_i++) {
		top_id1[triplet_i]=triplets_id[triplet_i][0];
		top_id2[triplet_i]=triplets_id[triplet_i][1];
		top_label[triplet_i]=labels[triplet_i];
	}
}

template<typename Dtype>
void ImageDataPairLayer<Dtype>::Forward_gpu(const vector<Blob<Dtype>*>& bottom,
		vector<Blob<Dtype>*>* top) {

// First, join the thread
	//CHECK(!pthread_join(thread_, NULL)) << "Pthread joining failed.";

	 DataLayerPrefetch(this);
// Copy the data
	CUDA_CHECK(
			cudaMemcpy((*top)[0]->mutable_gpu_data(),
					this->prefetch_data_.cpu_data(),
					sizeof(Dtype) * this->prefetch_data_.count(),
					cudaMemcpyHostToDevice));
//	CUDA_CHECK(
//			cudaMemcpy((*top)[1]->mutable_gpu_data(),
//					prefetch_label_->cpu_data(),
//					sizeof(Dtype) * prefetch_label_->count(),
//					cudaMemcpyHostToDevice));


	/***Caffe::mutable_triplets() = Caffe::mutable_prefetch_triplets();

	Caffe::mutable_labels() = Caffe::mutable_prefetch_labels();

	Caffe::mutable_triplets_id() = Caffe::mutable_prefetch_triplets_id();

	Caffe::mutable_name2id() = Caffe::mutable_prefetch_name2id();*/

// zhu
//	if (Caffe::phase() == Caffe::TEST) {
//	filenames_ = *prefetch_filenames_;
//	offset_ = *prefetch_offset_;
//	CUDA_CHECK(
//			cudaMemcpy(labels_->mutable_gpu_data(), prefetch_label_->cpu_data(),
//					sizeof(Dtype) * prefetch_label_->count(),
//					cudaMemcpyHostToDevice));
//	}

	/***CHECK(!pthread_create(&thread_, NULL, ImageDataPairLayer<Dtype>::DataLayerPrefetch,
					reinterpret_cast<void*>(this)))
												<< "Pthread execution failed.";*/
	vector<vector<int> >& triplets_id = prefetch_triplets_id_;
	vector<double> & labels = prefetch_labels_;
	const int class_per_iter = this->layer_param_.image_data_pair_param().class_per_iter();
			const int triplet_per_class = this->layer_param_.image_data_pair_param().triplet_per_class();
			//LOG(INFO) << class_per_iter*triplet_per_class << "," << triplets_id.size();
			CHECK_EQ(class_per_iter*triplet_per_class,triplets_id.size()) << "triplet_id.size";
			Dtype* top_id1 =(*top)[1]->mutable_cpu_data();
			Dtype* top_id2 =(*top)[2]->mutable_cpu_data();
			Dtype* top_label =(*top)[3]->mutable_cpu_data();
	for (int triplet_i = 0; triplet_i < triplets_id.size(); triplet_i++) {
		top_id1[triplet_i]=triplets_id[triplet_i][0];
		top_id2[triplet_i]=triplets_id[triplet_i][1];
		top_label[triplet_i]=labels[triplet_i];
	}
    //LOG(INFO) << "triplet id:" << triplets_id.size();
}

template<typename Dtype>
void ImageDataPairLayer<Dtype>::Backward_cpu(const vector<Blob<Dtype>*>& top,
		const vector<bool>& propagate_down, vector<Blob<Dtype>*>* bottom) {
	//return Dtype(0.);
}

template<typename Dtype>
void ImageDataPairLayer<Dtype>::Backward_gpu(const vector<Blob<Dtype>*>& top,
		const vector<bool>& propagate_down, vector<Blob<Dtype>*>* bottom) {
	//return Dtype(0.);
}

INSTANTIATE_CLASS(ImageDataPairLayer);

}  // namespace caffe
