data_shard.h

#ifndef INCLUDE_UTILS_SHARD_H_
#define INCLUDE_UTILS_SHARD_H_

#include <google/protobuf/message.h>
#include <fstream>
#include <string>
#include <unordered_set>


using google::protobuf::Message;

namespace singa {

class DataShard {
 public:
  enum {
    kRead=0,
    kCreate=1,
    kAppend=2
  };

 public:
  DataShard(std::string folder, char mode, int capacity=104857600);
  ~DataShard();

  bool Next(std::string *key, Message* val);
  bool Next(std::string *key, std::string* val);

  bool Insert(const std::string& key, const Message& tuple);
  bool Insert(const std::string& key, const std::string& tuple);
  void SeekToFirst();
  void Flush() ;
  const int Count();
  const std::string path(){
    return path_;
  }

 protected:
  int Next(std::string *key);
  int PrepareForAppend(std::string path);
  bool PrepareNextField(int size);

 private:
  char mode_;
  std::string path_;
  // either ifstream or ofstream
  std::fstream fdat_;
  // to avoid replicated record
  std::unordered_set<std::string> keys_;
  // internal buffer
  char* buf_;
  // offset inside the buf_
  int offset_;
  // allocated bytes for the buf_
  int capacity_;
  // bytes in buf_, used in reading
  int bufsize_;
};
} /* singa */
#endif  // INCLUDE_UTILS_SHARD_H_