json.h

Go to the documentation of this file.

#ifndef DMLC_JSON_H_
#define DMLC_JSON_H_

// This code requires C++11 to compile
#include <vector>
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
#include <iostream>
#include <sstream>
#endif
#include <cctype>
#include <string>
#include <algorithm>
#include <map>
#include <list>
#include <utility>

#include "./base.h"
#include "./logging.h"
#include "./type_traits.h"

#if DMLC_USE_CXX11
#include <typeindex>
#include <typeinfo>
#include <unordered_map>
#if DMLC_STRICT_CXX11
#if DMLC_ENABLE_RTTI
#include "./any.h"
#endif  // DMLC_ENABLE_RTTI
#endif  // DMLC_STRICT_CXX11
#endif  // DMLC_USE_CXX11

namespace dmlc {
class JSONReader {
 public:
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
  explicit JSONReader(std::istream *is)
#else
  explicit JSONReader(std::string *is)
#endif
      : is_(is),
        line_count_r_(0),
        line_count_n_(0) {}
  inline void ReadString(std::string *out_str);
  template<typename ValueType>
  inline void ReadNumber(ValueType *out_value);
  inline void BeginObject();
  inline void BeginArray();
  inline bool NextObjectItem(std::string *out_key);
  inline bool NextArrayItem();
  template<typename ValueType>
  inline void Read(ValueType *out_value);

  inline std::string line_info() const {
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
    char temp[64];
    std::ostringstream os;
    os << " Line " << std::max(line_count_r_, line_count_n_);
    is_->getline(temp, 64);
    os << ", around ^`" << temp << "`";
    return os.str();
#else
    std::string info = " Line ";
    info += std::to_string(std::max(line_count_r_, line_count_n_));

    // string getline
    size_t end_pos = is_->find('\n');
    end_pos = std::min((size_t)64,
        end_pos == std::string::npos ? is_->size() : end_pos);
    std::string line = is_->substr(0, end_pos);
    is_->erase(0, line.size() + 1);  // +1 for \n

    info += ", around ^`" + line + "`";
    return info;
#endif
  }

 private:
#ifndef _LIBCPP_SGX_NO_IOSTREAMS

  std::istream *is_;
#else

  std::string *is_;
#endif

  size_t line_count_r_;
  size_t line_count_n_;
  std::vector<size_t> scope_counter_;
  inline int NextNonSpace();
  inline int PeekNextNonSpace();
  inline int NextChar();
  inline int PeekNextChar();
};

class JSONWriter {
 public:
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
  explicit JSONWriter(std::ostream *os)
#else
  explicit JSONWriter(std::string *os)
#endif
      : os_(os) {}
  inline void WriteNoEscape(const std::string &s);
  inline void WriteString(const std::string &s);
  template<typename ValueType>
  inline void WriteNumber(const ValueType &v);
  inline void BeginArray(bool multi_line = true);
  inline void EndArray();
  inline void BeginObject(bool multi_line = true);
  inline void EndObject();
  template<typename ValueType>
  inline void WriteObjectKeyValue(const std::string &key,
                                  const ValueType &value);
  inline void WriteArraySeperator();
  template<typename ValueType>
  inline void WriteArrayItem(const ValueType &value);
  template<typename ValueType>
  inline void Write(const ValueType &value);

 private:
#ifndef _LIBCPP_SGX_NO_IOSTREAMS

  std::ostream *os_;
#else
  std::string *os_;
#endif

  std::vector<size_t> scope_counter_;
  std::vector<bool> scope_multi_line_;
  inline void WriteSeperator();
};

class JSONObjectReadHelper {
 public:
  template<typename T>
  inline void DeclareField(const std::string &key, T *addr) {
    DeclareFieldInternal(key, addr, false);
  }
  template<typename T>
  inline void DeclareOptionalField(const std::string &key, T *addr) {
    DeclareFieldInternal(key, addr, true);
  }
  inline void ReadAllFields(JSONReader *reader);

 private:
  template<typename T>
  inline void DeclareFieldInternal(const std::string &key, T *addr, bool optional);
  template<typename T>
  inline static void ReaderFunction(JSONReader *reader, void *addr);
  typedef void (*ReadFunction)(JSONReader *reader, void *addr);
  struct Entry {
    ReadFunction func;
    void *addr;
    bool optional;
  };
  std::map<std::string, Entry> map_;
};

#define DMLC_JSON_ENABLE_ANY_VAR_DEF(KeyName)                  \
  static DMLC_ATTRIBUTE_UNUSED ::dmlc::json::AnyJSONManager&   \
  __make_AnyJSONType ## _ ## KeyName ## __

#define DMLC_JSON_ENABLE_ANY(Type, KeyName)                             \
  DMLC_STR_CONCAT(DMLC_JSON_ENABLE_ANY_VAR_DEF(KeyName), __COUNTER__) = \
    ::dmlc::json::AnyJSONManager::Global()->EnableType<Type>(#KeyName) \

namespace json {

template<typename T>
struct Handler;

template<typename ValueType>
struct NumericHandler {
  inline static void Write(JSONWriter *writer, const ValueType &value) {
    writer->WriteNumber<ValueType>(value);
  }
  inline static void Read(JSONReader *reader, ValueType *value) {
    reader->ReadNumber<ValueType>(value);
  }
};

template<typename ContainerType>
struct ArrayHandler {
  inline static void Write(JSONWriter *writer, const ContainerType &array) {
    typedef typename ContainerType::value_type ElemType;
    writer->BeginArray(array.size() > 10 || !dmlc::is_pod<ElemType>::value);
    for (typename ContainerType::const_iterator it = array.begin();
         it != array.end(); ++it) {
      writer->WriteArrayItem(*it);
    }
    writer->EndArray();
  }
  inline static void Read(JSONReader *reader, ContainerType *array) {
    typedef typename ContainerType::value_type ElemType;
    array->clear();
    reader->BeginArray();
    while (reader->NextArrayItem()) {
      ElemType value;
      Handler<ElemType>::Read(reader, &value);
      array->insert(array->end(), value);
    }
  }
};

template<typename ContainerType>
struct MapHandler{
  inline static void Write(JSONWriter *writer, const ContainerType &map) {
    writer->BeginObject(map.size() > 1);
    for (typename ContainerType::const_iterator it = map.begin(); it != map.end(); ++it) {
      writer->WriteObjectKeyValue(it->first, it->second);
    }
    writer->EndObject();
  }
  inline static void Read(JSONReader *reader, ContainerType *map) {
    typedef typename ContainerType::mapped_type ElemType;
    map->clear();
    reader->BeginObject();
    std::string key;
    while (reader->NextObjectItem(&key)) {
      ElemType value;
      reader->Read(&value);
      (*map)[key] = value;
    }
  }
};

template<typename T>
struct CommonJSONSerializer {
  inline static void Write(JSONWriter *writer, const T &value) {
    value.Save(writer);
  }
  inline static void Read(JSONReader *reader, T *value) {
    value->Load(reader);
  }
};

template<>
struct Handler<std::string> {
  inline static void Write(JSONWriter *writer, const std::string &value) {
    writer->WriteString(value);
  }
  inline static void Read(JSONReader *reader, std::string *str) {
    reader->ReadString(str);
  }
};

template<typename T>
struct Handler<std::vector<T> > : public ArrayHandler<std::vector<T> > {
};

template<typename K, typename V>
struct Handler<std::pair<K, V> > {
  inline static void Write(JSONWriter *writer, const std::pair<K, V> &kv) {
    writer->BeginArray();
    writer->WriteArrayItem(kv.first);
    writer->WriteArrayItem(kv.second);
    writer->EndArray();
  }
  inline static void Read(JSONReader *reader, std::pair<K, V> *kv) {
    reader->BeginArray();
    CHECK(reader->NextArrayItem())
        << "Expect array of length 2";
    Handler<K>::Read(reader, &(kv->first));
    CHECK(reader->NextArrayItem())
        << "Expect array of length 2";
    Handler<V>::Read(reader, &(kv->second));
    CHECK(!reader->NextArrayItem())
        << "Expect array of length 2";
  }
};

template<typename T>
struct Handler<std::list<T> > : public ArrayHandler<std::list<T> > {
};

template<typename V>
struct Handler<std::map<std::string, V> > : public MapHandler<std::map<std::string, V> > {
};

#if DMLC_USE_CXX11
template<typename V>
struct Handler<std::unordered_map<std::string, V> >
    : public MapHandler<std::unordered_map<std::string, V> > {
};
#endif  // DMLC_USE_CXX11

template<typename T>
struct Handler {
  inline static void Write(JSONWriter *writer, const T &data) {
    typedef typename dmlc::IfThenElseType<dmlc::is_arithmetic<T>::value,
                                          NumericHandler<T>,
                                          CommonJSONSerializer<T> >::Type THandler;
    THandler::Write(writer, data);
  }
  inline static void Read(JSONReader *reader, T *data) {
    typedef typename dmlc::IfThenElseType<dmlc::is_arithmetic<T>::value,
                                          NumericHandler<T>,
                                          CommonJSONSerializer<T> >::Type THandler;
    THandler::Read(reader, data);
  }
};

#if DMLC_STRICT_CXX11
#if DMLC_ENABLE_RTTI
// Manager to store json serialization strategy.
class AnyJSONManager {
 public:
  template<typename T>
  inline AnyJSONManager& EnableType(const std::string& type_name) {  // NOLINT(*)
    std::type_index tp = std::type_index(typeid(T));
    if (type_name_.count(tp) != 0) {
      CHECK(type_name_.at(tp) == type_name)
          << "Type has already been registered as another typename " << type_name_.at(tp);
      return *this;
    }
    CHECK(type_map_.count(type_name) == 0)
        << "Type name " << type_name << " already registered in registry";
    Entry e;
    e.read = ReadAny<T>;
    e.write = WriteAny<T>;
    type_name_[tp] = type_name;
    type_map_[type_name] = e;
    return *this;
  }
  // return global singleton
  inline static AnyJSONManager* Global() {
    static AnyJSONManager inst;
    return &inst;
  }

 private:
  AnyJSONManager() {}

  template<typename T>
  inline static void WriteAny(JSONWriter *writer, const any &data) {
    writer->Write(dmlc::unsafe_get<T>(data));
  }
  template<typename T>
  inline static void ReadAny(JSONReader *reader, any* data) {
    T temp;
    reader->Read(&temp);
    *data = std::move(temp);
  }
  // data entry to store vtable for any type
  struct Entry {
    void (*read)(JSONReader* reader, any *data);
    void (*write)(JSONWriter* reader, const any& data);
  };

  template<typename T>
  friend struct Handler;

  std::unordered_map<std::type_index, std::string> type_name_;
  std::unordered_map<std::string, Entry> type_map_;
};

template<>
struct Handler<any> {
  inline static void Write(JSONWriter *writer, const any &data) {
    std::unordered_map<std::type_index, std::string>&
        nmap = AnyJSONManager::Global()->type_name_;
    std::type_index id = std::type_index(data.type());
    auto it = nmap.find(id);
    CHECK(it != nmap.end() && it->first == id)
        << "Type " << id.name() << " has not been registered via DMLC_JSON_ENABLE_ANY";
    std::string type_name = it->second;
    AnyJSONManager::Entry e = AnyJSONManager::Global()->type_map_.at(type_name);
    writer->BeginArray(false);
    writer->WriteArrayItem(type_name);
    writer->WriteArraySeperator();
    e.write(writer, data);
    writer->EndArray();
  }
  inline static void Read(JSONReader *reader, any *data) {
    std::string type_name;
    reader->BeginArray();
    CHECK(reader->NextArrayItem()) << "invalid any json format";
    Handler<std::string>::Read(reader, &type_name);
    std::unordered_map<std::string, AnyJSONManager::Entry>&
        tmap = AnyJSONManager::Global()->type_map_;
    auto it = tmap.find(type_name);
    CHECK(it != tmap.end() && it->first == type_name)
        << "Typename " << type_name << " has not been registered via DMLC_JSON_ENABLE_ANY";
    AnyJSONManager::Entry e = it->second;
    CHECK(reader->NextArrayItem()) << "invalid any json format";
    e.read(reader, data);
    CHECK(!reader->NextArrayItem()) << "invalid any json format";
  }
};
#endif  // DMLC_ENABLE_RTTI
#endif  // DMLC_STRICT_CXX11

}  // namespace json

// implementations of JSONReader/Writer
inline int JSONReader::NextChar() {
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
  return is_->get();
#else
  int ch = is_->at(0);
  is_->erase(0, 1);
  return ch;
#endif
}

inline int JSONReader::PeekNextChar() {
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
  return is_->peek();
#else
  return is_->at(0);
#endif
}

inline int JSONReader::NextNonSpace() {
  int ch;
  do {
    ch = NextChar();
    if (ch == '\n') ++line_count_n_;
    if (ch == '\r') ++line_count_r_;
  } while (isspace(ch));
  return ch;
}

inline int JSONReader::PeekNextNonSpace() {
  int ch;
  while (true) {
    ch = PeekNextChar();
    if (ch == '\n') ++line_count_n_;
    if (ch == '\r') ++line_count_r_;
    if (!isspace(ch)) break;
    NextChar();
  }
  return ch;
}

namespace {
  template<typename T>
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
  void Extend(std::ostream *os, T item) {
    *os << item;
  }
#else
  void Extend(std::string *ostr, T item) {
    *ostr += item;
  }
#endif
}  // namespace

inline void JSONReader::ReadString(std::string *out_str) {
  int ch = NextNonSpace();
  CHECK_EQ(ch, '\"')
      << "Error at" << line_info()
      << ", Expect \'\"\' but get \'" << static_cast<char>(ch) << '\'';
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
  std::ostringstream output;
#else
  std::string output = "";
#endif
  while (true) {
    ch = NextChar();
    if (ch == '\\') {
      char sch = static_cast<char>(NextChar());
      switch (sch) {
        case 'r': Extend(&output, "\r"); break;
        case 'n': Extend(&output, "\n"); break;
        case '\\': Extend(&output, "\\"); break;
        case 't': Extend(&output, "\t"); break;
        case '\"': Extend(&output, "\""); break;
        default: LOG(FATAL) << "unknown string escape \\" << sch;
      }
    } else {
      if (ch == '\"') break;
      Extend(&output, static_cast<char>(ch));
    }
    if (ch == EOF || ch == '\r' || ch == '\n') {
      LOG(FATAL)
          << "Error at" << line_info()
          << ", Expect \'\"\' but reach end of line ";
    }
  }
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
  *out_str = output.str();
#else
  *out_str = output;
#endif
}

template<typename ValueType>
inline void JSONReader::ReadNumber(ValueType *out_value) {
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
  *is_ >> *out_value;
  CHECK(!is_->fail())
      << "Error at" << line_info()
      << ", Expect number";
#else
  char* endptr;
  const char* icstr = is_->c_str();
  unsigned number = strtol(icstr, &endptr, 10);
  is_->erase(0, endptr - icstr);
  *out_value = static_cast<ValueType>(number);
#endif
}

inline void JSONReader::BeginObject() {
  int ch = NextNonSpace();
  CHECK_EQ(ch, '{')
      << "Error at" << line_info()
      << ", Expect \'{\' but get \'" << static_cast<char>(ch) << '\'';
  scope_counter_.push_back(0);
}

inline void JSONReader::BeginArray() {
  int ch = NextNonSpace();
  CHECK_EQ(ch, '[')
      << "Error at" << line_info()
      << ", Expect \'[\' but get \'" << static_cast<char>(ch) << '\'';
  scope_counter_.push_back(0);
}

inline bool JSONReader::NextObjectItem(std::string *out_key) {
  bool next = true;
  if (scope_counter_.back() != 0) {
    int ch = NextNonSpace();
    if (ch == EOF) {
      next = false;
    } else if (ch == '}') {
      next = false;
    } else {
      CHECK_EQ(ch, ',')
          << "Error at" << line_info()
          << ", JSON object expect \'}\' or \',\' \'" << static_cast<char>(ch) << '\'';
    }
  } else {
    int ch = PeekNextNonSpace();
    if (ch == '}') {
      NextChar();
      next = false;
    }
  }
  if (!next) {
    scope_counter_.pop_back();
    return false;
  } else {
    scope_counter_.back() += 1;
    ReadString(out_key);
    int ch = NextNonSpace();
    CHECK_EQ(ch, ':')
        << "Error at" << line_info()
        << ", Expect \':\' but get \'" << static_cast<char>(ch) << '\'';
    return true;
  }
}

inline bool JSONReader::NextArrayItem() {
  bool next = true;
  if (scope_counter_.back() != 0) {
    int ch = NextNonSpace();
    if (ch == EOF) {
      next = false;
    } else if (ch == ']') {
      next = false;
    } else {
      CHECK_EQ(ch, ',')
          << "Error at" << line_info()
          << ", JSON array expect \']\' or \',\'. Get \'" << static_cast<char>(ch) << "\' instead";
    }
  } else {
    int ch = PeekNextNonSpace();
    if (ch == ']') {
      NextChar();
      next = false;
    }
  }
  if (!next) {
    scope_counter_.pop_back();
    return false;
  } else {
    scope_counter_.back() += 1;
    return true;
  }
}

template<typename ValueType>
inline void JSONReader::Read(ValueType *out_value) {
  json::Handler<ValueType>::Read(this, out_value);
}

inline void JSONWriter::WriteNoEscape(const std::string &s) {
  Extend(os_, '\"');
  Extend(os_, s);
  Extend(os_, '\"');
}

inline void JSONWriter::WriteString(const std::string &s) {
  Extend(os_, '\"');
  for (size_t i = 0; i < s.length(); ++i) {
    char ch = s[i];
    switch (ch) {
      case '\r': Extend(os_, "\\r"); break;
      case '\n': Extend(os_, "\\n"); break;
      case '\\': Extend(os_, "\\\\"); break;
      case '\t': Extend(os_, "\\t"); break;
      case '\"': Extend(os_, "\\\""); break;
      default: Extend(os_, ch);
    }
  }
  Extend(os_, '\"');
}

template<typename ValueType>
inline void JSONWriter::WriteNumber(const ValueType &v) {
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
  Extend(os_, v);
#else
  Extend(os_, std::to_string(v));
#endif
}

inline void JSONWriter::BeginArray(bool multi_line) {
  Extend(os_, '[');
  scope_multi_line_.push_back(multi_line);
  scope_counter_.push_back(0);
}

inline void JSONWriter::EndArray() {
  CHECK_NE(scope_multi_line_.size(), 0U);
  CHECK_NE(scope_counter_.size(), 0U);
  bool newline = scope_multi_line_.back();
  size_t nelem = scope_counter_.back();
  scope_multi_line_.pop_back();
  scope_counter_.pop_back();
  if (newline && nelem != 0) WriteSeperator();
  Extend(os_, ']');
}

inline void JSONWriter::BeginObject(bool multi_line) {
  Extend(os_, '{');
  scope_multi_line_.push_back(multi_line);
  scope_counter_.push_back(0);
}

inline void JSONWriter::EndObject() {
  CHECK_NE(scope_multi_line_.size(), 0U);
  CHECK_NE(scope_counter_.size(), 0U);
  bool newline = scope_multi_line_.back();
  size_t nelem = scope_counter_.back();
  scope_multi_line_.pop_back();
  scope_counter_.pop_back();
  if (newline && nelem != 0) WriteSeperator();
  Extend(os_, '}');
}

template<typename ValueType>
inline void JSONWriter::WriteObjectKeyValue(const std::string &key,
                                            const ValueType &value) {
  if (scope_counter_.back() > 0) {
    Extend(os_, ", ");
  }
  WriteSeperator();
  Extend(os_, '\"');
  Extend(os_, key);
  Extend(os_, "\": ");
  scope_counter_.back() += 1;
  json::Handler<ValueType>::Write(this, value);
}

inline void JSONWriter::WriteArraySeperator() {
  if (scope_counter_.back() != 0) {
    Extend(os_, ", ");
  }
  scope_counter_.back() += 1;
  WriteSeperator();
}

template<typename ValueType>
inline void JSONWriter::WriteArrayItem(const ValueType &value) {
  this->WriteArraySeperator();
  json::Handler<ValueType>::Write(this, value);
}

template<typename ValueType>
inline void JSONWriter::Write(const ValueType &value) {
  size_t nscope = scope_multi_line_.size();
  json::Handler<ValueType>::Write(this, value);
  CHECK_EQ(nscope, scope_multi_line_.size())
      << "Uneven scope, did you call EndArray/EndObject after each BeginObject/Array?";
}

inline void JSONWriter::WriteSeperator() {
  if (scope_multi_line_.size() == 0 || scope_multi_line_.back()) {
    Extend(os_, '\n');
    Extend(os_, std::string(scope_multi_line_.size() * 2, ' '));
  }
}

inline void JSONObjectReadHelper::ReadAllFields(JSONReader *reader) {
  reader->BeginObject();
  std::map<std::string, int> visited;
  std::string key;
  while (reader->NextObjectItem(&key)) {
    if (map_.count(key) != 0) {
      Entry e = map_[key];
      (*e.func)(reader, e.addr);
      visited[key] = 0;
    } else {
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
      std::ostringstream err;
#else
      std::string err("");
#endif
      Extend(&err, "JSONReader: Unknown field ");
      Extend(&err, key);
      Extend(&err, ", candidates are: \n");
      for (std::map<std::string, Entry>::iterator
               it = map_.begin(); it != map_.end(); ++it) {
        Extend(&err, '\"');
        Extend(&err, it->first);
        Extend(&err, "\"\n");
      }
#ifndef _LIBCPP_SGX_NO_IOSTREAMS
      LOG(FATAL) << err.str();
#else
      LOG(FATAL) << err;
#endif
    }
  }
  if (visited.size() != map_.size()) {
    for (std::map<std::string, Entry>::iterator
             it = map_.begin(); it != map_.end(); ++it) {
      if (it->second.optional) continue;
      CHECK_NE(visited.count(it->first), 0U)
          << "JSONReader: Missing field \"" << it->first << "\"\n At "
          << reader->line_info();
    }
  }
}

template<typename T>
inline void JSONObjectReadHelper::ReaderFunction(JSONReader *reader, void *addr) {
  json::Handler<T>::Read(reader, static_cast<T*>(addr));
}

template<typename T>
inline void JSONObjectReadHelper::
DeclareFieldInternal(const std::string &key, T *addr, bool optional) {
  CHECK_EQ(map_.count(key), 0U)
      << "Adding duplicate field " << key;
  Entry e;
  e.func = ReaderFunction<T>;
  e.addr = static_cast<void*>(addr);
  e.optional = optional;
  map_[key] = e;
}

}  // namespace dmlc
#endif  // DMLC_JSON_H_