mxnet
serialization.h
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements. See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership. The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License. You may obtain a copy of the License at
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing,
13  * software distributed under the License is distributed on an
14  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15  * KIND, either express or implied. See the License for the
16  * specific language governing permissions and limitations
17  * under the License.
18  */
19 
27 #ifndef MXNET_COMMON_SERIALIZATION_H_
28 #define MXNET_COMMON_SERIALIZATION_H_
29 
30 #include <dmlc/logging.h>
31 #include <mxnet/graph_attr_types.h>
32 #include <nnvm/graph_attr_types.h>
33 #include <nnvm/tuple.h>
34 
35 #include <cstring>
36 #include <map>
37 #include <set>
38 #include <string>
39 #include <tuple>
40 #include <unordered_map>
41 #include <unordered_set>
42 #include <utility>
43 #include <vector>
44 
45 
46 namespace mxnet {
47 namespace common {
48 
49 template<typename T>
50 inline size_t SerializedSize(const T &obj);
51 
52 template<typename T>
53 inline size_t SerializedSize(const nnvm::Tuple <T> &obj);
54 
55 template<typename K, typename V>
56 inline size_t SerializedSize(const std::map <K, V> &obj);
57 
58 template<>
59 inline size_t SerializedSize(const std::string &obj);
60 
61 template<typename... Args>
62 inline size_t SerializedSize(const std::tuple<Args...> &obj);
63 
64 template<typename T>
65 inline void Serialize(const T &obj, char **buffer);
66 
67 template<typename T>
68 inline void Serialize(const nnvm::Tuple <T> &obj, char **buffer);
69 
70 template<typename K, typename V>
71 inline void Serialize(const std::map <K, V> &obj, char **buffer);
72 
73 template<>
74 inline void Serialize(const std::string &obj, char **buffer);
75 
76 template<typename... Args>
77 inline void Serialize(const std::tuple<Args...> &obj, char **buffer);
78 
79 template<typename T>
80 inline void Deserialize(T *obj, const std::string &buffer, size_t *curr_pos);
81 
82 template<typename T>
83 inline void Deserialize(nnvm::Tuple <T> *obj, const std::string &buffer, size_t *curr_pos);
84 
85 template<typename K, typename V>
86 inline void Deserialize(std::map <K, V> *obj, const std::string &buffer, size_t *curr_pos);
87 
88 template<>
89 inline void Deserialize(std::string *obj, const std::string &buffer, size_t *curr_pos);
90 
91 template<typename... Args>
92 inline void Deserialize(std::tuple<Args...> *obj, const std::string &buffer, size_t *curr_pos);
93 
94 
95 template<typename T>
96 struct is_container {
97  static const bool value = !std::is_pod<T>::value;
98 };
99 
100 template<typename T>
101 inline size_t SerializedSize(const T &obj) {
102  return sizeof(T);
103 }
104 
105 template<typename T>
106 inline size_t SerializedSize(const nnvm::Tuple <T> &obj) {
108  size_t sum_val = 4;
109  for (const auto& el : obj) {
110  sum_val += SerializedSize(el);
111  }
112  return sum_val;
113  } else {
114  return 4 + (obj.ndim() * sizeof(T));
115  }
116 }
117 
118 template<typename K, typename V>
119 inline size_t SerializedSize(const std::map <K, V> &obj) {
120  size_t sum_val = 4;
122  for (const auto& p : obj) {
123  sum_val += SerializedSize(p.first) + SerializedSize(p.second);
124  }
125  } else if (is_container<K>::value) {
126  for (const auto& p : obj) {
127  sum_val += SerializedSize(p.first);
128  }
129  sum_val += sizeof(V) * obj.size();
130  } else if (is_container<V>::value) {
131  for (const auto& p : obj) {
132  sum_val += SerializedSize(p.second);
133  }
134  sum_val += sizeof(K) * obj.size();
135  } else {
136  sum_val += (sizeof(K) + sizeof(V)) * obj.size();
137  }
138  return sum_val;
139 }
140 
141 template<>
142 inline size_t SerializedSize(const std::string &obj) {
143  return obj.size() + 4;
144 }
145 
146 template<int I>
148  template<typename... Args>
149  static inline size_t Compute(const std::tuple<Args...> &obj) {
150  return SerializedSize(std::get<I>(obj)) + serialized_size_tuple<I-1>::Compute(obj);
151  }
152 };
153 
154 template<>
156  template<typename... Args>
157  static inline size_t Compute(const std::tuple<Args...> &obj) {
158  return SerializedSize(std::get<0>(obj));
159  }
160 };
161 
162 template<typename... Args>
163 inline size_t SerializedSize(const std::tuple<Args...> &obj) {
164  return serialized_size_tuple<sizeof... (Args)-1>::Compute(obj);
165 }
166 
167 // Serializer
168 
169 template<typename T>
170 inline size_t SerializedContainerSize(const T &obj, char **buffer) {
171  uint32_t size = obj.size();
172  std::memcpy(*buffer, &size, 4);
173  *buffer += 4;
174  return (size_t) size;
175 }
176 
177 template<typename T>
178 inline void Serialize(const T &obj, char **buffer) {
179  std::memcpy(*buffer, &obj, sizeof(T));
180  *buffer += sizeof(T);
181 }
182 
183 template<typename T>
184 inline void Serialize(const nnvm::Tuple <T> &obj, char **buffer) {
185  uint32_t size = obj.ndim();
186  std::memcpy(*buffer, &size, 4);
187  *buffer += 4;
188  for (auto& el : obj) {
189  Serialize(el, buffer);
190  }
191 }
192 
193 template<typename K, typename V>
194 inline void Serialize(const std::map <K, V> &obj, char **buffer) {
195  SerializedContainerSize(obj, buffer);
196  for (auto& p : obj) {
197  Serialize(p.first, buffer);
198  Serialize(p.second, buffer);
199  }
200 }
201 
202 template<>
203 inline void Serialize(const std::string &obj, char **buffer) {
204  auto size = SerializedContainerSize(obj, buffer);
205  std::memcpy(*buffer, &obj[0], size);
206  *buffer += size;
207 }
208 
209 template<int I>
211  template<typename... Args>
212  static inline void Compute(const std::tuple<Args...> &obj, char **buffer) {
213  serialize_tuple<I-1>::Compute(obj, buffer);
214  Serialize(std::get<I>(obj), buffer);
215  }
216 };
217 
218 template<>
219 struct serialize_tuple<0> {
220  template<typename... Args>
221  static inline void Compute(const std::tuple<Args...> &obj, char **buffer) {
222  Serialize(std::get<0>(obj), buffer);
223  }
224 };
225 
226 template<typename... Args>
227 inline void Serialize(const std::tuple<Args...> &obj, char **buffer) {
228  serialize_tuple<sizeof... (Args)-1>::Compute(obj, buffer);
229 }
230 
231 // Deserializer
232 
233 template<typename T>
234 inline size_t DeserializedContainerSize(T *obj, const std::string &buffer, size_t *curr_pos) {
235  uint32_t size = obj->size();
236  std::memcpy(&size, &buffer[*curr_pos], 4);
237  *curr_pos += 4;
238  return (size_t) size;
239 }
240 
241 template<typename T>
242 inline void Deserialize(T *obj, const std::string &buffer, size_t *curr_pos) {
243  std::memcpy(obj, &buffer[*curr_pos], sizeof(T));
244  *curr_pos += sizeof(T);
245 }
246 
247 template<typename T>
248 inline void Deserialize(nnvm::Tuple <T> *obj, const std::string &buffer, size_t *curr_pos) {
249  uint32_t size = obj->ndim();
250  std::memcpy(&size, &buffer[*curr_pos], 4);
251  *curr_pos += 4;
252  obj->SetDim(size);
253  for (size_t i = 0; i < size; ++i) {
254  Deserialize((*obj)[i], buffer, curr_pos);
255  }
256 }
257 
258 template<typename K, typename V>
259 inline void Deserialize(std::map <K, V> *obj, const std::string &buffer, size_t *curr_pos) {
260  auto size = DeserializedContainerSize(obj, buffer, curr_pos);
261  K first;
262  for (size_t i = 0; i < size; ++i) {
263  Deserialize(&first, buffer, curr_pos);
264  Deserialize(&(*obj)[first], buffer, curr_pos);
265  }
266 }
267 
268 template<>
269 inline void Deserialize(std::string *obj, const std::string &buffer, size_t *curr_pos) {
270  auto size = DeserializedContainerSize(obj, buffer, curr_pos);
271  obj->resize(size);
272  std::memcpy(&(obj->front()), &buffer[*curr_pos], size);
273  *curr_pos += size;
274 }
275 
276 template<int I>
278  template<typename... Args>
279  static inline void Compute(std::tuple<Args...> *obj,
280  const std::string &buffer, size_t *curr_pos) {
281  deserialize_tuple<I-1>::Compute(obj, buffer, curr_pos);
282  Deserialize(&std::get<I>(*obj), buffer, curr_pos);
283  }
284 };
285 
286 template<>
287 struct deserialize_tuple<0> {
288  template<typename... Args>
289  static inline void Compute(std::tuple<Args...> *obj,
290  const std::string &buffer, size_t *curr_pos) {
291  Deserialize(&std::get<0>(*obj), buffer, curr_pos);
292  }
293 };
294 
295 template<typename... Args>
296 inline void Deserialize(std::tuple<Args...> *obj, const std::string &buffer, size_t *curr_pos) {
297  deserialize_tuple<sizeof... (Args)-1>::Compute(obj, buffer, curr_pos);
298 }
299 
300 
301 template<typename T>
302 inline void Serialize(const T& obj, std::string* serialized_data) {
303  serialized_data->resize(SerializedSize(obj));
304  char* curr_pos = &(serialized_data->front());
305  Serialize(obj, &curr_pos);
306  CHECK_EQ((int64_t)curr_pos - (int64_t)&(serialized_data->front()),
307  serialized_data->size());
308 }
309 
310 template<typename T>
311 inline void Deserialize(T* obj, const std::string& serialized_data) {
312  size_t curr_pos = 0;
313  Deserialize(obj, serialized_data, &curr_pos);
314  CHECK_EQ(curr_pos, serialized_data.size());
315 }
316 
317 } // namespace common
318 } // namespace mxnet
319 #endif // MXNET_COMMON_SERIALIZATION_H_
namespace of mxnet
Definition: base.h:118
Definition: serialization.h:96
static void Compute(const std::tuple< Args... > &obj, char **buffer)
Definition: serialization.h:212
static size_t Compute(const std::tuple< Args... > &obj)
Definition: serialization.h:149
size_t SerializedContainerSize(const T &obj, char **buffer)
Definition: serialization.h:170
Definition: serialization.h:277
static void Compute(std::tuple< Args... > *obj, const std::string &buffer, size_t *curr_pos)
Definition: serialization.h:289
size_t DeserializedContainerSize(T *obj, const std::string &buffer, size_t *curr_pos)
Definition: serialization.h:234
void Deserialize(T *obj, const std::string &buffer, size_t *curr_pos)
Definition: serialization.h:242
static const bool value
Definition: serialization.h:97
static void Compute(std::tuple< Args... > *obj, const std::string &buffer, size_t *curr_pos)
Definition: serialization.h:279
Definition: serialization.h:210
Data structures that can appear in graph attributes.
void Serialize(const T &obj, char **buffer)
Definition: serialization.h:178
Definition: serialization.h:147
static void Compute(const std::tuple< Args... > &obj, char **buffer)
Definition: serialization.h:221
size_t SerializedSize(const T &obj)
Definition: serialization.h:101
static size_t Compute(const std::tuple< Args... > &obj)
Definition: serialization.h:157