mxnet
io.h
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements. See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership. The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License. You may obtain a copy of the License at
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing,
13  * software distributed under the License is distributed on an
14  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15  * KIND, either express or implied. See the License for the
16  * specific language governing permissions and limitations
17  * under the License.
18  */
19 
25 #ifndef MXNET_IO_H_
26 #define MXNET_IO_H_
27 
28 #include <dmlc/data.h>
29 #include <dmlc/registry.h>
30 #include <vector>
31 #include <string>
32 #include <utility>
33 #include <queue>
34 #include "./base.h"
35 #include "./ndarray.h"
36 
37 namespace mxnet {
42 template<typename DType>
43 class IIterator : public dmlc::DataIter<DType> {
44  public:
49  virtual void Init(const std::vector<std::pair<std::string, std::string> >& kwargs) = 0;
51  virtual void BeforeFirst(void) = 0;
53  virtual bool Next(void) = 0;
55  virtual const DType &Value(void) const = 0;
57  virtual ~IIterator(void) {}
59  std::vector<std::string> data_names;
61  inline void SetDataName(const std::string data_name) {
62  data_names.push_back(data_name);
63  }
64 }; // class IIterator
65 
67 struct DataInst {
69  unsigned index;
71  std::vector<TBlob> data;
73  std::string extra_data;
74 }; // struct DataInst
75 
79 struct DataBatch {
81  std::vector<NDArray> data;
83  std::vector<uint64_t> index;
85  std::string extra_data;
88 }; // struct DataBatch
89 
91 typedef std::function<IIterator<DataBatch> *()> DataIteratorFactory;
96  : public dmlc::FunctionRegEntryBase<DataIteratorReg,
97  DataIteratorFactory> {
98 };
99 //--------------------------------------------------------------
100 // The following part are API Registration of Iterators
101 //--------------------------------------------------------------
114 #define MXNET_REGISTER_IO_ITER(name) \
115  DMLC_REGISTRY_REGISTER(::mxnet::DataIteratorReg, DataIteratorReg, name)
116 } // namespace mxnet
117 #endif // MXNET_IO_H_
std::vector< std::string > data_names
store the name of each data, it could be used for making NDArrays
Definition: io.h:59
void SetDataName(const std::string data_name)
set data name to each attribute of data
Definition: io.h:61
namespace of mxnet
Definition: base.h:118
std::string extra_data
extra data to be fed to the network
Definition: io.h:73
int num_batch_padd
num of example padded to batch
Definition: io.h:87
std::vector< uint64_t > index
index of image data
Definition: io.h:83
unsigned index
unique id for instance
Definition: io.h:69
virtual const DType & Value(void) const =0
get current data
std::string extra_data
extra data to be fed to the network
Definition: io.h:85
iterator type
Definition: io.h:43
Registry entry for DataIterator factory functions.
Definition: io.h:95
virtual void Init(const std::vector< std::pair< std::string, std::string > > &kwargs)=0
set the parameters and init iter
virtual ~IIterator(void)
constructor
Definition: io.h:57
virtual bool Next(void)=0
move to next item
DataBatch of NDArray, returned by Iterator.
Definition: io.h:79
a single data instance
Definition: io.h:67
std::vector< TBlob > data
content of data
Definition: io.h:71
virtual void BeforeFirst(void)=0
reset the iterator
std::vector< NDArray > data
content of dense data, if this DataBatch is dense
Definition: io.h:81
std::function< IIterator< DataBatch > *()> DataIteratorFactory
typedef the factory function of data iterator
Definition: io.h:91