Source code for mxnet.contrib.io

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# coding: utf-8
"""Contrib data iterators for common data formats."""
from ..io import DataIter, DataDesc
from .. import ndarray as nd



[docs]class DataLoaderIter(DataIter):
    """Returns an iterator for ``mx.gluon.data.Dataloader`` so gluon dataloader
    can be used in symbolic module.

    Parameters
    ----------
    loader : mxnet.gluon.data.Dataloader
        Gluon dataloader instance
    data_name : str, optional
        The data name.
    label_name : str, optional
        The label name.
    dtype : str, optional
        The dtype specifier, can be float32 or float16

    Examples
    --------
    >>> import mxnet as mx
    >>> from mxnet.gluon.data.vision import MNIST
    >>> from mxnet.gluon.data import DataLoader
    >>> train_dataset = MNIST(train=True)
    >>> train_data = mx.gluon.data.DataLoader(train_dataset, 32, shuffle=True, num_workers=4)
    >>> dataiter = mx.io.DataloaderIter(train_data)
    >>> for batch in dataiter:
    ...     batch.data[0].shape
    ...
    (32L, 28L, 28L, 1L)
    """
    def __init__(self, loader, data_name='data', label_name='softmax_label', dtype='float32'):
        super(DataLoaderIter, self).__init__()
        self._loader = loader
        self._iter = iter(self._loader)
        data, label = next(self._iter)
        self.batch_size = data.shape[0]
        self.dtype = dtype
        self.provide_data = [DataDesc(data_name, data.shape, dtype)]
        self.provide_label = [DataDesc(label_name, label.shape, dtype)]
        self._current_batch = None
        self.reset()


[docs]    def reset(self):
        self._iter = iter(self._loader)



[docs]    def iter_next(self):
        try:
            self._current_batch = next(self._iter)
        except StopIteration:
            self._current_batch = None
        return self._current_batch is not None



[docs]    def getdata(self):
        if self.getpad():
            dshape = self._current_batch[0].shape
            ret = nd.empty(shape=([self.batch_size] + list(dshape[1:])))
            ret[:dshape[0]] = self._current_batch[0].astype(self.dtype)
            return [ret]
        return [self._current_batch[0].astype(self.dtype)]



[docs]    def getlabel(self):
        if self.getpad():
            lshape = self._current_batch[1].shape
            ret = nd.empty(shape=([self.batch_size] + list(lshape[1:])))
            ret[:lshape[0]] = self._current_batch[1].astype(self.dtype)
            return [ret]
        return [self._current_batch[1].astype(self.dtype)]



[docs]    def getpad(self):
        return self.batch_size - self._current_batch[0].shape[0]



[docs]    def getindex(self):
        return None