Update June 09, 2018
- Starting from Tensorflow 1.9, one can pass
tf.data.Dataset
object directly intokeras.Model.fit()
and it would act similar tofit_generator
. - A complete example can be found on this gist.
# Load mnist training data
(x_train, y_train), _ = tf.keras.datasets.mnist.load_data()
training_set = tfdata_generator(x_train, y_train,is_training=True)
model = # your keras model here
model.fit(
training_set.make_one_shot_iterator(),
steps_per_epoch=len(x_train) // 128,
epochs=5,
verbose = 1)
tfdata_generator
is a function that returns an iterabletf.data.Dataset
.
def tfdata_generator(images, labels, is_training, batch_size=128):
'''Construct a data generator using `tf.Dataset`. '''
def map_fn(image, label):
'''Preprocess raw data to trainable input. '''
x = tf.reshape(tf.cast(image, tf.float32), (28, 28, 1))
y = tf.one_hot(tf.cast(label, tf.uint8), _NUM_CLASSES)
return x, y
dataset = tf.data.Dataset.from_tensor_slices((images, labels))
if is_training:
dataset = dataset.shuffle(1000) # depends on sample size
dataset = dataset.map(map_fn)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat()
dataset = dataset.prefetch(tf.contrib.data.AUTOTUNE)
return dataset
Old Solution:
In addition to @Yu-Yang’s answer, you can also modify tf.data.Dataset
to become a generator for fit_generator
as following
from tensorflow.contrib.learn.python.learn.datasets import mnist
data = mnist.load_mnist()
model = # your Keras model
model.fit_generator(generator = tfdata_generator(data.train.images, data.train.labels),
steps_per_epoch=200,
workers = 0 , # This is important
verbose = 1)
def tfdata_generator(images, labels, batch_size=128, shuffle=True,):
def map_func(image, label):
'''A transformation function'''
x_train = tf.reshape(tf.cast(image, tf.float32), image_shape)
y_train = tf.one_hot(tf.cast(label, tf.uint8), num_classes)
return [x_train, y_train]
dataset = tf.data.Dataset.from_tensor_slices((images, labels))
dataset = dataset.map(map_func)
dataset = dataset.shuffle().batch(batch_size).repeat()
iterator = dataset.make_one_shot_iterator()
next_batch = iterator.get_next()
while True:
yield K.get_session().run(next_batch)