The whole process is simplied using the Dataset API
. Here are both the parts: (1): Convert numpy array to tfrecords
and (2): read the tfrecords to generate batches
.
1. Creation of tfrecords from a numpy array:
Example arrays:
inputs = np.random.normal(size=(5, 32, 32, 3))
labels = np.random.randint(0,2,size=(5,))
def npy_to_tfrecords(inputs, labels, filename):
with tf.io.TFRecordWriter(filename) as writer:
for X, y in zip(inputs, labels):
# Feature contains a map of string to feature proto objects
feature = {}
feature['X'] = tf.train.Feature(float_list=tf.train.FloatList(value=X.flatten()))
feature['y'] = tf.train.Feature(int64_list=tf.train.Int64List(value=[y]))
# Construct the Example proto object
example = tf.train.Example(features=tf.train.Features(feature=feature))
# Serialize the example to a string
serialized = example.SerializeToString()
# write the serialized objec to the disk
writer.write(serialized)
npy_to_tfrecords(inputs, labels, 'numpy.tfrecord')
2. Read the tfrecords using the Dataset API:
filenames = ['numpy.tfrecord']
dataset = tf.data.TFRecordDataset(filenames)
# for version 1.5 and above use tf.data.TFRecordDataset
# example proto decode
def _parse_function(example_proto):
keys_to_features = {'X':tf.io.FixedLenFeature(shape=(32, 32, 3), dtype=tf.float32),
'y': tf.io.FixedLenFeature((), tf.int64, default_value=0)}
parsed_features = tf.io.parse_single_example(example_proto, keys_to_features)
return parsed_features['X'], parsed_features['y']
# Parse the record into tensors.
dataset = dataset.map(_parse_function)
# Generate batches
dataset = dataset.batch(5)
Check the generated batches are proper:
for data in dataset:
break
np.testing.assert_allclose(inputs[0] ,data[0][0])
np.testing.assert_allclose(labels[0] ,data[1][0])