使用数据集生成器的 Tensorflow model.fit()

Question

我正在使用数据集 API 生成训练数据并将其分类为 NN 的批次。

这是我的代码的最小工作示例：

import tensorflow as tf

import numpy as np

import random

def my_generator():

    while True:

        x = np.random.rand(4, 20)

        y = random.randint(0, 11)

        label = tf.one_hot(y, depth=12)

        yield x.reshape(4, 20, 1), label

def my_input_fn():

    dataset = tf.data.Dataset.from_generator(lambda: my_generator(),

                                             output_types=(tf.float64, tf.int32))

    dataset = dataset.batch(32)

    iterator = dataset.make_one_shot_iterator()

    batch_features, batch_labels = iterator.get_next()

    return batch_features, batch_labels

if __name__ == "__main__":

    tf.enable_eager_execution()

    model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),

                                 tf.keras.layers.Dense(128, activation=tf.nn.relu),

                                 tf.keras.layers.Dense(12, activation=tf.nn.softmax)])

    model.compile(optimizer='adam',

                  loss='categorical_crossentropy',

                  metrics=['accuracy'])

    data_generator = my_input_fn()

    model.fit(data_generator)

但batch_size不是 的公认关键字fit_generator()。

我对这些错误消息感到困惑，如果有人能对它们有所了解，或者指出我做错了什么，我将不胜感激。

Answer 1

虽然错误的根源仍然模糊不清，但我已经找到了使代码工作的解决方案。我会把它贴在这里，以防它对处于类似情况的任何人有用。基本上，我将其更改my_input_fn()为生成器并使用model.fit_generator()如下：import tensorflow as tfimport numpy as npimport randomdef my_generator(total_items):&nbsp; &nbsp; i = 0&nbsp; &nbsp; while i < total_items:&nbsp; &nbsp; &nbsp; &nbsp; x = np.random.rand(4, 20)&nbsp; &nbsp; &nbsp; &nbsp; y = random.randint(0, 11)&nbsp; &nbsp; &nbsp; &nbsp; label = tf.one_hot(y, depth=12)&nbsp; &nbsp; &nbsp; &nbsp; yield x.reshape(4, 20, 1), label&nbsp; &nbsp; &nbsp; &nbsp; i += 1def my_input_fn(total_items, epochs):&nbsp; &nbsp; dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;output_types=(tf.float64, tf.int64))&nbsp; &nbsp; dataset = dataset.repeat(epochs)&nbsp; &nbsp; dataset = dataset.batch(32)&nbsp; &nbsp; iterator = dataset.make_one_shot_iterator()&nbsp; &nbsp; while True:&nbsp; &nbsp; &nbsp; &nbsp; batch_features, batch_labels = iterator.get_next()&nbsp; &nbsp; &nbsp; &nbsp; yield batch_features, batch_labelsif __name__ == "__main__":&nbsp; &nbsp; tf.enable_eager_execution()&nbsp; &nbsp; model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;tf.keras.layers.Dense(64, activation=tf.nn.relu),&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;tf.keras.layers.Dense(12, activation=tf.nn.softmax)])&nbsp; &nbsp; model.compile(optimizer='adam',&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; loss='categorical_crossentropy',&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; metrics=['accuracy'])&nbsp; &nbsp; total_items = 200&nbsp; &nbsp; batch_size = 32&nbsp; &nbsp; epochs = 10&nbsp; &nbsp; num_batches = int(total_items/batch_size)&nbsp; &nbsp; train_data_generator = my_input_fn(total_items, epochs)&nbsp; &nbsp; model.fit_generator(generator=train_data_generator, steps_per_epoch=num_batches, epochs=epochs, verbose=1)编辑正如 giser_yugang 在评论中暗示的那样，也可以将其my_input_fn()作为返回dataset而不是单个批次的函数来执行。def my_input_fn(total_items, epochs):&nbsp; &nbsp; dataset = tf.data.Dataset.from_generator(lambda: my_generator(total_items),&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;output_types=(tf.float64, tf.int64))&nbsp; &nbsp; dataset = dataset.repeat(epochs)&nbsp; &nbsp; dataset = dataset.batch(32)&nbsp; &nbsp; return datasetif __name__ == "__main__":&nbsp; &nbsp; tf.enable_eager_execution()&nbsp; &nbsp; model = tf.keras.Sequential([tf.keras.layers.Flatten(input_shape=(4, 20, 1)),&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;tf.keras.layers.Dense(64, activation=tf.nn.relu),&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp;tf.keras.layers.Dense(12, activation=tf.nn.softmax)])&nbsp; &nbsp; model.compile(optimizer='adam',&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; loss='categorical_crossentropy',&nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; &nbsp; metrics=['accuracy'])&nbsp; &nbsp; total_items = 100&nbsp; &nbsp; batch_size = 32&nbsp; &nbsp; epochs = 10&nbsp; &nbsp; num_batches = int(total_items/batch_size)&nbsp; &nbsp; dataset = my_input_fn(total_items, epochs)&nbsp; &nbsp; model.fit_generator(dataset, epochs=epochs, steps_per_epoch=num_batches)这些方法之间似乎没有任何平均性能差异。