数据集的读取方式

• 文件目录方式
• DataFrame 方式读取
• tensorflow data 方式读取

列举的三种是个人认为实际开发中比较常用的方式、后面我们使用这三种方式分别来进行数据集的读取

keras内置的学习数据集

在keras学习的时候、我们常常会用到它自带的一些数据集来做网络学习例如,在使用的时候我们会觉得使用起来非常的便捷

from keras.datasets import cifar10 
(x_train, y_train), (x_test, y_test) = cifar10.load_data()

from keras.datasets import imdb 
(x_train, y_train), (x_test, y_test) = 
  imdb.load_data(path="imdb.npz", num_words=None, skip_top=0, maxlen=None, 
  seed=113, start_char=1, oov_char=2, index_from=3)

模型定义

使用上面的数据集之前、我们先定义一个模型结构、如下图所示

from typing import Optional, Callable
import keras
from keras.src.layers import GlobalAveragePooling2D, Dense, Dropout


def buildResNet50(input_shape=(224, 224, 3), trainable=False):
    net = keras.applications.ResNet50(weights='imagenet', include_top=False, input_shape=input_shape, )
    net.trainable = trainable
    return net

def buildResNet101(input_shape=(224, 224, 3), trainable=False):
    net = keras.applications.ResNet101(weights="imagenet", include_top=False, input_shape=input_shape)
    net.trainable = trainable
    return net


class ResNet(object):
    def __init__(self, base_model_callback: Optional[Callable] = None):
        super(ResNet, self).__init__()
        self.base_model = base_model_callback()

    def addLayers(self, *input_layer):
        x = self.base_model.output
        for layer in input_layer:
            x = layer(x)
        return x

    def build_model(self):
        layers = [
            GlobalAveragePooling2D(),
            Dense(2048, activation='relu'),
            Dropout(0.5, name='dropout'),
        ]
        x = self.addLayers(*layers)
        out_put = Dense(4, activation='softmax',name='output')(x)
        return keras.Model(inputs=[self.base_model.input,], outputs=[out_put,])


if __name__ == '__main__':
    input_shape = (224, 224, 3)
    resNet = ResNet(base_model_callback=lambda: buildResNet50(input_shape=input_shape))
    model = resNet.build_model()

    model.summary()

    keras.utils.plot_model(model, to_file='model.png', show_layer_names=True,show_shapes=True, rankdir='TB')

数据准备

有了上面的网络结构、我们需要做训练的前置准备`model.fit()` 我选择了4个分类的车辆数据、作为学习使用

文件目录方式获取

• 定义 MutableDataset 数据集类

class MutableDataset(object):
    def __init__(self):
        super(MutableDataset, self).__init__()

    """使用文件目录的方式读取、每个子文件夹作为一个分类"""
    def data_from_dict(self, root_dict,batch_size = 4):
        train_datagen = ImageDataGenerator(rescale=1. / 255)
        train_generator = train_datagen.flow_from_directory(root_dict,target_size=(224, 224),batch_size=batch_size,class_mode='categorical')
        #val_datagen = ImageDataGenerator(rescale=1. / 255)
        #val_generator = val_datagen.flow_from_directory(root_dict,target_size=(224, 224),batch_size=4,class_mode='categorical',subset='validation')
        return train_generator #val_generator
        
#####################使用上面的网络模型######################        
if __name__ == '__main__':
    input_shape = (224, 224, 3)
    resNet = ResNet(base_model_callback=lambda: buildResNet50(input_shape=input_shape))
    model = resNet.build_model()

    multi_dataset = MutableDataset()
    train_data = multi_dataset.data_from_dict(root_dict="../data")
    # x_batch, y_batch = next(train_data)  # 获取一个批次的数据
    # print(x_batch.shape)  # 应该是 (batch_size, 224, 224, 3)

    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    model.fit(train_data, epochs=10, batch_size=4,)

Dataframe方式获取数据

class MutableDataset(object):
    def __init__(self):
        super(MutableDataset, self).__init__()

    """使用文件目录的方式读取"""
    def data_from_dict(self, root_dict,batch_size = 4):
        train_datagen = ImageDataGenerator(rescale=1. / 255)
        train_generator = train_datagen.flow_from_directory(root_dict,target_size=(224, 224),batch_size=batch_size,class_mode='categorical')
        #val_datagen = ImageDataGenerator(rescale=1. / 255)
        #val_generator = val_datagen.flow_from_directory(root_dict,target_size=(224, 224),batch_size=4,class_mode='categorical',subset='validation')
        return train_generator #val_generator
		
    """读取文件目录下面的文件信息、可以自己定义我们自己的结构"""
    def get_file(self, root_dict,category,images,labels):
        for file in os.listdir(root_dict):
            full_path = os.path.join(root_dict, file)
            if os.path.isdir(full_path):
                self.get_file(full_path,file,images,labels)
            else:
                images.append(full_path)
                labels.append(category)

        return images,labels
		
    """从dataframe加载对应的数据"""
    def data_from_frame(self,root_dict,batch_size = 4):
        images,labels = self.get_file(root_dict,category=None,images=[],labels=[])
        data_frame = DataFrame({"paths":images,"labels":labels})
        train_datagen = ImageDataGenerator(rescale=1. / 255)
        train_generator = train_datagen.flow_from_dataframe(dataframe=data_frame,x_col="paths",y_col="labels",batch_size=batch_size,target_size=(224, 224),class_mode='categorical')
        return train_generator
        
# 训练的时候调整对应的数据获取方式        
if __name__ == '__main__':
    input_shape = (224, 224, 3)
    resNet = ResNet(base_model_callback=lambda: buildResNet50(input_shape=input_shape))
    model = resNet.build_model()

    multi_dataset = MutableDataset()
    #train_data = multi_dataset.data_from_dict(root_dict="../data")
    train_data = multi_dataset.data_from_frame(root_dict="../data")
    # x_batch, y_batch = next(train_data)  # 获取一个批次的数据
    # print(x_batch.shape)  # 应该是 (batch_size, 224, 224, 3)
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    model.fit(train_data, epochs=10, batch_size=4,)

基于tensorflow的data获取

class MutableDataset(object):
    def __init__(self):
        super(MutableDataset, self).__init__()

    """使用文件目录的方式读取"""
    def data_from_dict(self, root_dict,batch_size = 4):
        train_datagen = ImageDataGenerator(rescale=1. / 255)
        train_generator = train_datagen.flow_from_directory(root_dict,target_size=(224, 224),batch_size=batch_size,class_mode='categorical')
        #val_datagen = ImageDataGenerator(rescale=1. / 255)
        #val_generator = val_datagen.flow_from_directory(root_dict,target_size=(224, 224),batch_size=4,class_mode='categorical',subset='validation')
        return train_generator #val_generator

    def get_file(self, root_dict,category,images,labels):
        for file in os.listdir(root_dict):
            full_path = os.path.join(root_dict, file)
            if os.path.isdir(full_path):
                self.get_file(full_path,file,images,labels)
            else:
                images.append(full_path)
                labels.append(category)

        return images,labels
		
    """dataframe方式"""
    def data_from_frame(self,root_dict,batch_size = 4):
        images,labels = self.get_file(root_dict,category=None,images=[],labels=[])
        data_frame = DataFrame({"paths":images,"labels":labels})
        train_datagen = ImageDataGenerator(rescale=1. / 255)
        train_generator = train_datagen.flow_from_dataframe(dataframe=data_frame,x_col="paths",y_col="labels",batch_size=batch_size,target_size=(224, 224),class_mode='categorical')
        return train_generator

    """这个是使用tf处理方式"""
    def _parse_function(self,x, y):
        label_category = {"bus":0,"family sedan":1,"minibus":2,"SUV":3}
        # 将图像路径解码
        original_path = x.numpy().decode("utf-8")
        original_label = y.numpy().decode("utf-8")
        original_label = label_category[original_label]

        image = cv2.imread(original_path)
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        image = cv2.resize(image, (224, 224))

        y_one_hot = tf.one_hot(original_label, depth=4)
        x = tf.cast(image, tf.float32)
        y = tf.cast(y_one_hot, tf.int32)
        return x, y

        # 另外一种使用tensorflow的api来处理
        # x = tf.io.read_file(x)
        # x = tf.image.decode_jpeg(x, channels=3)
        # x = tf.image.resize(x, [244, 244])
        # x = tf.cast(x, dtype=tf.float32) / 255.0
        # y = tf.convert_to_tensor(y)
        # y = tf.one_hot(y, depth=5)
        # return x, y

    def data_from_tf_dataset(self,root_dict,batch_size = 4):

        images, labels = self.get_file(root_dict, category=None, images=[], labels=[])

        db_train = (tf.data.Dataset.from_tensor_slices((images, labels))
                    .map(lambda x,y: tf.py_function(self._parse_function,inp=[x,y],Tout=[tf.float32,tf.int32])))

        # shape 强制定义
        db_train = db_train.map(lambda x,y: (tf.ensure_shape(x, [224, 224, 3]), tf.ensure_shape(y, [4])))
        db_train = db_train.batch(batch_size=batch_size)
        return db_train
        
 
 if __name__ == '__main__':
    input_shape = (224, 224, 3)
    resNet = ResNet(base_model_callback=lambda: buildResNet50(input_shape=input_shape))
    model = resNet.build_model()

    multi_dataset = MutableDataset()
    #train_data = multi_dataset.data_from_dict(root_dict="../data")
    #train_data = multi_dataset.data_from_frame(root_dict="../data")
    train_data = multi_dataset.data_from_tf_dataset(root_dict="../data")
    # x_batch, y_batch = next(train_data)  # 获取一个批次的数据
    # print(x_batch.shape)  # 应该是 (batch_size, 224, 224, 3)
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    model.fit(train_data, epochs=10, batch_size=4,)

网络结构如下