Keras——保存mnist数据集的image embedding
Keras - Save image embedding of the mnist data set
我为 MNIST 数据库编写了以下简单的 MLP 网络。
from __future__ import print_function
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras import callbacks
batch_size = 100
num_classes = 10
epochs = 20
tb = callbacks.TensorBoard(log_dir='/Users/shlomi.shwartz/tensorflow/notebooks/logs/minist', histogram_freq=10, batch_size=32,
write_graph=True, write_grads=True, write_images=True,
embeddings_freq=10, embeddings_layer_names=None,
embeddings_metadata=None)
early_stop = callbacks.EarlyStopping(monitor='val_loss', min_delta=0,
patience=3, verbose=1, mode='auto')
# the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(Dense(200, activation='relu', input_shape=(784,)))
model.add(Dropout(0.2))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(60, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(30, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = model.fit(x_train, y_train,
callbacks=[tb,early_stop],
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
模型 运行 很好，我可以在 TensorBoard 上看到标量信息。但是，当我更改 embeddings_freq=10 以尝试可视化图像（如 seen here）时，我收到以下错误：
Traceback (most recent call last):
File "/Users/shlomi.shwartz/IdeaProjects/TF/src/minist.py", line 65, in <module>
validation_data=(x_test, y_test))
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/models.py", line 870, in fit
initial_epoch=initial_epoch)
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/engine/training.py", line 1507, in fit
initial_epoch=initial_epoch)
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/engine/training.py", line 1117, in _fit_loop
callbacks.set_model(callback_model)
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/callbacks.py", line 52, in set_model
callback.set_model(model)
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/callbacks.py", line 719, in set_model
self.saver = tf.train.Saver(list(embeddings.values()))
File "/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/training/saver.py", line 1139, in __init__
self.build()
File "/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/training/saver.py", line 1161, in build
raise ValueError("No variables to save")
ValueError: No variables to save
问：我错过了什么？在 Keras 中这是正确的做法吗？
更新： 我知道使用嵌入投影有一些先决条件，但是我没有在 Keras 中找到这样做的好教程，任何帮助将不胜感激.
您至少需要一个 Keras 中的嵌入层。关于统计数据是对它们的一个很好的解释。它不是直接针对 Keras，但概念大致相同。 What is an embedding layer in a neural network
callbacks.TensorBoard这里所谓的"embedding"就是广义上的任意层权重。根据 Keras documentation:
embeddings_layer_names: a list of names of layers to keep eye on. If None or empty list all the embedding layer will be watched.
所以 默认情况下 ，它会监控 Embedding 层，但你并不真的需要 Embedding 层来使用这个可视化工具.
在您提供的 MLP 示例中，缺少的是 embeddings_layer_names 参数。你必须弄清楚你要可视化哪些层。假设您想可视化所有 Dense 层的权重（或者，kernel 在 Keras 中），您可以像这样指定 embeddings_layer_names：
model = Sequential()
model.add(Dense(200, activation='relu', input_shape=(784,)))
model.add(Dropout(0.2))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(60, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(30, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
embedding_layer_names = set(layer.name
for layer in model.layers
if layer.name.startswith('dense_'))
tb = callbacks.TensorBoard(log_dir='temp', histogram_freq=10, batch_size=32,
write_graph=True, write_grads=True, write_images=True,
embeddings_freq=10, embeddings_metadata=None,
embeddings_layer_names=embedding_layer_names)
model.compile(...)
model.fit(...)
然后，你可以在TensorBoard中看到类似这样的东西：
如果您想了解关于 embeddings_layer_names 的情况，您可以查看 the relevant lines in Keras source。
编辑：
所以这是可视化层输出的肮脏解决方案。由于原始 TensorBoard 回调不支持此功能，因此实施新的回调似乎是不可避免的。
由于在这里重写整个TensorBoard回调会占用很多页面space，所以我只是扩展原来的TensorBoard，写出不同的部分（已经很长了）。但为了避免重复计算和模型保存，重写 TensorBoard 回调将是一个更好更简洁的方法。
import tensorflow as tf
from tensorflow.contrib.tensorboard.plugins import projector
from keras import backend as K
from keras.models import Model
from keras.callbacks import TensorBoard
class TensorResponseBoard(TensorBoard):
def __init__(self, val_size, img_path, img_size, **kwargs):
super(TensorResponseBoard, self).__init__(**kwargs)
self.val_size = val_size
self.img_path = img_path
self.img_size = img_size
def set_model(self, model):
super(TensorResponseBoard, self).set_model(model)
if self.embeddings_freq and self.embeddings_layer_names:
embeddings = {}
for layer_name in self.embeddings_layer_names:
# initialize tensors which will later be used in `on_epoch_end()` to
# store the response values by feeding the val data through the model
layer = self.model.get_layer(layer_name)
output_dim = layer.output.shape[-1]
response_tensor = tf.Variable(tf.zeros([self.val_size, output_dim]),
name=layer_name + '_response')
embeddings[layer_name] = response_tensor
self.embeddings = embeddings
self.saver = tf.train.Saver(list(self.embeddings.values()))
response_outputs = [self.model.get_layer(layer_name).output
for layer_name in self.embeddings_layer_names]
self.response_model = Model(self.model.inputs, response_outputs)
config = projector.ProjectorConfig()
embeddings_metadata = {layer_name: self.embeddings_metadata
for layer_name in embeddings.keys()}
for layer_name, response_tensor in self.embeddings.items():
embedding = config.embeddings.add()
embedding.tensor_name = response_tensor.name
# for coloring points by labels
embedding.metadata_path = embeddings_metadata[layer_name]
# for attaching images to the points
embedding.sprite.image_path = self.img_path
embedding.sprite.single_image_dim.extend(self.img_size)
projector.visualize_embeddings(self.writer, config)
def on_epoch_end(self, epoch, logs=None):
super(TensorResponseBoard, self).on_epoch_end(epoch, logs)
if self.embeddings_freq and self.embeddings_ckpt_path:
if epoch % self.embeddings_freq == 0:
# feeding the validation data through the model
val_data = self.validation_data[0]
response_values = self.response_model.predict(val_data)
if len(self.embeddings_layer_names) == 1:
response_values = [response_values]
# record the response at each layers we're monitoring
response_tensors = []
for layer_name in self.embeddings_layer_names:
response_tensors.append(self.embeddings[layer_name])
K.batch_set_value(list(zip(response_tensors, response_values)))
# finally, save all tensors holding the layer responses
self.saver.save(self.sess, self.embeddings_ckpt_path, epoch)
要使用它：
tb = TensorResponseBoard(log_dir=log_dir, histogram_freq=10, batch_size=10,
write_graph=True, write_grads=True, write_images=True,
embeddings_freq=10,
embeddings_layer_names=['dense_1'],
embeddings_metadata='metadata.tsv',
val_size=len(x_test), img_path='images.jpg', img_size=[28, 28])
在启动 TensorBoard 之前，您需要将标签和图像保存到 log_dir 以便可视化：
from PIL import Image
img_array = x_test.reshape(100, 100, 28, 28)
img_array_flat = np.concatenate([np.concatenate([x for x in row], axis=1) for row in img_array])
img = Image.fromarray(np.uint8(255 * (1. - img_array_flat)))
img.save(os.path.join(log_dir, 'images.jpg'))
np.savetxt(os.path.join(log_dir, 'metadata.tsv'), np.where(y_test)[1], fmt='%d')
结果如下：
所以，我得出结论，您真正想要的（从您的 post 中还不完全清楚）是以类似的方式可视化模型的 预测 至 this Tensorboard demo.
首先，复制这些东西非常重要 even in Tensorflow, let alone Keras. The said demo makes very brief and passing references to things like metadata & sprite images，这是获得此类可视化效果所必需的。
底线：虽然不简单，但使用 Keras 确实可以做到。您不需要 Keras 回调；您只需要模型预测、必要的元数据和精灵图像，以及一些纯 TensorFlow 代码。所以，
第 1 步 - 获取测试集的模型预测：
emb = model.predict(x_test) # 'emb' for embedding
步骤 2a - 使用测试集的真实标签构建元数据文件：
import numpy as np
LOG_DIR = '/home/herc/SO/tmp' # FULL PATH HERE!!!
metadata_file = os.path.join(LOG_DIR, 'metadata.tsv')
with open(metadata_file, 'w') as f:
for i in range(len(y_test)):
c = np.nonzero(y_test[i])[0][0]
f.write('{}\n'.format(c))
步骤 2b - 获取 TensorFlow 人员 here 提供的精灵图像 mnist_10k_sprite.png，并将其放入您的 LOG_DIR
步骤 3 - 编写一些 Tensorflow 代码：
import tensorflow as tf
from tensorflow.contrib.tensorboard.plugins import projector
embedding_var = tf.Variable(emb, name='final_layer_embedding')
sess = tf.Session()
sess.run(embedding_var.initializer)
summary_writer = tf.summary.FileWriter(LOG_DIR)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
# Specify the metadata file:
embedding.metadata_path = os.path.join(LOG_DIR, 'metadata.tsv')
# Specify the sprite image:
embedding.sprite.image_path = os.path.join(LOG_DIR, 'mnist_10k_sprite.png')
embedding.sprite.single_image_dim.extend([28, 28]) # image size = 28x28
projector.visualize_embeddings(summary_writer, config)
saver = tf.train.Saver([embedding_var])
saver.save(sess, os.path.join(LOG_DIR, 'model2.ckpt'), 1)
然后，在你的 LOG_DIR 中 运行 Tensorboard，并按标签选择颜色，这就是你得到的：
修改它以获得对其他层的预测很简单，尽管在这种情况下，Keras Functional API 可能是更好的选择。
我为 MNIST 数据库编写了以下简单的 MLP 网络。
from __future__ import print_function
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras import callbacks
batch_size = 100
num_classes = 10
epochs = 20
tb = callbacks.TensorBoard(log_dir='/Users/shlomi.shwartz/tensorflow/notebooks/logs/minist', histogram_freq=10, batch_size=32,
write_graph=True, write_grads=True, write_images=True,
embeddings_freq=10, embeddings_layer_names=None,
embeddings_metadata=None)
early_stop = callbacks.EarlyStopping(monitor='val_loss', min_delta=0,
patience=3, verbose=1, mode='auto')
# the data, shuffled and split between train and test sets
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(60000, 784)
x_test = x_test.reshape(10000, 784)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(Dense(200, activation='relu', input_shape=(784,)))
model.add(Dropout(0.2))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(60, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(30, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = model.fit(x_train, y_train,
callbacks=[tb,early_stop],
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
模型 运行 很好，我可以在 TensorBoard 上看到标量信息。但是，当我更改 embeddings_freq=10 以尝试可视化图像（如 seen here）时，我收到以下错误：
Traceback (most recent call last):
File "/Users/shlomi.shwartz/IdeaProjects/TF/src/minist.py", line 65, in <module>
validation_data=(x_test, y_test))
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/models.py", line 870, in fit
initial_epoch=initial_epoch)
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/engine/training.py", line 1507, in fit
initial_epoch=initial_epoch)
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/engine/training.py", line 1117, in _fit_loop
callbacks.set_model(callback_model)
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/callbacks.py", line 52, in set_model
callback.set_model(model)
File "/Users/shlomi.shwartz/tensorflow/lib/python3.6/site-packages/keras/callbacks.py", line 719, in set_model
self.saver = tf.train.Saver(list(embeddings.values()))
File "/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/training/saver.py", line 1139, in __init__
self.build()
File "/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/tensorflow/python/training/saver.py", line 1161, in build
raise ValueError("No variables to save")
ValueError: No variables to save
问：我错过了什么？在 Keras 中这是正确的做法吗？
更新： 我知道使用嵌入投影有一些先决条件，但是我没有在 Keras 中找到这样做的好教程，任何帮助将不胜感激.
您至少需要一个 Keras 中的嵌入层。关于统计数据是对它们的一个很好的解释。它不是直接针对 Keras，但概念大致相同。 What is an embedding layer in a neural network
callbacks.TensorBoard这里所谓的"embedding"就是广义上的任意层权重。根据 Keras documentation:
embeddings_layer_names: a list of names of layers to keep eye on. If None or empty list all the embedding layer will be watched.
所以 默认情况下 ，它会监控 Embedding 层，但你并不真的需要 Embedding 层来使用这个可视化工具.
在您提供的 MLP 示例中，缺少的是 embeddings_layer_names 参数。你必须弄清楚你要可视化哪些层。假设您想可视化所有 Dense 层的权重（或者，kernel 在 Keras 中），您可以像这样指定 embeddings_layer_names：
model = Sequential()
model.add(Dense(200, activation='relu', input_shape=(784,)))
model.add(Dropout(0.2))
model.add(Dense(100, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(60, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(30, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
embedding_layer_names = set(layer.name
for layer in model.layers
if layer.name.startswith('dense_'))
tb = callbacks.TensorBoard(log_dir='temp', histogram_freq=10, batch_size=32,
write_graph=True, write_grads=True, write_images=True,
embeddings_freq=10, embeddings_metadata=None,
embeddings_layer_names=embedding_layer_names)
model.compile(...)
model.fit(...)
然后，你可以在TensorBoard中看到类似这样的东西：
如果您想了解关于 embeddings_layer_names 的情况，您可以查看 the relevant lines in Keras source。
编辑：
所以这是可视化层输出的肮脏解决方案。由于原始 TensorBoard 回调不支持此功能，因此实施新的回调似乎是不可避免的。
由于在这里重写整个TensorBoard回调会占用很多页面space，所以我只是扩展原来的TensorBoard，写出不同的部分（已经很长了）。但为了避免重复计算和模型保存，重写 TensorBoard 回调将是一个更好更简洁的方法。
import tensorflow as tf
from tensorflow.contrib.tensorboard.plugins import projector
from keras import backend as K
from keras.models import Model
from keras.callbacks import TensorBoard
class TensorResponseBoard(TensorBoard):
def __init__(self, val_size, img_path, img_size, **kwargs):
super(TensorResponseBoard, self).__init__(**kwargs)
self.val_size = val_size
self.img_path = img_path
self.img_size = img_size
def set_model(self, model):
super(TensorResponseBoard, self).set_model(model)
if self.embeddings_freq and self.embeddings_layer_names:
embeddings = {}
for layer_name in self.embeddings_layer_names:
# initialize tensors which will later be used in `on_epoch_end()` to
# store the response values by feeding the val data through the model
layer = self.model.get_layer(layer_name)
output_dim = layer.output.shape[-1]
response_tensor = tf.Variable(tf.zeros([self.val_size, output_dim]),
name=layer_name + '_response')
embeddings[layer_name] = response_tensor
self.embeddings = embeddings
self.saver = tf.train.Saver(list(self.embeddings.values()))
response_outputs = [self.model.get_layer(layer_name).output
for layer_name in self.embeddings_layer_names]
self.response_model = Model(self.model.inputs, response_outputs)
config = projector.ProjectorConfig()
embeddings_metadata = {layer_name: self.embeddings_metadata
for layer_name in embeddings.keys()}
for layer_name, response_tensor in self.embeddings.items():
embedding = config.embeddings.add()
embedding.tensor_name = response_tensor.name
# for coloring points by labels
embedding.metadata_path = embeddings_metadata[layer_name]
# for attaching images to the points
embedding.sprite.image_path = self.img_path
embedding.sprite.single_image_dim.extend(self.img_size)
projector.visualize_embeddings(self.writer, config)
def on_epoch_end(self, epoch, logs=None):
super(TensorResponseBoard, self).on_epoch_end(epoch, logs)
if self.embeddings_freq and self.embeddings_ckpt_path:
if epoch % self.embeddings_freq == 0:
# feeding the validation data through the model
val_data = self.validation_data[0]
response_values = self.response_model.predict(val_data)
if len(self.embeddings_layer_names) == 1:
response_values = [response_values]
# record the response at each layers we're monitoring
response_tensors = []
for layer_name in self.embeddings_layer_names:
response_tensors.append(self.embeddings[layer_name])
K.batch_set_value(list(zip(response_tensors, response_values)))
# finally, save all tensors holding the layer responses
self.saver.save(self.sess, self.embeddings_ckpt_path, epoch)
要使用它：
tb = TensorResponseBoard(log_dir=log_dir, histogram_freq=10, batch_size=10,
write_graph=True, write_grads=True, write_images=True,
embeddings_freq=10,
embeddings_layer_names=['dense_1'],
embeddings_metadata='metadata.tsv',
val_size=len(x_test), img_path='images.jpg', img_size=[28, 28])
在启动 TensorBoard 之前，您需要将标签和图像保存到 log_dir 以便可视化：
from PIL import Image
img_array = x_test.reshape(100, 100, 28, 28)
img_array_flat = np.concatenate([np.concatenate([x for x in row], axis=1) for row in img_array])
img = Image.fromarray(np.uint8(255 * (1. - img_array_flat)))
img.save(os.path.join(log_dir, 'images.jpg'))
np.savetxt(os.path.join(log_dir, 'metadata.tsv'), np.where(y_test)[1], fmt='%d')
结果如下：
所以，我得出结论，您真正想要的（从您的 post 中还不完全清楚）是以类似的方式可视化模型的 预测 至 this Tensorboard demo.
首先，复制这些东西非常重要 even in Tensorflow, let alone Keras. The said demo makes very brief and passing references to things like metadata & sprite images，这是获得此类可视化效果所必需的。
底线：虽然不简单，但使用 Keras 确实可以做到。您不需要 Keras 回调；您只需要模型预测、必要的元数据和精灵图像，以及一些纯 TensorFlow 代码。所以，
第 1 步 - 获取测试集的模型预测：
emb = model.predict(x_test) # 'emb' for embedding
步骤 2a - 使用测试集的真实标签构建元数据文件：
import numpy as np
LOG_DIR = '/home/herc/SO/tmp' # FULL PATH HERE!!!
metadata_file = os.path.join(LOG_DIR, 'metadata.tsv')
with open(metadata_file, 'w') as f:
for i in range(len(y_test)):
c = np.nonzero(y_test[i])[0][0]
f.write('{}\n'.format(c))
步骤 2b - 获取 TensorFlow 人员 here 提供的精灵图像 mnist_10k_sprite.png，并将其放入您的 LOG_DIR
步骤 3 - 编写一些 Tensorflow 代码：
import tensorflow as tf
from tensorflow.contrib.tensorboard.plugins import projector
embedding_var = tf.Variable(emb, name='final_layer_embedding')
sess = tf.Session()
sess.run(embedding_var.initializer)
summary_writer = tf.summary.FileWriter(LOG_DIR)
config = projector.ProjectorConfig()
embedding = config.embeddings.add()
embedding.tensor_name = embedding_var.name
# Specify the metadata file:
embedding.metadata_path = os.path.join(LOG_DIR, 'metadata.tsv')
# Specify the sprite image:
embedding.sprite.image_path = os.path.join(LOG_DIR, 'mnist_10k_sprite.png')
embedding.sprite.single_image_dim.extend([28, 28]) # image size = 28x28
projector.visualize_embeddings(summary_writer, config)
saver = tf.train.Saver([embedding_var])
saver.save(sess, os.path.join(LOG_DIR, 'model2.ckpt'), 1)
然后，在你的 LOG_DIR 中 运行 Tensorboard，并按标签选择颜色，这就是你得到的：
修改它以获得对其他层的预测很简单，尽管在这种情况下，Keras Functional API 可能是更好的选择。