error_ops.py revision 98ef53d5541049655c9160130595253fdefd4590
1c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# 3c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# Licensed under the Apache License, Version 2.0 (the "License"); 4c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# you may not use this file except in compliance with the License. 5c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# You may obtain a copy of the License at 6c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# 7c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# http://www.apache.org/licenses/LICENSE-2.0 8c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# 9c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# Unless required by applicable law or agreed to in writing, software 10c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# distributed under the License is distributed on an "AS IS" BASIS, 11c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# See the License for the specific language governing permissions and 13c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# limitations under the License. 14c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes# ============================================================================== 15c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes"""Ignore_errors dataset transformations.""" 16c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughesfrom __future__ import absolute_import 17c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughesfrom __future__ import division 18c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughesfrom __future__ import print_function 19e2f147b9b14c7645f29e92758f811a18258feef4Yi Kong 20adb70766cf4d670bd9837c944de801061493038bCalin Juravlefrom tensorflow.python.data.ops import dataset_ops 21adb70766cf4d670bd9837c944de801061493038bCalin Juravlefrom tensorflow.python.data.util import nest 22c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughesfrom tensorflow.python.data.util import sparse 23adb70766cf4d670bd9837c944de801061493038bCalin Juravlefrom tensorflow.python.ops import gen_dataset_ops 246b40837ee3a023bba698c38fd6d6e46ae0065a55Calin Juravle 25adb70766cf4d670bd9837c944de801061493038bCalin Juravle 26adb70766cf4d670bd9837c944de801061493038bCalin Juravledef ignore_errors(): 27adb70766cf4d670bd9837c944de801061493038bCalin Juravle """Creates a `Dataset` from another `Dataset` and silently ignores any errors. 28adb70766cf4d670bd9837c944de801061493038bCalin Juravle 29adb70766cf4d670bd9837c944de801061493038bCalin Juravle Use this transformation to produce a dataset that contains the same elements 30adb70766cf4d670bd9837c944de801061493038bCalin Juravle as the input, but silently drops any elements that caused an error. For 31adb70766cf4d670bd9837c944de801061493038bCalin Juravle example: 32adb70766cf4d670bd9837c944de801061493038bCalin Juravle 33adb70766cf4d670bd9837c944de801061493038bCalin Juravle ```python 34adb70766cf4d670bd9837c944de801061493038bCalin Juravle dataset = tf.data.Dataset.from_tensor_slices([1., 2., 0., 4.]) 35adb70766cf4d670bd9837c944de801061493038bCalin Juravle 36adb70766cf4d670bd9837c944de801061493038bCalin Juravle # Computing `tf.check_numerics(1. / 0.)` will raise an InvalidArgumentError. 37adb70766cf4d670bd9837c944de801061493038bCalin Juravle dataset = dataset.map(lambda x: tf.check_numerics(1. / x, "error")) 386b40837ee3a023bba698c38fd6d6e46ae0065a55Calin Juravle 39c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes # Using `ignore_errors()` will drop the element that causes an error. 40c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes dataset = 41c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes dataset.apply(tf.contrib.data.ignore_errors()) # ==> { 1., 0.5, 0.2 } 42c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes ``` 43c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes 44c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes Returns: 45c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes A `Dataset` transformation function, which can be passed to 46c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes @{tf.data.Dataset.apply}. 47c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes """ 48c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes 49c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes def _apply_fn(dataset): 50c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes return IgnoreErrorsDataset(dataset) 51c3adcfb1339de546aed6e7ff00a7edb020b8a85bElliott Hughes 52769ff93bf19951ac9a345e783927132438aabf3cElliott Hughes return _apply_fn 536b40837ee3a023bba698c38fd6d6e46ae0065a55Calin Juravle 54769ff93bf19951ac9a345e783927132438aabf3cElliott Hughes 55769ff93bf19951ac9a345e783927132438aabf3cElliott Hughesclass IgnoreErrorsDataset(dataset_ops.Dataset): 56769ff93bf19951ac9a345e783927132438aabf3cElliott Hughes """A `Dataset` that silently ignores errors when computing its input.""" 57769ff93bf19951ac9a345e783927132438aabf3cElliott Hughes 58769ff93bf19951ac9a345e783927132438aabf3cElliott Hughes def __init__(self, input_dataset): 59769ff93bf19951ac9a345e783927132438aabf3cElliott Hughes """See `Dataset.ignore_errors()` for details.""" 60ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle super(IgnoreErrorsDataset, self).__init__() 6146e3649f898305d185fc40bae2542e933a16b619Piotr Jastrzebski self._input_dataset = input_dataset 626b40837ee3a023bba698c38fd6d6e46ae0065a55Calin Juravle 63ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle def _as_variant_tensor(self): 64ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle return gen_dataset_ops.ignore_errors_dataset( 65ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle self._input_dataset._as_variant_tensor(), # pylint: disable=protected-access 66ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle output_shapes=nest.flatten( 67ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle sparse.as_dense_shapes(self.output_shapes, self.output_classes)), 68ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle output_types=nest.flatten( 69ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle sparse.as_dense_types(self.output_types, self.output_classes))) 70ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle 71ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle @property 72ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle def output_classes(self): 73ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle return self._input_dataset.output_classes 74ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle 75ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle @property 76ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle def output_shapes(self): 77ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle return self._input_dataset.output_shapes 78ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle 79ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle @property 80ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle def output_types(self): 81ae6e60bf18b455b1134b8b57a56b37cc588e179dCalin Juravle return self._input_dataset.output_types 8246e3649f898305d185fc40bae2542e933a16b619Piotr Jastrzebski