__init__.py revision a274c662b3f090193ead4138791896ffb65d680e 
1# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14# ==============================================================================
15"""##Ops for evaluation metrics and summary statistics.
16
17### API
18
19This module provides functions for computing streaming metrics: metrics computed
20on dynamically valued `Tensors`. Each metric declaration returns a
21"value_tensor", an idempotent operation that returns the current value of the
22metric, and an "update_op", an operation that accumulates the information
23from the current value of the `Tensors` being measured as well as returns the
24value of the "value_tensor".
25
26To use any of these metrics, one need only declare the metric, call `update_op`
27repeatedly to accumulate data over the desired number of `Tensor` values (often
28each one is a single batch) and finally evaluate the value_tensor. For example,
29to use the `streaming_mean`:
30
31```python
32value = ...
33mean_value, update_op = tf.contrib.metrics.streaming_mean(values)
34sess.run(tf.initialize_local_variables())
35
36for i in range(number_of_batches):
37  print('Mean after batch %d: %f' % (i, update_op.eval())
38print('Final Mean: %f' % mean_value.eval())
39```
40
41Each metric function adds nodes to the graph that hold the state necessary to
42compute the value of the metric as well as a set of operations that actually
43perform the computation. Every metric evaluation is composed of three steps
44
45* Initialization: initializing the metric state.
46* Aggregation: updating the values of the metric state.
47* Finalization: computing the final metric value.
48
49In the above example, calling streaming_mean creates a pair of state variables
50that will contain (1) the running sum and (2) the count of the number of samples
51in the sum.  Because the streaming metrics use local variables,
52the Initialization stage is performed by running the op returned
53by `tf.initialize_local_variables()`. It sets the sum and count variables to
54zero.
55
56Next, Aggregation is performed by examining the current state of `values`
57and incrementing the state variables appropriately. This step is executed by
58running the `update_op` returned by the metric.
59
60Finally, finalization is performed by evaluating the "value_tensor"
61
62In practice, we commonly want to evaluate across many batches and multiple
63metrics. To do so, we need only run the metric computation operations multiple
64times:
65
66```python
67labels = ...
68predictions = ...
69accuracy, update_op_acc = tf.contrib.metrics.streaming_accuracy(
70    labels, predictions)
71error, update_op_error = tf.contrib.metrics.streaming_mean_absolute_error(
72    labels, predictions)
73
74sess.run(tf.initialize_local_variables())
75for batch in range(num_batches):
76  sess.run([update_op_acc, update_op_error])
77
78accuracy, mean_absolute_error = sess.run([accuracy, mean_absolute_error])
79```
80
81Note that when evaluating the same metric multiple times on different inputs,
82one must specify the scope of each metric to avoid accumulating the results
83together:
84
85```python
86labels = ...
87predictions0 = ...
88predictions1 = ...
89
90accuracy0 = tf.contrib.metrics.accuracy(labels, predictions0, name='preds0')
91accuracy1 = tf.contrib.metrics.accuracy(labels, predictions1, name='preds1')
92```
93
94Certain metrics, such as streaming_mean or streaming_accuracy, can be weighted
95via a `weights` argument. The `weights` tensor must be the same size as the
96labels and predictions tensors and results in a weighted average of the metric.
97
98Other metrics, such as streaming_recall, streaming_precision, and streaming_auc,
99are not well defined with regard to weighted samples. However, a binary
100`ignore_mask` argument can be used to ignore certain values at graph executation
101time.
102
103## Metric `Ops`
104
105@@streaming_accuracy
106@@streaming_mean
107@@streaming_recall
108@@streaming_precision
109@@streaming_auc
110@@streaming_recall_at_k
111@@streaming_mean_absolute_error
112@@streaming_mean_iou
113@@streaming_mean_relative_error
114@@streaming_mean_squared_error
115@@streaming_root_mean_squared_error
116@@streaming_mean_cosine_distance
117@@streaming_percentage_less
118@@streaming_sensitivity_at_specificity
119@@streaming_sparse_precision_at_k
120@@streaming_sparse_recall_at_k
121@@streaming_specificity_at_sensitivity
122
123@@auc_using_histogram
124
125@@accuracy
126@@confusion_matrix
127
128@@aggregate_metrics
129@@aggregate_metric_map
130
131## Set `Ops`
132
133@@set_difference
134@@set_intersection
135@@set_size
136@@set_union
137
138"""
139from __future__ import absolute_import
140from __future__ import division
141from __future__ import print_function
142
143# pylint: disable=unused-import,line-too-long,g-importing-member,wildcard-import
144from tensorflow.contrib.metrics.python.metrics import *
145from tensorflow.contrib.metrics.python.ops.confusion_matrix_ops import confusion_matrix
146from tensorflow.contrib.metrics.python.ops.histogram_ops import auc_using_histogram
147from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metric_map
148from tensorflow.contrib.metrics.python.ops.metric_ops import aggregate_metrics
149from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_accuracy
150from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_auc
151from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean
152from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_absolute_error
153from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_cosine_distance
154from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_iou
155from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_relative_error
156from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_squared_error
157from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_mean_tensor
158from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_percentage_less
159from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_precision
160from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_precision_at_thresholds
161from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_recall
162from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_recall_at_k
163from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_recall_at_thresholds
164from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_root_mean_squared_error
165from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_sensitivity_at_specificity
166from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_sparse_precision_at_k
167from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_sparse_recall_at_k
168from tensorflow.contrib.metrics.python.ops.metric_ops import streaming_specificity_at_sensitivity
169from tensorflow.contrib.metrics.python.ops.set_ops import set_difference
170from tensorflow.contrib.metrics.python.ops.set_ops import set_intersection
171from tensorflow.contrib.metrics.python.ops.set_ops import set_size
172from tensorflow.contrib.metrics.python.ops.set_ops import set_union
173from tensorflow.python.util.all_util import make_all
174
175
176__all__ = make_all(__name__)
177