# Copyright 2015 The TensorFlow Authors. All Rights Reserved.

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

# http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

# =============================================================================

"""Implementation of Neural Net (NN) functions."""

from __future__ import absolute_import

from __future__ import division

from __future__ import print_function

import math

from tensorflow.python.compat import compat

from tensorflow.python.distribute import distribution_strategy_context as ds

from tensorflow.python.framework import constant_op

from tensorflow.python.framework import dtypes

from tensorflow.python.framework import ops

from tensorflow.python.ops import array_ops

from tensorflow.python.ops import candidate_sampling_ops

from tensorflow.python.ops import control_flow_ops

from tensorflow.python.ops import custom_gradient

from tensorflow.python.ops import embedding_ops

from tensorflow.python.ops import gen_array_ops # pylint: disable=unused-import

from tensorflow.python.ops import gen_nn_ops

from tensorflow.python.ops import linalg_ops

from tensorflow.python.ops import math_ops

from tensorflow.python.ops import nn_ops

from tensorflow.python.ops import gen_sparse_ops

from tensorflow.python.ops import variables

from tensorflow.python.ops.losses import util as losses_util

from tensorflow.python.util.deprecation import deprecated_args

from tensorflow.python.util.deprecation import deprecated_argument_lookup

from tensorflow.python.util.tf_export import tf_export

@tf_export("nn.log_poisson_loss")

def log_poisson_loss(targets, log_input, compute_full_loss=False, name=None):

"""Computes log Poisson loss given `log_input`.

Gives the log-likelihood loss between the prediction and the target under the

assumption that the target has a Poisson distribution.

Caveat: By default, this is not the exact loss, but the loss minus a

constant term [log(z!)]. That has no effect for optimization, but

does not play well with relative loss comparisons. To compute an

approximation of the log factorial term, specify

compute_full_loss=True to enable Stirling's Approximation.

For brevity, let `c = log(x) = log_input`, `z = targets`. The log Poisson

loss is

-log(exp(-x) * (x^z) / z!)

= -log(exp(-x) * (x^z)) + log(z!)

~ -log(exp(-x)) - log(x^z) [+ z * log(z) - z + 0.5 * log(2 * pi * z)]

[ Note the second term is the Stirling's Approximation for log(z!).

It is invariant to x and does not affect optimization, though

important for correct relative loss comparisons. It is only

computed when compute_full_loss == True. ]

= x - z * log(x) [+ z * log(z) - z + 0.5 * log(2 * pi * z)]

= exp(c) - z * c [+ z * log(z) - z + 0.5 * log(2 * pi * z)]

Args:

targets: A `Tensor` of the same type and shape as `log_input`.

log_input: A `Tensor` of type `float32` or `float64`.

compute_full_loss: whether to compute the full loss. If false, a constant

term is dropped in favor of more efficient optimization.

name: A name for the operation (optional).

Returns:

A `Tensor` of the same shape as `log_input` with the componentwise

logistic losses.

Raises:

ValueError: If `log_input` and `targets` do not have the same shape.

"""

with ops.name_scope(name, "log_poisson_loss", [log_input, targets]) as name:

log_input = ops.convert_to_tensor(log_input, name="log_input")

targets = ops.convert_to_tensor(targets, name="targets")

try:

targets.get_shape().merge_with(log_input.get_shape())

except ValueError:

raise ValueError(

"log_input and targets must have the same shape (%s vs %s)" %

(log_input.get_shape(), targets.get_shape()))

result = math_ops.exp(log_input) - log_input * targets

if compute_full_loss:

# need to create constant tensors here so that their dtypes can be matched

# to that of the targets.

point_five = constant_op.constant(0.5, dtype=targets.dtype)

two_pi = constant_op.constant(2 * math.pi, dtype=targets.dtype)

stirling_approx = (targets * math_ops.log(targets)) - targets + (

point_five * math_ops.log(two_pi * targets))

zeros = array_ops.zeros_like(targets, dtype=targets.dtype)

ones = array_ops.ones_like(targets, dtype=targets.dtype)

cond = math_ops.logical_and(targets >= zeros, targets <= ones)

result += array_ops.where(cond, zeros, stirling_approx)

return result

@tf_export(v1=["nn.sigmoid_cross_entropy_with_logits"])

def sigmoid_cross_entropy_with_logits( # pylint: disable=invalid-name

_sentinel=None,

labels=None,

logits=None,

name=None):

"""Computes sigmoid cross entropy given `logits`.

Measures the probability error in discrete classification tasks in which each

class is independent and not mutually exclusive. For instance, one could

perform multilabel classification where a picture can contain both an elephant

and a dog at the same time.

For brevity, let `x = logits`, `z = labels`. The logistic loss is

z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))

= z * -log(1 / (1 + exp(-x))) + (1 - z) * -log(exp(-x) / (1 + exp(-x)))

= z * log(1 + exp(-x)) + (1 - z) * (-log(exp(-x)) + log(1 + exp(-x)))

= z * log(1 + exp(-x)) + (1 - z) * (x + log(1 + exp(-x))

= (1 - z) * x + log(1 + exp(-x))

= x - x * z + log(1 + exp(-x))

For x < 0, to avoid overflow in exp(-x), we reformulate the above

x - x * z + log(1 + exp(-x))

= log(exp(x)) - x * z + log(1 + exp(-x))

= - x * z + log(1 + exp(x))

Hence, to ensure stability and avoid overflow, the implementation uses this

equivalent formulation

max(x, 0) - x * z + log(1 + exp(-abs(x)))

`logits` and `labels` must have the same type and shape.

Args:

_sentinel: Used to prevent positional parameters. Internal, do not use.

labels: A `Tensor` of the same type and shape as `logits`.

logits: A `Tensor` of type `float32` or `float64`.

name: A name for the operation (optional).

Returns:

A `Tensor` of the same shape as `logits` with the componentwise

logistic losses.

Raises:

ValueError: If `logits` and `labels` do not have the same shape.

"""

# pylint: disable=protected-access

nn_ops._ensure_xent_args("sigmoid_cross_entropy_with_logits", _sentinel,

labels, logits)

# pylint: enable=protected-access

with ops.name_scope(name, "logistic_loss", [logits, labels]) as name:

logits = ops.convert_to_tensor(logits, name="logits")

labels = ops.convert_to_tensor(labels, name="labels")

try:

labels.get_shape().merge_with(logits.get_shape())

except ValueError:

raise ValueError("logits and labels must have the same shape (%s vs %s)" %

(logits.get_shape(), labels.get_shape()))

# The logistic loss formula from above is

# x - x * z + log(1 + exp(-x))

# For x < 0, a more numerically stable formula is

# -x * z + log(1 + exp(x))

# Note that these two expressions can be combined into the following:

# max(x, 0) - x * z + log(1 + exp(-abs(x)))

# To allow computing gradients at zero, we define custom versions of max and

# abs functions.

zeros = array_ops.zeros_like(logits, dtype=logits.dtype)

cond = (logits >= zeros)

relu_logits = array_ops.where(cond, logits, zeros)

neg_abs_logits = array_ops.where(cond, -logits, logits)

return math_ops.add(

relu_logits - logits * labels,

math_ops.log1p(math_ops.exp(neg_abs_logits)),

name=name)

# Note: intentionally calling this v2 to not allow existing code with indirect

# imports to ignore the sentinel behavior.

@tf_export("nn.sigmoid_cross_entropy_with_logits", v1=[])

def sigmoid_cross_entropy_with_logits_v2( # pylint: disable=invalid-name

labels=None,

logits=None,

name=None):

"""Computes sigmoid cross entropy given `logits`.

Measures the probability error in discrete classification tasks in which each

class is independent and not mutually exclusive. For instance, one could

perform multilabel classification where a picture can contain both an elephant

and a dog at the same time.

For brevity, let `x = logits`, `z = labels`. The logistic loss is

z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))

= z * -log(1 / (1 + exp(-x))) + (1 - z) * -log(exp(-x) / (1 + exp(-x)))

= z * log(1 + exp(-x)) + (1 - z) * (-log(exp(-x)) + log(1 + exp(-x)))

= z * log(1 + exp(-x)) + (1 - z) * (x + log(1 + exp(-x))

= (1 - z) * x + log(1 + exp(-x))

= x - x * z + log(1 + exp(-x))

For x < 0, to avoid overflow in exp(-x), we reformulate the above

x - x * z + log(1 + exp(-x))

= log(exp(x)) - x * z + log(1 + exp(-x))

= - x * z + log(1 + exp(x))

Hence, to ensure stability and avoid overflow, the implementation uses this

equivalent formulation

max(x, 0) - x * z + log(1 + exp(-abs(x)))

`logits` and `labels` must have the same type and shape.

Args:

labels: A `Tensor` of the same type and shape as `logits`.

logits: A `Tensor` of type `float32` or `float64`.

name: A name for the operation (optional).

Returns:

A `Tensor` of the same shape as `logits` with the componentwise

logistic losses.

Raises:

ValueError: If `logits` and `labels` do not have the same shape.

"""

return sigmoid_cross_entropy_with_logits(

logits=logits, labels=labels, name=name)

@tf_export("nn.weighted_cross_entropy_with_logits", v1=[])

def weighted_cross_entropy_with_logits_v2(labels, logits, pos_weight,

name=None):

"""Computes a weighted cross entropy.

This is like `sigmoid_cross_entropy_with_logits()` except that `pos_weight`,

allows one to trade off recall and precision by up- or down-weighting the

cost of a positive error relative to a negative error.

The usual cross-entropy cost is defined as:

labels * -log(sigmoid(logits)) +

(1 - labels) * -log(1 - sigmoid(logits))

A value `pos_weight > 1` decreases the false negative count, hence increasing

the recall.

Conversely setting `pos_weight < 1` decreases the false positive count and

increases the precision.

This can be seen from the fact that `pos_weight` is introduced as a

multiplicative coefficient for the positive labels term

in the loss expression:

labels * -log(sigmoid(logits)) * pos_weight +

(1 - labels) * -log(1 - sigmoid(logits))

For brevity, let `x = logits`, `z = labels`, `q = pos_weight`.

The loss is:

qz * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))

= qz * -log(1 / (1 + exp(-x))) + (1 - z) * -log(exp(-x) / (1 + exp(-x)))

= qz * log(1 + exp(-x)) + (1 - z) * (-log(exp(-x)) + log(1 + exp(-x)))

= qz * log(1 + exp(-x)) + (1 - z) * (x + log(1 + exp(-x))

= (1 - z) * x + (qz + 1 - z) * log(1 + exp(-x))

= (1 - z) * x + (1 + (q - 1) * z) * log(1 + exp(-x))

Setting `l = (1 + (q - 1) * z)`, to ensure stability and avoid overflow,

the implementation uses

(1 - z) * x + l * (log(1 + exp(-abs(x))) + max(-x, 0))

`logits` and `labels` must have the same type and shape.

Args:

labels: A `Tensor` of the same type and shape as `logits`.

logits: A `Tensor` of type `float32` or `float64`.

pos_weight: A coefficient to use on the positive examples.

name: A name for the operation (optional).

Returns:

A `Tensor` of the same shape as `logits` with the componentwise

weighted logistic losses.

Raises:

ValueError: If `logits` and `labels` do not have the same shape.

"""

with ops.name_scope(name, "logistic_loss", [logits, labels]) as name:

logits = ops.convert_to_tensor(logits, name="logits")

labels = ops.convert_to_tensor(labels, name="labels")

try:

labels.get_shape().merge_with(logits.get_shape())

except ValueError:

raise ValueError("logits and labels must have the same shape (%s vs %s)" %

(logits.get_shape(), labels.get_shape()))

# The logistic loss formula from above is

# (1 - z) * x + (1 + (q - 1) * z) * log(1 + exp(-x))

# For x < 0, a more numerically stable formula is

# (1 - z) * x + (1 + (q - 1) * z) * log(1 + exp(x)) - l * x

# To avoid branching, we use the combined version

# (1 - z) * x + l * (log(1 + exp(-abs(x))) + max(-x, 0))

log_weight = 1 + (pos_weight - 1) * labels

return math_ops.add(

(1 - labels) * logits,

log_weight * (math_ops.log1p(math_ops.exp(-math_ops.abs(logits))) +

nn_ops.relu(-logits)),

name=name)

@tf_export(v1=["nn.weighted_cross_entropy_with_logits"])

@deprecated_args(None, "targets is deprecated, use labels instead", "targets")

def weighted_cross_entropy_with_logits(labels=None,

logits=None,

pos_weight=None,

name=None,

targets=None):

"""Computes a weighted cross entropy.

This is like `sigmoid_cross_entropy_with_logits()` except that `pos_weight`,

allows one to trade off recall and precision by up- or down-weighting the

cost of a positive error relative to a negative error.

The usual cross-entropy cost is defined as:

labels * -log(sigmoid(logits)) +

(1 - labels) * -log(1 - sigmoid(logits))

A value `pos_weight > 1` decreases the false negative count, hence increasing

the recall.

Conversely setting `pos_weight < 1` decreases the false positive count and

increases the precision.

This can be seen from the fact that `pos_weight` is introduced as a

multiplicative coefficient for the positive labels term

in the loss expression:

labels * -log(sigmoid(logits)) * pos_weight +

(1 - labels) * -log(1 - sigmoid(logits))

For brevity, let `x = logits`, `z = labels`, `q = pos_weight`.

The loss is:

qz * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))

= qz * -log(1 / (1 + exp(-x))) + (1 - z) * -log(exp(-x) / (1 + exp(-x)))

= qz * log(1 + exp(-x)) + (1 - z) * (-log(exp(-x)) + log(1 + exp(-x)))

= qz * log(1 + exp(-x)) + (1 - z) * (x + log(1 + exp(-x))

= (1 - z) * x + (qz + 1 - z) * log(1 + exp(-x))

= (1 - z) * x + (1 + (q - 1) * z) * log(1 + exp(-x))

Setting `l = (1 + (q - 1) * z)`, to ensure stability and avoid overflow,

the implementation uses

(1 - z) * x + l * (log(1 + exp(-abs(x))) + max(-x, 0))

`logits` and `labels` must have the same type and shape.

Args:

labels: A `Tensor` of the same type and shape as `logits`.

logits: A `Tensor` of type `float32` or `float64`.

pos_weight: A coefficient to use on the positive examples.

name: A name for the operation (optional).

targets: Deprecated alias for labels.

Returns:

A `Tensor` of the same shape as `logits` with the componentwise

weighted logistic losses.

Raises:

ValueError: If `logits` and `labels` do not have the same shape.

"""

labels = deprecated_argument_lookup("labels", labels, "targets", targets)

return weighted_cross_entropy_with_logits_v2(labels, logits, pos_weight, name)

@tf_export("nn.compute_average_loss")

def compute_average_loss(per_example_loss,

sample_weight=None,

global_batch_size=None):

"""Scales per-example losses with sample_weights and computes their average.

Usage with distribution strategy and custom training loop:

```python

with strategy.scope():

def compute_loss(labels, predictions, sample_weight=None):

# If you are using a `Loss` class instead, set reduction to `NONE` so that

# we can do the reduction afterwards and divide by global batch size.

per_example_loss = tf.keras.losses.sparse_categorical_crossentropy(

labels, predictions)

# Compute loss that is scaled by sample_weight and by global batch size.

return tf.compute_average_loss(

per_example_loss,

sample_weight=sample_weight,

global_batch_size=GLOBAL_BATCH_SIZE)

```

Args:

per_example_loss: Per-example loss.

sample_weight: Optional weighting for each example.

global_batch_size: Optional global batch size value. Defaults to (size of

first dimension of `losses`) * (number of replicas).

Returns:

Scalar loss value.

""" # pylint: disable=g-doc-exception

per_example_loss = ops.convert_to_tensor(per_example_loss)

input_dtype = per_example_loss.dtype

with losses_util.check_per_example_loss_rank(per_example_loss):

if sample_weight is not None:

per_example_loss = losses_util.scale_losses_by_sample_weight(

per_example_loss, sample_weight)

per_example_loss = math_ops.cast(per_example_loss, input_dtype)

if global_batch_size is None:

if ds.has_strategy() and ds.in_cross_replica_context():

raise RuntimeError(

"You are calling `compute_average_loss` in cross replica context, "

"while it was expected to be called in replica context.")

num_replicas = ds.get_strategy().num_replicas_in_sync

per_replica_batch_size = array_ops.shape_v2(per_example_loss)[0]

global_batch_size = per_replica_batch_size * num_replicas

global_batch_size = math_ops.cast(global_batch_size, input_dtype)

return math_ops.reduce_sum(per_example_loss) / global_batch_size

@tf_export("nn.scale_regularization_loss")

def scale_regularization_loss(regularization_loss):

"""Scales the sum of the given regularization losses by number of replicas.

Usage with distribution strategy and custom training loop:

```python

with strategy.scope():

def compute_loss(self, label, predictions):

per_example_loss = tf.keras.losses.sparse_categorical_crossentropy(

labels, predictions)

# Compute loss that is scaled by sample_weight and by global batch size.

loss = tf.compute_average_loss(

per_example_loss,

sample_weight=sample_weight,

global_batch_size=GLOBAL_BATCH_SIZE)

# Add scaled regularization losses.

loss += tf.scale_regularization_loss(tf.nn.l2_loss(weights))

return loss

```

Args:

regularization_loss: Regularization loss.

Returns:

Scalar loss value.

""" # pylint: disable=g-doc-exception

if ds.has_strategy() and ds.in_cross_replica_context():

raise RuntimeError(

"You are calling `scale_regularization_loss` in cross replica context, "

"while it was expected to be called in replica context.")

num_replicas = ds.get_strategy().num_replicas_in_sync

return math_ops.reduce_sum(regularization_loss) / num_replicas

@tf_export(v1=["nn.relu_layer"])

def relu_layer(x, weights, biases, name=None):

"""Computes Relu(x * weight + biases).

Args:

x: a 2D tensor. Dimensions typically: batch, in_units

weights: a 2D tensor. Dimensions typically: in_units, out_units

biases: a 1D tensor. Dimensions: out_units

name: A name for the operation (optional). If not specified

"nn_relu_layer" is used.

Returns:

A 2-D Tensor computing relu(matmul(x, weights) + biases).

Dimensions typically: batch, out_units.

"""

with ops.name_scope(name, "relu_layer", [x, weights, biases]) as name:

x = ops.convert_to_tensor(x, name="x")

weights = ops.convert_to_tensor(weights, name="weights")

biases = ops.convert_to_tensor(biases, name="biases")

xw_plus_b = nn_ops.bias_add(math_ops.matmul(x, weights), biases)

return nn_ops.relu(xw_plus_b, name=name)

@tf_export("nn.swish")

@custom_gradient.custom_gradient

def swish(features):

# pylint: disable=g-doc-args

"""Computes the Swish activation function: `x * sigmoid(x)`.

Source: "Searching for Activation Functions" (Ramachandran et al. 2017)

https://arxiv.org/abs/1710.05941

Args:

features: A `Tensor` representing preactivation values.

name: A name for the operation (optional).

Returns:

The activation value.

"""

# pylint: enable=g-doc-args

features = ops.convert_to_tensor(features, name="features")

def grad(dy):

"""Gradient for the Swish activation function"""

# Naively, x * tf.nn.sigmoid(x) requires keeping both x and sigmoid(x)

# around for backprop, effectively doubling the tensor's memory consumption.

# We use a control dependency here so that sigmoid(features) is re-computed

# during backprop (the control dep prevents it being de-duped with the

# forward pass) and we can free the sigmoid(features) expression immediately

# after use during the forward pass.

with ops.control_dependencies([dy]):

sigmoid_features = math_ops.sigmoid(features)

activation_grad = (

sigmoid_features * (1.0 + features * (1.0 - sigmoid_features)))

return dy * activation_grad

return features * math_ops.sigmoid(features), grad

# pylint: disable=redefined-builtin

@tf_export("linalg.normalize")

def normalize(tensor, ord="euclidean", axis=None, name=None):

"""Normalizes `tensor` along dimension `axis` using specified norm.

This uses `tf.linalg.norm` to compute the norm along `axis`.

This function can compute several different vector norms (the 1-norm, the

Euclidean or 2-norm, the inf-norm, and in general the p-norm for p > 0) and

matrix norms (Frobenius, 1-norm, 2-norm and inf-norm).

Args:

tensor: `Tensor` of types `float32`, `float64`, `complex64`, `complex128`

ord: Order of the norm. Supported values are `'fro'`, `'euclidean'`, `1`,

`2`, `np.inf` and any positive real number yielding the corresponding

p-norm. Default is `'euclidean'` which is equivalent to Frobenius norm if

`tensor` is a matrix and equivalent to 2-norm for vectors.

Some restrictions apply: a) The Frobenius norm `'fro'` is not defined for

vectors, b) If axis is a 2-tuple (matrix norm), only `'euclidean'`,

'`fro'`, `1`, `2`, `np.inf` are supported. See the description of `axis`

on how to compute norms for a batch of vectors or matrices stored in a

tensor.

axis: If `axis` is `None` (the default), the input is considered a vector

and a single vector norm is computed over the entire set of values in the

tensor, i.e. `norm(tensor, ord=ord)` is equivalent to

`norm(reshape(tensor, [-1]), ord=ord)`. If `axis` is a Python integer, the

input is considered a batch of vectors, and `axis` determines the axis in

`tensor` over which to compute vector norms. If `axis` is a 2-tuple of

Python integers it is considered a batch of matrices and `axis` determines

the axes in `tensor` over which to compute a matrix norm.

Negative indices are supported. Example: If you are passing a tensor that

can be either a matrix or a batch of matrices at runtime, pass

`axis=[-2,-1]` instead of `axis=None` to make sure that matrix norms are

computed.

name: The name of the op.

Returns:

normalized: A normalized `Tensor` with the same shape as `tensor`.

norm: The computed norms with the same shape and dtype `tensor` but the

final axis is 1 instead. Same as running

`tf.cast(tf.linalg.norm(tensor, ord, axis keepdims=True), tensor.dtype)`.

Raises:

ValueError: If `ord` or `axis` is invalid.

"""

with ops.name_scope(name, "normalize", [tensor]) as name:

tensor = ops.convert_to_tensor(tensor)

norm = linalg_ops.norm(tensor, ord, axis, keepdims=True)

norm = math_ops.cast(norm, tensor.dtype)

normalized = tensor / norm

return normalized, norm

@tf_export(v1=["math.l2_normalize", "linalg.l2_normalize", "nn.l2_normalize"])

@deprecated_args(None, "dim is deprecated, use axis instead", "dim")

def l2_normalize(x, axis=None, epsilon=1e-12, name=None, dim=None):

"""Normalizes along dimension `axis` using an L2 norm.

For a 1-D tensor with `axis = 0`, computes

output = x / sqrt(max(sum(x**2), epsilon))

For `x` with more dimensions, independently normalizes each 1-D slice along

dimension `axis`.

Args:

x: A `Tensor`.

axis: Dimension along which to normalize. A scalar or a vector of

integers.

epsilon: A lower bound value for the norm. Will use `sqrt(epsilon)` as the

divisor if `norm < sqrt(epsilon)`.

name: A name for this operation (optional).

dim: Deprecated alias for axis.

Returns:

A `Tensor` with the same shape as `x`.

"""

axis = deprecated_argument_lookup("axis", axis, "dim", dim)

return l2_normalize_v2(x, axis, epsilon, name)

@tf_export("math.l2_normalize", "linalg.l2_normalize", "nn.l2_normalize", v1=[])

def l2_normalize_v2(x, axis=None, epsilon=1e-12, name=None):

"""Normalizes along dimension `axis` using an L2 norm.

For a 1-D tensor with `axis = 0`, computes

output = x / sqrt(max(sum(x**2), epsilon))

For `x` with more dimensions, independently normalizes each 1-D slice along

dimension `axis`.

Args:

x: A `Tensor`.

axis: Dimension along which to normalize. A scalar or a vector of

integers.

epsilon: A lower bound value for the norm. Will use `sqrt(epsilon)` as the

divisor if `norm < sqrt(epsilon)`.

name: A name for this operation (optional).

Returns:

A `Tensor` with the same shape as `x`.

"""

with ops.name_scope(name, "l2_normalize", [x]) as name:

x = ops.convert_to_tensor(x, name="x")

square_sum = math_ops.reduce_sum(math_ops.square(x), axis, keepdims=True)

x_inv_norm = math_ops.rsqrt(math_ops.maximum(square_sum, epsilon))

return math_ops.multiply(x, x_inv_norm, name=name)

def _count_nonzero(input_tensor, dtype=dtypes.int64):

"""Same as math_ops.count_nonzero.

The reduction is done in dtype, which can be faster for 32-bit dtypes.

Args:

input_tensor: numeric tensor

dtype: reduction dtype

Returns:

number of nonzero values with type dtype

"""

with ops.name_scope("count_nonzero", values=[input_tensor]):

zero = array_ops.zeros([], dtype=input_tensor.dtype)

nonzero_count = math_ops.reduce_sum(

math_ops.cast(

math_ops.not_equal(input_tensor, zero),

dtype=dtype), name="nonzero_count")

return nonzero_count

@tf_export("math.zero_fraction", "nn.zero_fraction")

def zero_fraction(value, name=None):

"""Returns the fraction of zeros in `value`.

If `value` is empty, the result is `nan`.

This is useful in summaries to measure and report sparsity. For example,

```python

z = tf.nn.relu(...)

summ = tf.compat.v1.summary.scalar('sparsity', tf.nn.zero_fraction(z))

```

Args:

value: A tensor of numeric type.

name: A name for the operation (optional).

Returns:

The fraction of zeros in `value`, with type `float32`.

"""

with ops.name_scope(name, "zero_fraction", [value]):

value = ops.convert_to_tensor(value, name="value")

size = array_ops.size(value, out_type=dtypes.int64)

# If the count is small, we can save memory/CPU with an int32 reduction.

num_nonzero = control_flow_ops.cond(

size <= dtypes.int32.max,

# pylint: disable=g-long-lambda

true_fn=lambda: math_ops.cast(

_count_nonzero(value, dtype=dtypes.int32),

dtype=dtypes.int64),

false_fn=lambda: _count_nonzero(value, dtype=dtypes.int64))

with ops.name_scope("counts_to_fraction"):

num_zero = size - num_nonzero

num_zero_float32 = math_ops.cast(num_zero, dtype=dtypes.float32)

size_float32 = math_ops.cast(size, dtype=dtypes.float32)

zero_fraction_float32 = num_zero_float32 / size_float32

return array_ops.identity(zero_fraction_float32, "fraction")

# copybara:strip_begin

# TODO(b/138808492): Remove code inside copybara

# to make TPU code and CPU code consistent.

def _enclosing_tpu_context():

# pylint: disable=protected-access

context = ops.get_default_graph()._get_control_flow_context()

# pylint: enable=protected-access

while context is not None and not isinstance(

context, control_flow_ops.XLAControlFlowContext):

context = context.outer_context

return context

# copybara:strip_end

# pylint: disable=redefined-builtin

@tf_export(v1=["nn.depthwise_conv2d"])

def depthwise_conv2d(input,

filter,

strides,

padding,

rate=None,

name=None,

data_format=None,

dilations=None):

"""Depthwise 2-D convolution.

Given a 4D input tensor ('NHWC' or 'NCHW' data formats)

and a filter tensor of shape

`[filter_height, filter_width, in_channels, channel_multiplier]`

containing `in_channels` convolutional filters of depth 1, `depthwise_conv2d`

applies a different filter to each input channel (expanding from 1 channel

to `channel_multiplier` channels for each), then concatenates the results

together. The output has `in_channels * channel_multiplier` channels.

In detail, with the default NHWC format,

output[b, i, j, k * channel_multiplier + q] = sum_{di, dj}

filter[di, dj, k, q] * input[b, strides[1] * i + rate[0] * di,

strides[2] * j + rate[1] * dj, k]

Must have `strides[0] = strides[3] = 1`. For the most common case of the

same horizontal and vertical strides, `strides = [1, stride, stride, 1]`.

If any value in `rate` is greater than 1, we perform atrous depthwise

convolution, in which case all values in the `strides` tensor must be equal

to 1.

Args:

input: 4-D with shape according to `data_format`.

filter: 4-D with shape

`[filter_height, filter_width, in_channels, channel_multiplier]`.

strides: 1-D of size 4. The stride of the sliding window for each

dimension of `input`.

padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm.

See the "returns" section of `tf.nn.convolution` for details.

rate: 1-D of size 2. The dilation rate in which we sample input values

across the `height` and `width` dimensions in atrous convolution. If it is

greater than 1, then all values of strides must be 1.

name: A name for this operation (optional).

data_format: The data format for input. Either "NHWC" (default) or "NCHW".

dilations: Alias of rate.

Returns:

A 4-D `Tensor` with shape according to `data_format`. E.g., for

"NHWC" format, shape is

`[batch, out_height, out_width, in_channels * channel_multiplier].`

"""

rate = deprecated_argument_lookup("dilations", dilations, "rate", rate)

with ops.name_scope(name, "depthwise", [input, filter]) as name:

input = ops.convert_to_tensor(input, name="tensor_in")

filter = ops.convert_to_tensor(filter, name="filter_in")

if rate is None:

rate = [1, 1]

# copybara:strip_begin

# TODO(b/138808492): Remove code inside copybara

# to make TPU code and CPU code consistent.

# Use depthwise_conv2d_native if executing on TPU.

if _enclosing_tpu_context() is not None:

if data_format == "NCHW":

dilations = [1, 1, rate[0], rate[1]]

else:

dilations = [1, rate[0], rate[1], 1]

return nn_ops.depthwise_conv2d_native(

input=input,

filter=filter,

strides=strides,

padding=padding,

data_format=data_format,

dilations=dilations,

name=name)

# copybara:strip_end

def op(input_converted, _, padding):

return nn_ops.depthwise_conv2d_native(

input=input_converted,

filter=filter,

strides=strides,

padding=padding,

data_format=data_format,

name=name)

return nn_ops.with_space_to_batch(

input=input,

filter_shape=array_ops.shape(filter),

dilation_rate=rate,

padding=padding,

data_format=data_format,

op=op)

@tf_export("nn.depthwise_conv2d", v1=[])

def depthwise_conv2d_v2(input,

filter,

strides,

padding,

data_format=None,

dilations=None,

name=None):

"""Depthwise 2-D convolution.

Given a 4D input tensor ('NHWC' or 'NCHW' data formats)

and a filter tensor of shape

`[filter_height, filter_width, in_channels, channel_multiplier]`

containing `in_channels` convolutional filters of depth 1, `depthwise_conv2d`

applies a different filter to each input channel (expanding from 1 channel

to `channel_multiplier` channels for each), then concatenates the results

together. The output has `in_channels * channel_multiplier` channels.

In detail, with the default NHWC format,

output[b, i, j, k * channel_multiplier + q] = sum_{di, dj}

filter[di, dj, k, q] * input[b, strides[1] * i + rate[0] * di,

strides[2] * j + rate[1] * dj, k]

Must have `strides[0] = strides[3] = 1`. For the most common case of the

same horizontal and vertical strides, `strides = [1, stride, stride, 1]`.

If any value in `rate` is greater than 1, we perform atrous depthwise

convolution, in which case all values in the `strides` tensor must be equal

to 1.

Args:

input: 4-D with shape according to `data_format`.

filter: 4-D with shape

`[filter_height, filter_width, in_channels, channel_multiplier]`.

strides: 1-D of size 4. The stride of the sliding window for each

dimension of `input`.

padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm.

See the "returns" section of `tf.nn.convolution` for details.

data_format: The data format for input. Either "NHWC" (default) or "NCHW".

dilations: 1-D of size 2. The dilation rate in which we sample input values

across the `height` and `width` dimensions in atrous convolution. If it is

greater than 1, then all values of strides must be 1.

name: A name for this operation (optional).

Returns:

A 4-D `Tensor` with shape according to `data_format`. E.g., for

"NHWC" format, shape is

`[batch, out_height, out_width, in_channels * channel_multiplier].`

"""

return depthwise_conv2d(input=input,

filter=filter,

strides=strides,

padding=padding,

rate=dilations,

name=name,

data_format=data_format)

# pylint: enable=redefined-builtin

# pylint: disable=redefined-builtin,line-too-long

@tf_export(v1=["nn.separable_conv2d"])

def separable_conv2d(input,

depthwise_filter,

pointwise_filter,

strides,

padding,

rate=None,

name=None,

data_format=None,

dilations=None):

"""2-D convolution with separable filters.

Performs a depthwise convolution that acts separately on channels followed by

a pointwise convolution that mixes channels. Note that this is separability

between dimensions `[1, 2]` and `3`, not spatial separability between

dimensions `1` and `2`.

In detail, with the default NHWC format,

output[b, i, j, k] = sum_{di, dj, q, r}

input[b, strides[1] * i + di, strides[2] * j + dj, q] *

depthwise_filter[di, dj, q, r] *

pointwise_filter[0, 0, q * channel_multiplier + r, k]

`strides` controls the strides for the depthwise convolution only, since

the pointwise convolution has implicit strides of `[1, 1, 1, 1]`. Must have

`strides[0] = strides[3] = 1`. For the most common case of the same

horizontal and vertical strides, `strides = [1, stride, stride, 1]`.

If any value in `rate` is greater than 1, we perform atrous depthwise

convolution, in which case all values in the `strides` tensor must be equal

to 1.

Args:

input: 4-D `Tensor` with shape according to `data_format`.

depthwise_filter: 4-D `Tensor` with shape

`[filter_height, filter_width, in_channels, channel_multiplier]`.

Contains `in_channels` convolutional filters of depth 1.

pointwise_filter: 4-D `Tensor` with shape

`[1, 1, channel_multiplier * in_channels, out_channels]`. Pointwise

filter to mix channels after `depthwise_filter` has convolved spatially.

strides: 1-D of size 4. The strides for the depthwise convolution for

each dimension of `input`.

padding: A string, either `'VALID'` or `'SAME'`. The padding algorithm.

See the "returns" section of `tf.nn.convolution` for details.

rate: 1-D of size 2. The dilation rate in which we sample input values

across the `height` and `width` dimensions in atrous convolution. If it is

greater than 1, then all values of strides must be 1.

name: A name for this operation (optional).

data_format: The data format for input. Either "NHWC" (default) or "NCHW".

dilations: Alias of rate.

Returns:

A 4-D `Tensor` with shape according to 'data_format'. For

example, with data_format="NHWC", shape is [batch, out_height,

out_width, out_channels].

"""

rate = deprecated_argument_lookup("dilations", dilations, "rate", rate)

with ops.name_scope(name, "separable_conv2d",

[input, depthwise_filter, pointwise_filter]) as name:

input = ops.convert_to_tensor(input, name="tensor_in")

depthwise_filter = ops.convert_to_tensor(

depthwise_filter, name="depthwise_filter")

pointwise_filter = ops.convert_to_tensor(

pointwise_filter, name="pointwise_filter")

pointwise_filter_shape = pointwise_filter.get_shape().with_rank(4)

pointwise_filter_shape.dims[0].assert_is_compatible_with(1)

pointwise_filter_shape.dims[1].assert_is_compatible_with(1)

if rate is None:

rate = [1, 1]

# The layout of the ops in the graph are expected to be as follows:

# depthwise_conv2d // Conv2D op corresponding to native deptwise conv.

# separable_conv2d // Conv2D op corresponding to the pointwise conv.

def op(input_converted, _, padding):

return nn_ops.depthwise_conv2d_native(

input=input_converted,

filter=depthwise_filter,

strides=strides,

padding=padding,

data_format=data_format,

name="depthwise")

depthwise = nn_ops.with_space_to_batch(

input=input,

filter_shape=array_ops.shape(depthwise_filter),

dilation_rate=rate,

padding=padding,

data_format=data_format,

op=op)

return nn_ops.conv2d(

depthwise,

pointwise_filter, [1, 1, 1, 1],

padding="VALID",

data_format=data_format,

name=name)

@tf_export("nn.separable_conv2d", v1=[])

def separable_conv2d_v2(

input,

depthwise_filter,

pointwise_filter,

strides,

padding,

data_format=None,

dilations=None,

name=None,

):

"""2-D convolution with separable filters.

Performs a depthwise convolution that acts separately on channels followed by

a pointwise convolution that mixes channels. Note that this is separability

between dimensions `[1, 2]` and `3`, not spatial separability between

dimensions `1` and `2`.