12ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
22ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
32ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian YanLicensed under the Apache License, Version 2.0 (the "License");
42ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yanyou may not use this file except in compliance with the License.
52ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian YanYou may obtain a copy of the License at
62ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
72ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan    http://www.apache.org/licenses/LICENSE-2.0
82ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
92ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian YanUnless required by applicable law or agreed to in writing, software
102ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yandistributed under the License is distributed on an "AS IS" BASIS,
112ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian YanWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
122ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian YanSee the License for the specific language governing permissions and
132ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yanlimitations under the License.
142ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan==============================================================================*/
152ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan#include <iostream>
162ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan#include <string>
172ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan#include <vector>
182ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
192ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan#include "absl/memory/memory.h"
202ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan#include "absl/strings/string_view.h"
212ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
222ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan#include "tensorflow/contrib/lite/toco/graph_transformations/lstm_utils.h"
232ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan#include "tensorflow/contrib/lite/toco/model.h"
242ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan#include "tensorflow/contrib/lite/toco/tooling_util.h"
252ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
262ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yannamespace toco {
272ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
282ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yanbool MergeLstmCellInputs::Run(Model* model, std::size_t op_index) {
292ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Find lstm cell.
302ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  auto op_it = model->operators.begin() + op_index;
312ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  auto src_op = op_it->get();
322ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  if (src_op->type != OperatorType::kLstmCell) {
332ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan    return false;
342ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  }
352ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
362ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Already a compact LstmCell with LstmCellOperator::NUM_INPUTS of inputs,
372ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // do not need to merge cell inputs.
382ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  if (src_op->inputs.size() == LstmCellOperator::NUM_INPUTS) {
392ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan    return false;
402ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  }
412ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
422ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Identify prev_activ_input, prev_state_input as required Op inputs,
432ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // using the rnn_states in the model flag.
442ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  string prev_activ_input;
452ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  if (!GetMatchingRnnArray(model, src_op->outputs[kOutputTensor],
462ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                           &prev_activ_input)) {
472ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan    return false;
482ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  }
492ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  string prev_state_input;
502ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  if (!GetMatchingRnnArray(model, src_op->outputs[kCellStateTensor],
512ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                           &prev_state_input)) {
522ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan    return false;
532ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  }
542ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
552ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Get LstmCell's cell, input, output size.
562ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  int num_cell = model->GetArray(src_op->inputs[kInputToInputWeightsTensor])
572ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                     .shape()
582ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                     .dims(0);
592ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  int num_input = model->GetArray(src_op->inputs[kInputToInputWeightsTensor])
602ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      .shape()
612ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      .dims(1);
622ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  int num_output =
632ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      model->GetArray(src_op->inputs[kRecurrentToInputWeightsTensor])
642ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan          .shape()
652ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan          .dims(1);
662ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
672ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Make sure n_cell and n_output are equal as there is no projection.
682ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CHECK_EQ(num_cell, num_output);
692ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
702ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Create tensorflow_graphdef style's one big weight tensor.
712ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  const string base_name(FindLongestCommonPrefix(
722ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      src_op->outputs[kOutputTensor], src_op->outputs[kCellStateTensor]));
732ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  string merged_weights = AvailableArrayName(*model, base_name + "weights");
742ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  auto& array = model->GetOrCreateArray(merged_weights);
752ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  array.data_type = ArrayDataType::kFloat;
762ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  int weights_dim1 = 4 * num_cell;
772ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  int weights_dim2 = num_input + num_output;
782ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  Shape shape = Shape({weights_dim1, weights_dim2});
792ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  array.copy_shape(shape);
802ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  auto& buffer = array.GetMutableBuffer<ArrayDataType::kFloat>();
812ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  buffer.data.resize(weights_dim1 * weights_dim2);
822ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
832ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Merge 8 small weight tensors to 1 weight tensor.
842ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(
852ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      buffer, weights_dim2,
862ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      model->GetArray(src_op->inputs[kInputToInputWeightsTensor]), 0, 0);
872ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(
882ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      buffer, weights_dim2,
892ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      model->GetArray(src_op->inputs[kInputToCellWeightsTensor]), num_cell, 0);
902ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(
912ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      buffer, weights_dim2,
922ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      model->GetArray(src_op->inputs[kInputToForgetWeightsTensor]),
932ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      num_cell * 2, 0);
942ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(
952ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      buffer, weights_dim2,
962ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      model->GetArray(src_op->inputs[kInputToOutputWeightsTensor]),
972ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      num_cell * 3, 0);
982ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(
992ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      buffer, weights_dim2,
1002ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      model->GetArray(src_op->inputs[kRecurrentToInputWeightsTensor]), 0,
1012ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      num_input);
1022ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(
1032ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      buffer, weights_dim2,
1042ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      model->GetArray(src_op->inputs[kRecurrentToCellWeightsTensor]), num_cell,
1052ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      num_input);
1062ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(
1072ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      buffer, weights_dim2,
1082ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      model->GetArray(src_op->inputs[kRecurrentToForgetWeightsTensor]),
1092ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      num_cell * 2, num_input);
1102ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(
1112ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      buffer, weights_dim2,
1122ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      model->GetArray(src_op->inputs[kRecurrentToOutputWeightsTensor]),
1132ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      num_cell * 3, num_input);
1142ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
1152ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Create tensorflow_graphdef style's one big bias tensor.
1162ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  string merged_biases = AvailableArrayName(*model, base_name + "biases");
1172ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  auto& bias_array = model->GetOrCreateArray(merged_biases);
1182ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  bias_array.data_type = ArrayDataType::kFloat;
1192ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  bias_array.copy_shape(Shape({weights_dim1}));
1202ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  auto& bias_buffer = bias_array.GetMutableBuffer<ArrayDataType::kFloat>();
1212ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  bias_buffer.data.resize(weights_dim1);
1222ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
1232ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Merge 4 small bias tensors into a big one.
1242ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(bias_buffer, weights_dim2,
1252ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      model->GetArray(src_op->inputs[kInputGateBiasTensor]), 0,
1262ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      0);
1272ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(bias_buffer, weights_dim2,
1282ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      model->GetArray(src_op->inputs[kCellGateBiasTensor]),
1292ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      num_cell, 0);
1302ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(bias_buffer, weights_dim2,
1312ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      model->GetArray(src_op->inputs[kForgetGateBiasTensor]),
1322ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      num_cell * 2, 0);
1332ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  CopyArrayToSubArray(bias_buffer, weights_dim2,
1342ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      model->GetArray(src_op->inputs[kOutputGateBiasTensor]),
1352ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan                      num_cell * 3, 0);
1362ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
1372ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Emplace a new LSTM cell operator (use basic 5 inputs kernel).
1382ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  auto lstm_cell_op = absl::make_unique<LstmCellOperator>();
1392ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
1402ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Compact LstmCell's 5 inputs.
1412ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->inputs.resize(LstmCellOperator::NUM_INPUTS);
1422ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->inputs[LstmCellOperator::DATA_INPUT] =
1432ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      src_op->inputs[kInputTensor];
1442ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->inputs[LstmCellOperator::WEIGHTS_INPUT] = merged_weights;
1452ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->inputs[LstmCellOperator::BIASES_INPUT] = merged_biases;
1462ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->inputs[LstmCellOperator::PREV_ACTIV_INPUT] = prev_activ_input;
1472ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->inputs[LstmCellOperator::PREV_STATE_INPUT] = prev_state_input;
1482ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
1492ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Reorder LstmCell's 4 outputs.
1502ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->outputs.resize(LstmCellOperator::NUM_OUTPUTS);
1512ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->outputs[LstmCellOperator::ACTIV_OUTPUT] =
1522ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      src_op->outputs[kOutputTensor];
1532ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->outputs[LstmCellOperator::STATE_OUTPUT] =
1542ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      src_op->outputs[kCellStateTensor];
1552ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->outputs[LstmCellOperator::CONCAT_TEMP] =
1562ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      src_op->outputs[kScratchBufferTensor];
1572ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  lstm_cell_op->outputs[LstmCellOperator::ACTIV_TEMP] =
1582ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan      src_op->outputs[kOutputStateTensor];
1592ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
1602ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Add the op into model.
1612ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  model->operators.emplace(op_it, std::move(lstm_cell_op));
1622ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  AddMessageF("Creating compact LstmCell replacing previous lstm cell");
1632ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
1642ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Delete arrays and operators replaced by the LSTM cell operator. Order is
1652ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // important - DeleteArrayIfUnused() only succeeds if dependent operators
1662ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // have been removed first. Start at the output and work towards the input.
1672ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  // Erase curr lstm op being replaced.
1682ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kInputToInputWeightsTensor], model);
1692ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kInputToForgetWeightsTensor], model);
1702ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kInputToCellWeightsTensor], model);
1712ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kInputToOutputWeightsTensor], model);
1722ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kRecurrentToInputWeightsTensor], model);
1732ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kRecurrentToForgetWeightsTensor], model);
1742ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kRecurrentToCellWeightsTensor], model);
1752ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kRecurrentToOutputWeightsTensor], model);
1762ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kInputGateBiasTensor], model);
1772ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kForgetGateBiasTensor], model);
1782ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kCellGateBiasTensor], model);
1792ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  DeleteArrayIfUnused(src_op->inputs[kOutputGateBiasTensor], model);
1802ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  model->operators.erase(FindOp(*model, src_op));
1812ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
1822ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan  return true;
1832ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan}
1842ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan
1852ec89055c17cd17dc1b1259d9eb76d1177c9b0e8Zhixian Yan}  // namespace toco
186