From 5f3738b3c921732a8858d22a76cbb9748eff08c2 Mon Sep 17 00:00:00 2001 From: piggies <724022815@qq.com> Date: Tue, 12 Apr 2022 14:19:10 +0800 Subject: [PATCH 1/6] update --- .../test/train_performance_1p.sh | 9 +- .../trainer/task.py | 48 +- .../{utils => util}/dataloader.py | 0 .../util/dnn_linear_combined.py | 1146 +++++++++++++++++ .../{utils => util}/hooks/benchmark_hooks.py | 0 .../{utils => util}/hooks/training_hooks.py | 0 .../{utils => util}/metrics.py | 0 .../{utils => util}/schedulers.py | 0 8 files changed, 1183 insertions(+), 20 deletions(-) rename TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/{utils => util}/dataloader.py (100%) create mode 100644 TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/dnn_linear_combined.py rename TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/{utils => util}/hooks/benchmark_hooks.py (100%) rename TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/{utils => util}/hooks/training_hooks.py (100%) rename TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/{utils => util}/metrics.py (100%) rename TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/{utils => util}/schedulers.py (100%) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/test/train_performance_1p.sh b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/test/train_performance_1p.sh index 64ab2617e..4658e94f6 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/test/train_performance_1p.sh +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/test/train_performance_1p.sh @@ -27,6 +27,9 @@ RankSize=1 #训练epoch，可选 train_epochs=1 +#迭代下沉循环次数 +iteration_per_loop=0 + #参数配置 data_path="" @@ -104,13 +107,15 @@ wait cd $cur_path/../ start=$(date +%s) -python3 -m trainer.task --gpu \ +python3 -m trainer.task \ --Adam \ + --iteration_per_loop=$iteration_per_loop \ --train_data_pattern=$data_path/outbrain/tfrecords/train/part* \ --eval_data_pattern=$data_path/outbrain/tfrecords/eval/part* \ --model_dir=$cur_path/output/$ASCEND_DEVICE_ID/ckpt \ --transformed_metadata_path=$data_path/outbrain/tfrecords \ - --num_epochs=$train_epochs > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & + --num_epochs=$train_epochs \ + --global_batch_size=$batch_size > $cur_path/output/$ASCEND_DEVICE_ID/train_$ASCEND_DEVICE_ID.log 2>&1 & wait end=$(date +%s) e2etime=$(( $end - $start )) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/trainer/task.py b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/trainer/task.py index c22742f76..08ca14365 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/trainer/task.py +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/trainer/task.py @@ -27,10 +27,11 @@ import tensorflow as tf import tensorflow_transform as tft from tensorflow.core.protobuf import rewriter_config_pb2 from trainer import features -from utils.dataloader import separate_input_fn -from utils.hooks.benchmark_hooks import BenchmarkLoggingHook -from utils.metrics import map_custom_metric, map_custom_metric_with_leak -from utils.schedulers import learning_rate_scheduler +from util.dataloader import separate_input_fn +from util.hooks.benchmark_hooks import BenchmarkLoggingHook +from util.metrics import map_custom_metric, map_custom_metric_with_leak +from util.schedulers import learning_rate_scheduler +from util.dnn_linear_combined import DNNLinearCombinedClassifier MODEL_TYPES = ['wide', 'deep', 'wide_n_deep'] WIDE, DEEP, WIDE_N_DEEP = MODEL_TYPES @@ -239,6 +240,11 @@ def create_parser(): help='Number of steps for train performance benchmark', type=int, default=100) + parser.add_argument( + '--iteration_per_loop', + help='Number of iters per loop', + type=int, + default=0) return parser @@ -262,7 +268,7 @@ def construct_estimator(model_type, run_config, optimizer=deep_optimizer) elif model_type == WIDE_N_DEEP: - estimator = tf.estimator.DNNLinearCombinedClassifier( + estimator = DNNLinearCombinedClassifier( config=npu_run_config_init(run_config=run_config), linear_feature_columns=wide_columns, linear_optimizer=wide_optimizer, @@ -329,15 +335,14 @@ def main(FLAGS): log_device_placement=FLAGS.log_device_placement ) else: - #session_config = tf.compat.v1.ConfigProto( - # device_count={'GPU': 0}, - # log_device_placement=FLAGS.log_device_placement - #) session_config = tf.ConfigProto() custom_op = session_config.graph_options.rewrite_options.custom_optimizers.add() custom_op.name = "NpuOptimizer" custom_op.parameter_map["use_off_line"].b = True custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes("allow_mix_precision") + if FLAGS.iteration_per_loop: + custom_op.parameter_map["enable_data_pre_proc"].b = True + custom_op.parameter_map["iterations_per_loop"].i = FLAGS.iteration_per_loop session_config.graph_options.rewrite_options.remapping = RewriterConfig.OFF session_config.graph_options.rewrite_options.memory_optimization = RewriterConfig.OFF @@ -361,12 +366,14 @@ def main(FLAGS): int(FLAGS.eval_epoch_interval * steps_per_epoch) count_steps = FLAGS.benchmark_steps + 1 if FLAGS.benchmark else 100 - run_config = tf.estimator.RunConfig(model_dir=model_dir, save_summary_steps=0) \ - .replace(session_config=session_config, - save_checkpoints_steps=save_checkpoints_steps, - save_summary_steps=count_steps, - log_step_count_steps=count_steps, - keep_checkpoint_max=1) + # run_config = tf.estimator.RunConfig(model_dir=model_dir, save_summary_steps=0) \ + # .replace(session_config=session_config, + # save_checkpoints_steps=save_checkpoints_steps, + # save_summary_steps=count_steps, + # log_step_count_steps=count_steps, + # keep_checkpoint_max=1) + + run_config = tf.estimator.RunConfig(model_dir=model_dir, save_summary_steps=0, session_config=session_config, save_checkpoints_steps=save_checkpoints_steps, log_step_count_steps=count_steps, keep_checkpoint_max=1) def wide_optimizer(): opt = tf.compat.v1.train.FtrlOptimizer( @@ -431,6 +438,8 @@ def main(FLAGS): estimator = tf.estimator.add_metrics(estimator, map_custom_metric_with_leak) hooks = [] + if FLAGS.iteration_per_loop: + hooks.append(npu_hook.SetIterationsVarHook(FLAGS.iteration_per_loop)) if FLAGS.hvd: hooks.append(NPUBroadcastGlobalVariablesHook(0, int(os.getenv('RANK_ID', '0')))) @@ -482,6 +491,7 @@ def main(FLAGS): train_throughput = benchmark_hook.mean_throughput.value() dllogger.log(data={'train_throughput': train_throughput}, step=tuple()) else: + print('train and eval') train_spec = tf.estimator.TrainSpec(input_fn=train_input_fn, max_steps=max_steps, hooks=hooks) @@ -498,18 +508,20 @@ def main(FLAGS): if __name__ == '__main__': + FLAGS = create_parser().parse_args() session_config = tf.ConfigProto() custom_op = session_config.graph_options.rewrite_options.custom_optimizers.add() custom_op.name = "NpuOptimizer" custom_op.parameter_map["use_off_line"].b = True custom_op.parameter_map["precision_mode"].s = tf.compat.as_bytes("allow_mix_precision") - custom_op.parameter_map["graph_memory_max_size"].s= tf.compat.as_bytes(str(16 * 1024 * 1024 * 1024)) - custom_op.parameter_map["variable_memory_max_size"].s = tf.compat.as_bytes(str(15 * 1024 * 1024 * 1024)) + if FLAGS.iteration_per_loop: + print('>>>>>>>>> iteration per loop var: %d'%(FLAGS.iteration_per_loop)) + custom_op.parameter_map["enable_data_pre_proc"].b = True + custom_op.parameter_map["iterations_per_loop"].i = FLAGS.iteration_per_loop session_config.graph_options.rewrite_options.remapping = RewriterConfig.OFF session_config.graph_options.rewrite_options.memory_optimization = RewriterConfig.OFF (npu_sess, npu_shutdown) = init_resource(config=session_config) - FLAGS = create_parser().parse_args() main(FLAGS) shutdown_resource(npu_sess, npu_shutdown) close_session(npu_sess) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/dataloader.py b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/dataloader.py similarity index 100% rename from TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/dataloader.py rename to TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/dataloader.py diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/dnn_linear_combined.py b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/dnn_linear_combined.py new file mode 100644 index 000000000..b4ee6a980 --- /dev/null +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/dnn_linear_combined.py @@ -0,0 +1,1146 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorFlow estimators for Linear and DNN joined training models.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import math + +import six +import tensorflow as tf +from tensorflow.python.util.tf_export import estimator_export +from tensorflow_estimator.python.estimator import estimator +from tensorflow_estimator.python.estimator.canned import dnn +from tensorflow_estimator.python.estimator.canned import head as head_lib +from tensorflow_estimator.python.estimator.canned import linear +from tensorflow_estimator.python.estimator.canned import optimizers +from tensorflow_estimator.python.estimator.head import head_utils +from tensorflow_estimator.python.estimator.head import regression_head +from tensorflow_estimator.python.estimator.mode_keys import ModeKeys + +# The default learning rates are a historical artifact of the initial +# implementation. +_DNN_LEARNING_RATE = 0.001 +_LINEAR_LEARNING_RATE = 0.005 + + +def _check_no_sync_replicas_optimizer(optimizer): + if isinstance(optimizer, tf.compat.v1.train.SyncReplicasOptimizer): + raise ValueError( + 'SyncReplicasOptimizer does not support multi optimizers case. ' + 'Therefore, it is not supported in DNNLinearCombined model. ' + 'If you want to use this optimizer, please use either DNN or Linear ' + 'model.') + + +def _linear_learning_rate(num_linear_feature_columns): + """Returns the default learning rate of the linear model. + + The calculation is a historical artifact of this initial implementation, but + has proven a reasonable choice. + + Args: + num_linear_feature_columns: The number of feature columns of the linear + model. + + Returns: + A float. + """ + default_learning_rate = 1. / math.sqrt(num_linear_feature_columns) + return min(_LINEAR_LEARNING_RATE, default_learning_rate) + + +def _add_layer_summary(value, tag): + tf.compat.v1.summary.scalar('%s/fraction_of_zero_values' % tag, + tf.math.zero_fraction(value)) + tf.compat.v1.summary.histogram('%s/activation' % tag, value) + + +def _validate_feature_columns(linear_feature_columns, dnn_feature_columns): + """Validates feature columns DNNLinearCombinedRegressor.""" + linear_feature_columns = linear_feature_columns or [] + dnn_feature_columns = dnn_feature_columns or [] + feature_columns = (list(linear_feature_columns) + list(dnn_feature_columns)) + if not feature_columns: + raise ValueError('Either linear_feature_columns or dnn_feature_columns ' + 'must be defined.') + return feature_columns + + +def _dnn_linear_combined_model_fn_v2( + features, + labels, + mode, + head, + linear_feature_columns=None, + linear_optimizer='Ftrl', + dnn_feature_columns=None, + dnn_optimizer='Adagrad', + dnn_hidden_units=None, + dnn_activation_fn=tf.nn.relu, + dnn_dropout=None, + config=None, + batch_norm=False, + linear_sparse_combiner='sum', + loss_reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE): + """Deep Neural Net and Linear combined model_fn. + + Args: + features: dict of `Tensor`. + labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of dtype + `int32` or `int64` in the range `[0, n_classes)`. + mode: Defines whether this is training, evaluation or prediction. See + `ModeKeys`. + head: A `Head` instance. + linear_feature_columns: An iterable containing all the feature columns used + by the Linear model. + linear_optimizer: string, `Optimizer` object, or callable that defines the + optimizer to use for training the Linear model. Defaults to the Ftrl + optimizer. + dnn_feature_columns: An iterable containing all the feature columns used by + the DNN model. + dnn_optimizer: string, `Optimizer` object, or callable that defines the + optimizer to use for training the DNN model. Defaults to the Adagrad + optimizer. + dnn_hidden_units: List of hidden units per DNN layer. + dnn_activation_fn: Activation function applied to each DNN layer. If `None`, + will use `tf.nn.relu`. + dnn_dropout: When not `None`, the probability we will drop out a given DNN + coordinate. + config: `RunConfig` object to configure the runtime settings. + batch_norm: Whether to use batch normalization after each hidden layer. + linear_sparse_combiner: A string specifying how to reduce the linear model + if a categorical column is multivalent. One of "mean", "sqrtn", and + "sum". + loss_reduction: One of `tf.keras.losses.Reduction` except `NONE`. Describes + how to reduce training loss over batch. Defaults to `SUM_OVER_BATCH_SIZE`. + + Returns: + An `EstimatorSpec` instance. + + Raises: + ValueError: If both `linear_feature_columns` and `dnn_features_columns` + are empty at the same time, or `input_layer_partitioner` is missing, + or features has the wrong type. + """ + if not isinstance(features, dict): + raise ValueError('features should be a dictionary of `Tensor`s. ' + 'Given type: {}'.format(type(features))) + if not linear_feature_columns and not dnn_feature_columns: + raise ValueError( + 'Either linear_feature_columns or dnn_feature_columns must be defined.') + + del config + + # Build DNN Logits. + if not dnn_feature_columns: + dnn_logits = None + else: + if mode == ModeKeys.TRAIN: + dnn_optimizer = optimizers.get_optimizer_instance_v2( + dnn_optimizer, learning_rate=_DNN_LEARNING_RATE) + _check_no_sync_replicas_optimizer(dnn_optimizer) + + if not dnn_hidden_units: + raise ValueError( + 'dnn_hidden_units must be defined when dnn_feature_columns is ' + 'specified.') + dnn_logits, dnn_trainable_variables, dnn_update_ops = ( + dnn._dnn_model_fn_builder_v2( # pylint: disable=protected-access + units=head.logits_dimension, + hidden_units=dnn_hidden_units, + feature_columns=dnn_feature_columns, + activation_fn=dnn_activation_fn, + dropout=dnn_dropout, + batch_norm=batch_norm, + features=features, + mode=mode)) + + if not linear_feature_columns: + linear_logits = None + else: + if mode == ModeKeys.TRAIN: + linear_optimizer = optimizers.get_optimizer_instance_v2( + linear_optimizer, + learning_rate=_linear_learning_rate(len(linear_feature_columns))) + _check_no_sync_replicas_optimizer(linear_optimizer) + + linear_logits, linear_trainable_variables = ( + linear._linear_model_fn_builder_v2( # pylint: disable=protected-access + units=head.logits_dimension, + feature_columns=linear_feature_columns, + sparse_combiner=linear_sparse_combiner, + features=features)) + _add_layer_summary(linear_logits, 'linear') + + # Combine logits and build full model. + if dnn_logits is not None and linear_logits is not None: + logits = dnn_logits + linear_logits + elif dnn_logits is not None: + logits = dnn_logits + else: + logits = linear_logits + + def _train_op_fn(loss): + """Returns the op to optimize the loss.""" + train_ops = [] + # Scale loss by number of replicas. + if loss_reduction == tf.losses.Reduction.SUM_OVER_BATCH_SIZE: + num_replicas = tf.distribute.get_strategy().num_replicas_in_sync + if num_replicas > 1: + loss *= (1. / num_replicas) + + if dnn_logits is not None: + train_ops.extend(dnn_optimizer.get_updates(loss, dnn_trainable_variables)) + if dnn_update_ops is not None: + train_ops.extend(dnn_update_ops) + if linear_logits is not None: + train_ops.extend( + linear_optimizer.get_updates(loss, linear_trainable_variables)) + train_op = tf.group(*train_ops) + return train_op + + # In TRAIN mode, asssign global_step variable to optimizer.iterations to + # make global_step increased correctly, as Hooks relies on global step as + # step counter. Note that, Only one model's optimizer needs this assignment. + if mode == ModeKeys.TRAIN: + if dnn_logits is not None: + dnn_optimizer.iterations = tf.compat.v1.train.get_or_create_global_step() + else: + linear_optimizer.iterations = \ + tf.compat.v1.train.get_or_create_global_step() + + return head.create_estimator_spec( + features=features, + mode=mode, + labels=labels, + train_op_fn=_train_op_fn, + logits=logits) + + +def _dnn_linear_combined_model_fn(features, + labels, + mode, + head, + linear_feature_columns=None, + linear_optimizer='Ftrl', + dnn_feature_columns=None, + dnn_optimizer='Adagrad', + dnn_hidden_units=None, + dnn_activation_fn=tf.nn.relu, + dnn_dropout=None, + input_layer_partitioner=None, + config=None, + batch_norm=False, + linear_sparse_combiner='sum'): + """Deep Neural Net and Linear combined model_fn. + + Args: + features: dict of `Tensor`. + labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of dtype + `int32` or `int64` in the range `[0, n_classes)`. + mode: Defines whether this is training, evaluation or prediction. See + `ModeKeys`. + head: A `Head` instance. + linear_feature_columns: An iterable containing all the feature columns used + by the Linear model. + linear_optimizer: string, `Optimizer` object, or callable that defines the + optimizer to use for training the Linear model. Defaults to the Ftrl + optimizer. + dnn_feature_columns: An iterable containing all the feature columns used by + the DNN model. + dnn_optimizer: string, `Optimizer` object, or callable that defines the + optimizer to use for training the DNN model. Defaults to the Adagrad + optimizer. + dnn_hidden_units: List of hidden units per DNN layer. + dnn_activation_fn: Activation function applied to each DNN layer. If `None`, + will use `tf.nn.relu`. + dnn_dropout: When not `None`, the probability we will drop out a given DNN + coordinate. + input_layer_partitioner: Partitioner for input layer. + config: `RunConfig` object to configure the runtime settings. + batch_norm: Whether to use batch normalization after each hidden layer. + linear_sparse_combiner: A string specifying how to reduce the linear model + if a categorical column is multivalent. One of "mean", "sqrtn", and + "sum". + + Returns: + An `EstimatorSpec` instance. + + Raises: + ValueError: If both `linear_feature_columns` and `dnn_features_columns` + are empty at the same time, or `input_layer_partitioner` is missing, + or features has the wrong type. + """ + if not isinstance(features, dict): + raise ValueError('features should be a dictionary of `Tensor`s. ' + 'Given type: {}'.format(type(features))) + if not linear_feature_columns and not dnn_feature_columns: + raise ValueError( + 'Either linear_feature_columns or dnn_feature_columns must be defined.') + + num_ps_replicas = config.num_ps_replicas if config else 0 + input_layer_partitioner = input_layer_partitioner or ( + tf.compat.v1.min_max_variable_partitioner( + max_partitions=num_ps_replicas, min_slice_size=64 << 20)) + + # Build DNN Logits. + dnn_parent_scope = 'dnn' + + if not dnn_feature_columns: + dnn_logits = None + else: + dnn_optimizer = optimizers.get_optimizer_instance( + dnn_optimizer, learning_rate=_DNN_LEARNING_RATE) + _check_no_sync_replicas_optimizer(dnn_optimizer) + if not dnn_hidden_units: + raise ValueError( + 'dnn_hidden_units must be defined when dnn_feature_columns is ' + 'specified.') + dnn_partitioner = ( + tf.compat.v1.min_max_variable_partitioner( + max_partitions=num_ps_replicas)) + with tf.compat.v1.variable_scope( + dnn_parent_scope, + values=tuple(six.itervalues(features)), + partitioner=dnn_partitioner) as scope: + dnn_absolute_scope = scope.name + dnn_logit_fn = dnn.dnn_logit_fn_builder( + units=head.logits_dimension, + hidden_units=dnn_hidden_units, + feature_columns=dnn_feature_columns, + activation_fn=dnn_activation_fn, + dropout=dnn_dropout, + batch_norm=batch_norm, + input_layer_partitioner=input_layer_partitioner) + dnn_logits = dnn_logit_fn(features=features, mode=mode) + + linear_parent_scope = 'linear' + + if not linear_feature_columns: + linear_logits = None + else: + linear_optimizer = optimizers.get_optimizer_instance( + linear_optimizer, + learning_rate=_linear_learning_rate(len(linear_feature_columns))) + _check_no_sync_replicas_optimizer(linear_optimizer) + with tf.compat.v1.variable_scope( + linear_parent_scope, + values=tuple(six.itervalues(features)), + partitioner=input_layer_partitioner) as scope: + linear_absolute_scope = scope.name + logit_fn = linear.linear_logit_fn_builder( + units=head.logits_dimension, + feature_columns=linear_feature_columns, + sparse_combiner=linear_sparse_combiner) + linear_logits = logit_fn(features=features) + _add_layer_summary(linear_logits, scope.name) + + # Combine logits and build full model. + if dnn_logits is not None and linear_logits is not None: + logits = dnn_logits + linear_logits + elif dnn_logits is not None: + logits = dnn_logits + else: + logits = linear_logits + + def _train_op_fn(loss): + """Returns the op to optimize the loss.""" + train_ops = [] + global_step = tf.compat.v1.train.get_global_step() + if dnn_logits is not None: + train_ops.append( + dnn_optimizer.minimize( + loss, + var_list=tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.TRAINABLE_VARIABLES, + scope=dnn_absolute_scope))) + if linear_logits is not None: + train_ops.append( + linear_optimizer.minimize( + loss, + var_list=tf.compat.v1.get_collection( + tf.compat.v1.GraphKeys.TRAINABLE_VARIABLES, + scope=linear_absolute_scope))) + + train_op = tf.group(*train_ops, name="IterationOp") + with tf.control_dependencies([train_op]): + return tf.compat.v1.assign_add(global_step, 1).op + + return head.create_estimator_spec( + features=features, + mode=mode, + labels=labels, + train_op_fn=_train_op_fn, + logits=logits) + + +@estimator_export('estimator.DNNLinearCombinedClassifier', v1=[]) +class DNNLinearCombinedClassifierV2(estimator.EstimatorV2): + """An estimator for TensorFlow Linear and DNN joined classification models. + + Note: This estimator is also known as wide-n-deep. + + Example: + + ```python + numeric_feature = numeric_column(...) + categorical_column_a = categorical_column_with_hash_bucket(...) + categorical_column_b = categorical_column_with_hash_bucket(...) + + categorical_feature_a_x_categorical_feature_b = crossed_column(...) + categorical_feature_a_emb = embedding_column( + categorical_column=categorical_feature_a, ...) + categorical_feature_b_emb = embedding_column( + categorical_id_column=categorical_feature_b, ...) + + estimator = tf.estimator.DNNLinearCombinedClassifier( + # wide settings + linear_feature_columns=[categorical_feature_a_x_categorical_feature_b], + linear_optimizer=tf.keras.optimizers.Ftrl(...), + # deep settings + dnn_feature_columns=[ + categorical_feature_a_emb, categorical_feature_b_emb, + numeric_feature], + dnn_hidden_units=[1000, 500, 100], + dnn_optimizer=tf.keras.optimizers.Adagrad(...), + # warm-start settings + warm_start_from="/path/to/checkpoint/dir") + + # To apply L1 and L2 regularization, you can set dnn_optimizer to: + tf.compat.v1.train.ProximalAdagradOptimizer( + learning_rate=0.1, + l1_regularization_strength=0.001, + l2_regularization_strength=0.001) + # To apply learning rate decay, you can set dnn_optimizer to a callable: + lambda: tf.keras.optimizers.Adam( + learning_rate=tf.compat.v1.train.exponential_decay( + learning_rate=0.1, + global_step=tf.compat.v1.train.get_global_step(), + decay_steps=10000, + decay_rate=0.96) + # It is the same for linear_optimizer. + + # Input builders + def input_fn_train: + # Returns tf.data.Dataset of (x, y) tuple where y represents label's class + # index. + pass + def input_fn_eval: + # Returns tf.data.Dataset of (x, y) tuple where y represents label's class + # index. + pass + def input_fn_predict: + # Returns tf.data.Dataset of (x, None) tuple. + pass + estimator.train(input_fn=input_fn_train, steps=100) + metrics = estimator.evaluate(input_fn=input_fn_eval, steps=10) + predictions = estimator.predict(input_fn=input_fn_predict) + ``` + + Input of `train` and `evaluate` should have following features, + otherwise there will be a `KeyError`: + + * for each `column` in `dnn_feature_columns` + `linear_feature_columns`: + - if `column` is a `CategoricalColumn`, a feature with `key=column.name` + whose `value` is a `SparseTensor`. + - if `column` is a `WeightedCategoricalColumn`, two features: the first + with `key` the id column name, the second with `key` the weight column + name. Both features' `value` must be a `SparseTensor`. + - if `column` is a `DenseColumn`, a feature with `key=column.name` + whose `value` is a `Tensor`. + + Loss is calculated by using softmax cross entropy. + + @compatibility(eager) + Estimators can be used while eager execution is enabled. Note that `input_fn` + and all hooks are executed inside a graph context, so they have to be written + to be compatible with graph mode. Note that `input_fn` code using `tf.data` + generally works in both graph and eager modes. + @end_compatibility + """ + + def __init__(self, + model_dir=None, + linear_feature_columns=None, + linear_optimizer='Ftrl', + dnn_feature_columns=None, + dnn_optimizer='Adagrad', + dnn_hidden_units=None, + dnn_activation_fn=tf.nn.relu, + dnn_dropout=None, + n_classes=2, + weight_column=None, + label_vocabulary=None, + config=None, + warm_start_from=None, + loss_reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE, + batch_norm=False, + linear_sparse_combiner='sum'): + """Initializes a DNNLinearCombinedClassifier instance. + + Args: + model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator to + continue training a previously saved model. + linear_feature_columns: An iterable containing all the feature columns + used by linear part of the model. All items in the set must be instances + of classes derived from `FeatureColumn`. + linear_optimizer: An instance of `tf.keras.optimizers.*` used to apply + gradients to the linear part of the model. Can also be a string (one of + 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to + FTRL optimizer. + dnn_feature_columns: An iterable containing all the feature columns used + by deep part of the model. All items in the set must be instances of + classes derived from `FeatureColumn`. + dnn_optimizer: An instance of `tf.keras.optimizers.*` used to apply + gradients to the deep part of the model. Can also be a string (one of + 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to + Adagrad optimizer. + dnn_hidden_units: List of hidden units per layer. All layers are fully + connected. + dnn_activation_fn: Activation function applied to each layer. If None, + will use `tf.nn.relu`. + dnn_dropout: When not None, the probability we will drop out a given + coordinate. + n_classes: Number of label classes. Defaults to 2, namely binary + classification. Must be > 1. + weight_column: A string or a `_NumericColumn` created by + `tf.feature_column.numeric_column` defining feature column representing + weights. It is used to down weight or boost examples during training. It + will be multiplied by the loss of the example. If it is a string, it is + used as a key to fetch weight tensor from the `features`. If it is a + `_NumericColumn`, raw tensor is fetched by key `weight_column.key`, then + weight_column.normalizer_fn is applied on it to get weight tensor. + label_vocabulary: A list of strings represents possible label values. If + given, labels must be string type and have any value in + `label_vocabulary`. If it is not given, that means labels are already + encoded as integer or float within [0, 1] for `n_classes=2` and encoded + as integer values in {0, 1,..., n_classes-1} for `n_classes`>2 . Also + there will be errors if vocabulary is not provided and labels are + string. + config: RunConfig object to configure the runtime settings. + warm_start_from: A string filepath to a checkpoint to warm-start from, or + a `WarmStartSettings` object to fully configure warm-starting. If the + string filepath is provided instead of a `WarmStartSettings`, then all + weights are warm-started, and it is assumed that vocabularies and Tensor + names are unchanged. + loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how + to reduce training loss over batch. Defaults to `SUM_OVER_BATCH_SIZE`. + batch_norm: Whether to use batch normalization after each hidden layer. + linear_sparse_combiner: A string specifying how to reduce the linear model + if a categorical column is multivalent. One of "mean", "sqrtn", and + "sum" -- these are effectively different ways to do example-level + normalization, which can be useful for bag-of-words features. For more + details, see `tf.feature_column.linear_model`. + + Raises: + ValueError: If both linear_feature_columns and dnn_features_columns are + empty at the same time. + """ + self._feature_columns = _validate_feature_columns( + linear_feature_columns=linear_feature_columns, + dnn_feature_columns=dnn_feature_columns) + + head = head_utils.binary_or_multi_class_head( + n_classes, + weight_column=weight_column, + label_vocabulary=label_vocabulary, + loss_reduction=loss_reduction) + estimator._canned_estimator_api_gauge.get_cell('Classifier').set( # pylint: disable=protected-access + 'DNNLinearCombined') + + def _model_fn(features, labels, mode, config): + """Call the _dnn_linear_combined_model_fn.""" + return _dnn_linear_combined_model_fn_v2( + features=features, + labels=labels, + mode=mode, + head=head, + linear_feature_columns=linear_feature_columns, + linear_optimizer=linear_optimizer, + dnn_feature_columns=dnn_feature_columns, + dnn_optimizer=dnn_optimizer, + dnn_hidden_units=dnn_hidden_units, + dnn_activation_fn=dnn_activation_fn, + dnn_dropout=dnn_dropout, + config=config, + batch_norm=batch_norm, + linear_sparse_combiner=linear_sparse_combiner, + loss_reduction=loss_reduction) + + super(DNNLinearCombinedClassifierV2, self).__init__( + model_fn=_model_fn, + model_dir=model_dir, + config=config, + warm_start_from=warm_start_from) + + +@estimator_export(v1=['estimator.DNNLinearCombinedClassifier']) # pylint: disable=missing-docstring +class DNNLinearCombinedClassifier(estimator.Estimator): + __doc__ = DNNLinearCombinedClassifierV2.__doc__.replace( + 'SUM_OVER_BATCH_SIZE', 'SUM') + + def __init__(self, + model_dir=None, + linear_feature_columns=None, + linear_optimizer='Ftrl', + dnn_feature_columns=None, + dnn_optimizer='Adagrad', + dnn_hidden_units=None, + dnn_activation_fn=tf.nn.relu, + dnn_dropout=None, + n_classes=2, + weight_column=None, + label_vocabulary=None, + input_layer_partitioner=None, + config=None, + warm_start_from=None, + loss_reduction=tf.compat.v1.losses.Reduction.SUM, + batch_norm=False, + linear_sparse_combiner='sum'): + self._feature_columns = _validate_feature_columns( + linear_feature_columns=linear_feature_columns, + dnn_feature_columns=dnn_feature_columns) + + head = head_lib._binary_logistic_or_multi_class_head( # pylint: disable=protected-access + n_classes, weight_column, label_vocabulary, loss_reduction) + estimator._canned_estimator_api_gauge.get_cell('Classifier').set( + 'DNNLinearCombined') # pylint: disable=protected-access + + def _model_fn(features, labels, mode, config): + """Call the _dnn_linear_combined_model_fn.""" + return _dnn_linear_combined_model_fn( + features=features, + labels=labels, + mode=mode, + head=head, + linear_feature_columns=linear_feature_columns, + linear_optimizer=linear_optimizer, + dnn_feature_columns=dnn_feature_columns, + dnn_optimizer=dnn_optimizer, + dnn_hidden_units=dnn_hidden_units, + dnn_activation_fn=dnn_activation_fn, + dnn_dropout=dnn_dropout, + input_layer_partitioner=input_layer_partitioner, + config=config, + batch_norm=batch_norm, + linear_sparse_combiner=linear_sparse_combiner) + + super(DNNLinearCombinedClassifier, self).__init__( + model_fn=_model_fn, + model_dir=model_dir, + config=config, + warm_start_from=warm_start_from) + + +def _init_dnn_linear_combined_estimator(head, linear_feature_columns, + linear_optimizer, dnn_feature_columns, + dnn_optimizer, dnn_hidden_units, + dnn_activation_fn, dnn_dropout, + input_layer_partitioner, + linear_sparse_combiner): + """Helper function for the initialization of DNNLinearCombinedEstimator.""" + linear_feature_columns = linear_feature_columns or [] + dnn_feature_columns = dnn_feature_columns or [] + feature_columns = (list(linear_feature_columns) + list(dnn_feature_columns)) + if not feature_columns: + raise ValueError('Either linear_feature_columns or dnn_feature_columns ' + 'must be defined.') + + def _model_fn(features, labels, mode, config): + """Call the _dnn_linear_combined_model_fn.""" + return _dnn_linear_combined_model_fn( + features=features, + labels=labels, + mode=mode, + head=head, + linear_feature_columns=linear_feature_columns, + linear_optimizer=linear_optimizer, + dnn_feature_columns=dnn_feature_columns, + dnn_optimizer=dnn_optimizer, + dnn_hidden_units=dnn_hidden_units, + dnn_activation_fn=dnn_activation_fn, + dnn_dropout=dnn_dropout, + input_layer_partitioner=input_layer_partitioner, + config=config, + linear_sparse_combiner=linear_sparse_combiner) + + return feature_columns, _model_fn + + +@estimator_export('estimator.DNNLinearCombinedEstimator', v1=[]) +class DNNLinearCombinedEstimatorV2(estimator.EstimatorV2): + """An estimator for TensorFlow Linear and DNN joined models with custom head. + + Note: This estimator is also known as wide-n-deep. + + Example: + + ```python + numeric_feature = numeric_column(...) + categorical_column_a = categorical_column_with_hash_bucket(...) + categorical_column_b = categorical_column_with_hash_bucket(...) + + categorical_feature_a_x_categorical_feature_b = crossed_column(...) + categorical_feature_a_emb = embedding_column( + categorical_column=categorical_feature_a, ...) + categorical_feature_b_emb = embedding_column( + categorical_column=categorical_feature_b, ...) + + estimator = tf.estimator.DNNLinearCombinedEstimator( + head=tf.estimator.MultiLabelHead(n_classes=3), + # wide settings + linear_feature_columns=[categorical_feature_a_x_categorical_feature_b], + linear_optimizer=tf.keras.optimizers.Ftrl(...), + # deep settings + dnn_feature_columns=[ + categorical_feature_a_emb, categorical_feature_b_emb, + numeric_feature], + dnn_hidden_units=[1000, 500, 100], + dnn_optimizer=tf.keras.optimizers.Adagrad(...)) + + # To apply L1 and L2 regularization, you can set dnn_optimizer to: + tf.compat.v1.train.ProximalAdagradOptimizer( + learning_rate=0.1, + l1_regularization_strength=0.001, + l2_regularization_strength=0.001) + # To apply learning rate decay, you can set dnn_optimizer to a callable: + lambda: tf.keras.optimizers.Adam( + learning_rate=tf.compat.v1.train.exponential_decay( + learning_rate=0.1, + global_step=tf.compat.v1.train.get_global_step(), + decay_steps=10000, + decay_rate=0.96) + # It is the same for linear_optimizer. + + # Input builders + def input_fn_train: + # Returns tf.data.Dataset of (x, y) tuple where y represents label's class + # index. + pass + def input_fn_eval: + # Returns tf.data.Dataset of (x, y) tuple where y represents label's class + # index. + pass + def input_fn_predict: + # Returns tf.data.Dataset of (x, None) tuple. + pass + estimator.train(input_fn=input_fn_train, steps=100) + metrics = estimator.evaluate(input_fn=input_fn_eval, steps=10) + predictions = estimator.predict(input_fn=input_fn_predict) + ``` + + Input of `train` and `evaluate` should have following features, + otherwise there will be a `KeyError`: + + * for each `column` in `dnn_feature_columns` + `linear_feature_columns`: + - if `column` is a `CategoricalColumn`, a feature with `key=column.name` + whose `value` is a `SparseTensor`. + - if `column` is a `WeightedCategoricalColumn`, two features: the first + with `key` the id column name, the second with `key` the weight column + name. Both features' `value` must be a `SparseTensor`. + - if `column` is a `DenseColumn`, a feature with `key=column.name` + whose `value` is a `Tensor`. + + Loss is calculated by using mean squared error. + + @compatibility(eager) + Estimators can be used while eager execution is enabled. Note that `input_fn` + and all hooks are executed inside a graph context, so they have to be written + to be compatible with graph mode. Note that `input_fn` code using `tf.data` + generally works in both graph and eager modes. + @end_compatibility + """ + + def __init__(self, + head, + model_dir=None, + linear_feature_columns=None, + linear_optimizer='Ftrl', + dnn_feature_columns=None, + dnn_optimizer='Adagrad', + dnn_hidden_units=None, + dnn_activation_fn=tf.nn.relu, + dnn_dropout=None, + config=None, + batch_norm=False, + linear_sparse_combiner='sum'): + """Initializes a DNNLinearCombinedEstimator instance. + + Args: + head: A `Head` instance constructed with a method such as + `tf.estimator.MultiLabelHead`. + model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into an estimator to + continue training a previously saved model. + linear_feature_columns: An iterable containing all the feature columns + used by linear part of the model. All items in the set must be instances + of classes derived from `FeatureColumn`. + linear_optimizer: An instance of `tf.keras.optimizers.*` used to apply + gradients to the linear part of the model. Can also be a string (one of + 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to + FTRL optimizer. + dnn_feature_columns: An iterable containing all the feature columns used + by deep part of the model. All items in the set must be instances of + classes derived from `FeatureColumn`. + dnn_optimizer: An instance of `tf.keras.optimizers.*` used to apply + gradients to the deep part of the model. Can also be a string (one of + 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to + Adagrad optimizer. + dnn_hidden_units: List of hidden units per layer. All layers are fully + connected. + dnn_activation_fn: Activation function applied to each layer. If None, + will use `tf.nn.relu`. + dnn_dropout: When not None, the probability we will drop out a given + coordinate. + config: RunConfig object to configure the runtime settings. + batch_norm: Whether to use batch normalization after each hidden layer. + linear_sparse_combiner: A string specifying how to reduce the linear model + if a categorical column is multivalent. One of "mean", "sqrtn", and + "sum" -- these are effectively different ways to do example-level + normalization, which can be useful for bag-of-words features. For more + details, see `tf.feature_column.linear_model`. + + Raises: + ValueError: If both linear_feature_columns and dnn_features_columns are + empty at the same time. + """ + self._feature_columns = _validate_feature_columns( + linear_feature_columns=linear_feature_columns, + dnn_feature_columns=dnn_feature_columns) + estimator._canned_estimator_api_gauge.get_cell('Estimator').set( + 'DNNLinearCombined') # pylint: disable=protected-access + + def _model_fn(features, labels, mode, config): + """Call the _dnn_linear_combined_model_fn.""" + return _dnn_linear_combined_model_fn_v2( + features=features, + labels=labels, + mode=mode, + head=head, + linear_feature_columns=linear_feature_columns, + linear_optimizer=linear_optimizer, + dnn_feature_columns=dnn_feature_columns, + dnn_optimizer=dnn_optimizer, + dnn_hidden_units=dnn_hidden_units, + dnn_activation_fn=dnn_activation_fn, + dnn_dropout=dnn_dropout, + config=config, + batch_norm=batch_norm, + linear_sparse_combiner=linear_sparse_combiner) + + super(DNNLinearCombinedEstimatorV2, self).__init__( + model_fn=_model_fn, model_dir=model_dir, config=config) + + +@estimator_export(v1=['estimator.DNNLinearCombinedEstimator']) # pylint: disable=missing-docstring +class DNNLinearCombinedEstimator(estimator.Estimator): + __doc__ = DNNLinearCombinedEstimatorV2.__doc__ + + def __init__(self, + head, + model_dir=None, + linear_feature_columns=None, + linear_optimizer='Ftrl', + dnn_feature_columns=None, + dnn_optimizer='Adagrad', + dnn_hidden_units=None, + dnn_activation_fn=tf.nn.relu, + dnn_dropout=None, + input_layer_partitioner=None, + config=None, + batch_norm=False, + linear_sparse_combiner='sum'): + self._feature_columns = _validate_feature_columns( + linear_feature_columns=linear_feature_columns, + dnn_feature_columns=dnn_feature_columns) + estimator._canned_estimator_api_gauge.get_cell('Estimator').set( + 'DNNLinearCombined') # pylint: disable=protected-access + + def _model_fn(features, labels, mode, config): + """Call the _dnn_linear_combined_model_fn.""" + return _dnn_linear_combined_model_fn( + features=features, + labels=labels, + mode=mode, + head=head, + linear_feature_columns=linear_feature_columns, + linear_optimizer=linear_optimizer, + dnn_feature_columns=dnn_feature_columns, + dnn_optimizer=dnn_optimizer, + dnn_hidden_units=dnn_hidden_units, + dnn_activation_fn=dnn_activation_fn, + dnn_dropout=dnn_dropout, + input_layer_partitioner=input_layer_partitioner, + config=config, + batch_norm=batch_norm, + linear_sparse_combiner=linear_sparse_combiner) + + super(DNNLinearCombinedEstimator, self).__init__( + model_fn=_model_fn, model_dir=model_dir, config=config) + + +@estimator_export('estimator.DNNLinearCombinedRegressor', v1=[]) +class DNNLinearCombinedRegressorV2(estimator.EstimatorV2): + """An estimator for TensorFlow Linear and DNN joined models for regression. + + Note: This estimator is also known as wide-n-deep. + + Example: + + ```python + numeric_feature = numeric_column(...) + categorical_column_a = categorical_column_with_hash_bucket(...) + categorical_column_b = categorical_column_with_hash_bucket(...) + + categorical_feature_a_x_categorical_feature_b = crossed_column(...) + categorical_feature_a_emb = embedding_column( + categorical_column=categorical_feature_a, ...) + categorical_feature_b_emb = embedding_column( + categorical_column=categorical_feature_b, ...) + + estimator = tf.estimator.DNNLinearCombinedRegressor( + # wide settings + linear_feature_columns=[categorical_feature_a_x_categorical_feature_b], + linear_optimizer=tf.keras.optimizers.Ftrl(...), + # deep settings + dnn_feature_columns=[ + categorical_feature_a_emb, categorical_feature_b_emb, + numeric_feature], + dnn_hidden_units=[1000, 500, 100], + dnn_optimizer=tf.keras.optimizers.Adagrad(...), + # warm-start settings + warm_start_from="/path/to/checkpoint/dir") + + # To apply L1 and L2 regularization, you can set dnn_optimizer to: + tf.compat.v1.train.ProximalAdagradOptimizer( + learning_rate=0.1, + l1_regularization_strength=0.001, + l2_regularization_strength=0.001) + # To apply learning rate decay, you can set dnn_optimizer to a callable: + lambda: tf.keras.optimizers.Adam( + learning_rate=tf.compat.v1.train.exponential_decay( + learning_rate=0.1, + global_step=tf.compat.v1.train.get_global_step(), + decay_steps=10000, + decay_rate=0.96) + # It is the same for linear_optimizer. + + # Input builders + def input_fn_train: + # Returns tf.data.Dataset of (x, y) tuple where y represents label's class + # index. + pass + def input_fn_eval: + # Returns tf.data.Dataset of (x, y) tuple where y represents label's class + # index. + pass + def input_fn_predict: + # Returns tf.data.Dataset of (x, None) tuple. + pass + estimator.train(input_fn=input_fn_train, steps=100) + metrics = estimator.evaluate(input_fn=input_fn_eval, steps=10) + predictions = estimator.predict(input_fn=input_fn_predict) + ``` + + Input of `train` and `evaluate` should have following features, + otherwise there will be a `KeyError`: + + * for each `column` in `dnn_feature_columns` + `linear_feature_columns`: + - if `column` is a `CategoricalColumn`, a feature with `key=column.name` + whose `value` is a `SparseTensor`. + - if `column` is a `WeightedCategoricalColumn`, two features: the first + with `key` the id column name, the second with `key` the weight column + name. Both features' `value` must be a `SparseTensor`. + - if `column` is a `DenseColumn`, a feature with `key=column.name` + whose `value` is a `Tensor`. + + Loss is calculated by using mean squared error. + + @compatibility(eager) + Estimators can be used while eager execution is enabled. Note that `input_fn` + and all hooks are executed inside a graph context, so they have to be written + to be compatible with graph mode. Note that `input_fn` code using `tf.data` + generally works in both graph and eager modes. + @end_compatibility + """ + + def __init__(self, + model_dir=None, + linear_feature_columns=None, + linear_optimizer='Ftrl', + dnn_feature_columns=None, + dnn_optimizer='Adagrad', + dnn_hidden_units=None, + dnn_activation_fn=tf.nn.relu, + dnn_dropout=None, + label_dimension=1, + weight_column=None, + config=None, + warm_start_from=None, + loss_reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE, + batch_norm=False, + linear_sparse_combiner='sum'): + """Initializes a DNNLinearCombinedRegressor instance. + + Args: + model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator to + continue training a previously saved model. + linear_feature_columns: An iterable containing all the feature columns + used by linear part of the model. All items in the set must be instances + of classes derived from `FeatureColumn`. + linear_optimizer: An instance of `tf.keras.optimizers.*` used to apply + gradients to the linear part of the model. Can also be a string (one of + 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to + FTRL optimizer. + dnn_feature_columns: An iterable containing all the feature columns used + by deep part of the model. All items in the set must be instances of + classes derived from `FeatureColumn`. + dnn_optimizer: An instance of `tf.keras.optimizers.*` used to apply + gradients to the deep part of the model. Can also be a string (one of + 'Adagrad', 'Adam', 'Ftrl', 'RMSProp', 'SGD'), or callable. Defaults to + Adagrad optimizer. + dnn_hidden_units: List of hidden units per layer. All layers are fully + connected. + dnn_activation_fn: Activation function applied to each layer. If None, + will use `tf.nn.relu`. + dnn_dropout: When not None, the probability we will drop out a given + coordinate. + label_dimension: Number of regression targets per example. This is the + size of the last dimension of the labels and logits `Tensor` objects + (typically, these have shape `[batch_size, label_dimension]`). + weight_column: A string or a `NumericColumn` created by + `tf.feature_column.numeric_column` defining feature column representing + weights. It is used to down weight or boost examples during training. It + will be multiplied by the loss of the example. If it is a string, it is + used as a key to fetch weight tensor from the `features`. If it is a + `_NumericColumn`, raw tensor is fetched by key `weight_column.key`, then + weight_column.normalizer_fn is applied on it to get weight tensor. + config: RunConfig object to configure the runtime settings. + warm_start_from: A string filepath to a checkpoint to warm-start from, or + a `WarmStartSettings` object to fully configure warm-starting. If the + string filepath is provided instead of a `WarmStartSettings`, then all + weights are warm-started, and it is assumed that vocabularies and Tensor + names are unchanged. + loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how + to reduce training loss over batch. Defaults to `SUM_OVER_BATCH_SIZE`. + batch_norm: Whether to use batch normalization after each hidden layer. + linear_sparse_combiner: A string specifying how to reduce the linear model + if a categorical column is multivalent. One of "mean", "sqrtn", and + "sum" -- these are effectively different ways to do example-level + normalization, which can be useful for bag-of-words features. For more + details, see `tf.feature_column.linear_model`. + + Raises: + ValueError: If both linear_feature_columns and dnn_features_columns are + empty at the same time. + """ + self._feature_columns = _validate_feature_columns( + linear_feature_columns=linear_feature_columns, + dnn_feature_columns=dnn_feature_columns) + + head = regression_head.RegressionHead( + label_dimension=label_dimension, + weight_column=weight_column, + loss_reduction=loss_reduction) + estimator._canned_estimator_api_gauge.get_cell('Regressor').set( + 'DNNLinearCombined') # pylint: disable=protected-access + + def _model_fn(features, labels, mode, config): + """Call the _dnn_linear_combined_model_fn.""" + return _dnn_linear_combined_model_fn_v2( + features=features, + labels=labels, + mode=mode, + head=head, + linear_feature_columns=linear_feature_columns, + linear_optimizer=linear_optimizer, + dnn_feature_columns=dnn_feature_columns, + dnn_optimizer=dnn_optimizer, + dnn_hidden_units=dnn_hidden_units, + dnn_activation_fn=dnn_activation_fn, + dnn_dropout=dnn_dropout, + config=config, + batch_norm=batch_norm, + linear_sparse_combiner=linear_sparse_combiner) + + super(DNNLinearCombinedRegressorV2, self).__init__( + model_fn=_model_fn, + model_dir=model_dir, + config=config, + warm_start_from=warm_start_from) + + +@estimator_export(v1=['estimator.DNNLinearCombinedRegressor']) # pylint: disable=missing-docstring +class DNNLinearCombinedRegressor(estimator.Estimator): + __doc__ = DNNLinearCombinedRegressorV2.__doc__.replace( + 'SUM_OVER_BATCH_SIZE', 'SUM') + + def __init__(self, + model_dir=None, + linear_feature_columns=None, + linear_optimizer='Ftrl', + dnn_feature_columns=None, + dnn_optimizer='Adagrad', + dnn_hidden_units=None, + dnn_activation_fn=tf.nn.relu, + dnn_dropout=None, + label_dimension=1, + weight_column=None, + input_layer_partitioner=None, + config=None, + warm_start_from=None, + loss_reduction=tf.compat.v1.losses.Reduction.SUM, + batch_norm=False, + linear_sparse_combiner='sum'): + self._feature_columns = _validate_feature_columns( + linear_feature_columns=linear_feature_columns, + dnn_feature_columns=dnn_feature_columns) + estimator._canned_estimator_api_gauge.get_cell('Regressor').set( + 'DNNLinearCombined') # pylint: disable=protected-access + + head = head_lib._regression_head( # pylint: disable=protected-access + label_dimension=label_dimension, + weight_column=weight_column, + loss_reduction=loss_reduction) + + def _model_fn(features, labels, mode, config): + """Call the _dnn_linear_combined_model_fn.""" + return _dnn_linear_combined_model_fn( + features=features, + labels=labels, + mode=mode, + head=head, + linear_feature_columns=linear_feature_columns, + linear_optimizer=linear_optimizer, + dnn_feature_columns=dnn_feature_columns, + dnn_optimizer=dnn_optimizer, + dnn_hidden_units=dnn_hidden_units, + dnn_activation_fn=dnn_activation_fn, + dnn_dropout=dnn_dropout, + input_layer_partitioner=input_layer_partitioner, + config=config, + batch_norm=batch_norm, + linear_sparse_combiner=linear_sparse_combiner) + + super(DNNLinearCombinedRegressor, self).__init__( + model_fn=_model_fn, + model_dir=model_dir, + config=config, + warm_start_from=warm_start_from) \ No newline at end of file diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/hooks/benchmark_hooks.py b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/hooks/benchmark_hooks.py similarity index 100% rename from TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/hooks/benchmark_hooks.py rename to TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/hooks/benchmark_hooks.py diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/hooks/training_hooks.py b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/hooks/training_hooks.py similarity index 100% rename from TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/hooks/training_hooks.py rename to TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/hooks/training_hooks.py diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/metrics.py b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/metrics.py similarity index 100% rename from TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/metrics.py rename to TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/metrics.py diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/schedulers.py b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/schedulers.py similarity index 100% rename from TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/utils/schedulers.py rename to TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/util/schedulers.py -- Gitee From 218f5595b9d3512868a1a36d9af949f81c8c600e Mon Sep 17 00:00:00 2001 From: piggies <724022815@qq.com> Date: Tue, 12 Apr 2022 14:44:45 +0800 Subject: [PATCH 2/6] update --- .../WideDeep_ID2940_for_TensorFlow/config/1p.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_0.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_1.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_2.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_3.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_4.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_5.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_6.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_7.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/8p.json | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p.json index 96078ec8d..8f32ddace 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p.json @@ -7,7 +7,7 @@ { "device_id": "0", "device_ip": "192.1.2.8", - "rank_id": "0", + "rank_id": "0" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json index 96078ec8d..8f32ddace 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json @@ -7,7 +7,7 @@ { "device_id": "0", "device_ip": "192.1.2.8", - "rank_id": "0", + "rank_id": "0" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json index 8270cbe0e..b2158211c 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json @@ -7,7 +7,7 @@ { "device_id": "1", "device_ip": "192.1.2.8", - "rank_id": "1", + "rank_id": "1" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json index 0438819dd..fe6f76a22 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json @@ -7,7 +7,7 @@ { "device_id": "2", "device_ip": "192.1.2.8", - "rank_id": "2", + "rank_id": "2" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json index ca9a5ff4e..0b61fe355 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json @@ -7,7 +7,7 @@ { "device_id": "3", "device_ip": "192.1.2.8", - "rank_id": "3", + "rank_id": "3" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json index 90872c5e1..32ac84e72 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json @@ -7,7 +7,7 @@ { "device_id": "4", "device_ip": "192.4.2.9", - "rank_id": "4", + "rank_id": "4" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json index 1b1322f11..6b2c248e9 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json @@ -7,7 +7,7 @@ { "device_id": "5", "device_ip": "192.4.2.9", - "rank_id": "5", + "rank_id": "5" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json index ea26227f6..67e1e5f17 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json @@ -7,7 +7,7 @@ { "device_id": "6", "device_ip": "192.4.2.9", - "rank_id": "6", + "rank_id": "6" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json index 1ccc40abf..66ccdd167 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json @@ -7,7 +7,7 @@ { "device_id": "7", "device_ip": "192.4.2.9", - "rank_id": "7", + "rank_id": "7" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/8p.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/8p.json index d13441e11..0a60d5f3e 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/8p.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/8p.json @@ -11,7 +11,7 @@ {"device_id":"4","device_ip":"192.4.2.9","rank_id":"4"}, {"device_id":"5","device_ip":"192.4.2.9","rank_id":"5"}, {"device_id":"6","device_ip":"192.4.2.9","rank_id":"6"}, - {"device_id":"7","device_ip":"192.4.2.9","rank_id":"7"}, + {"device_id":"7","device_ip":"192.4.2.9","rank_id":"7"} ], "server_id":"10.155.111.118" -- Gitee From 70feaa53ab8d8b49a80644152e3e666ca1ab3b9e Mon Sep 17 00:00:00 2001 From: piggies <724022815@qq.com> Date: Tue, 12 Apr 2022 14:53:00 +0800 Subject: [PATCH 3/6] update --- .../WideDeep_ID2940_for_TensorFlow/config/1p_1.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_2.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_3.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_4.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_5.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_6.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/config/1p_7.json | 2 +- .../WideDeep_ID2940_for_TensorFlow/test/train_performance_1p.sh | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json index b2158211c..fea1c4649 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json @@ -7,7 +7,7 @@ { "device_id": "1", "device_ip": "192.1.2.8", - "rank_id": "1" + "rank_id": "0" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json index fe6f76a22..70bc18bc1 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json @@ -7,7 +7,7 @@ { "device_id": "2", "device_ip": "192.1.2.8", - "rank_id": "2" + "rank_id": "0" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json index 0b61fe355..72ffe7207 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json @@ -7,7 +7,7 @@ { "device_id": "3", "device_ip": "192.1.2.8", - "rank_id": "3" + "rank_id": "0" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json index 32ac84e72..8ce06df65 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json @@ -7,7 +7,7 @@ { "device_id": "4", "device_ip": "192.4.2.9", - "rank_id": "4" + "rank_id": "0" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json index 6b2c248e9..856384d15 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json @@ -7,7 +7,7 @@ { "device_id": "5", "device_ip": "192.4.2.9", - "rank_id": "5" + "rank_id": "0" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json index 67e1e5f17..6f0b88997 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json @@ -7,7 +7,7 @@ { "device_id": "6", "device_ip": "192.4.2.9", - "rank_id": "6" + "rank_id": "0" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json index 66ccdd167..beef41b4e 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json @@ -7,7 +7,7 @@ { "device_id": "7", "device_ip": "192.4.2.9", - "rank_id": "7" + "rank_id": "0" } ], "server_id": "10.155.111.118" diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/test/train_performance_1p.sh b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/test/train_performance_1p.sh index 4658e94f6..f26c1ae41 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/test/train_performance_1p.sh +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/test/train_performance_1p.sh @@ -98,7 +98,7 @@ fi if [ -d $cur_path/../config/1p_$ASCEND_DEVICE.json ];then export RANK_TABLE_FILE=$cur_path/../config/1p_$ASCEND_DEVICE.json - export RANK_ID=$ASCEND_DEVICE_ID + export RANK_ID=0 else export RANK_TABLE_FILE=$cur_path/../config/1p_0.json export RANK_ID=0 -- Gitee From 8e289d77d90c18e9103f7e64a0ae7243f515a057 Mon Sep 17 00:00:00 2001 From: piggies <724022815@qq.com> Date: Tue, 12 Apr 2022 14:59:46 +0800 Subject: [PATCH 4/6] updat --- .../WideDeep_ID2940_for_TensorFlow/config/1p_0.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json index 8f32ddace..e7cd0f2df 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json @@ -1,6 +1,6 @@ { - "group_count": "1", - "group_list": + "server_count": "1", + "server_list": [ { "devices": [ -- Gitee From 6d83f1c455a368732a6f9459adf42e40b94a004a Mon Sep 17 00:00:00 2001 From: piggies <724022815@qq.com> Date: Tue, 12 Apr 2022 15:16:15 +0800 Subject: [PATCH 5/6] update --- .../config/1p.json | 4 +-- .../config/1p_0.json | 32 +++++++++---------- .../config/1p_1.json | 4 +-- .../config/1p_2.json | 4 +-- .../config/1p_3.json | 4 +-- .../config/1p_4.json | 4 +-- .../config/1p_5.json | 4 +-- .../config/1p_6.json | 4 +-- .../config/1p_7.json | 4 +-- 9 files changed, 32 insertions(+), 32 deletions(-) diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p.json index 8f32ddace..e7cd0f2df 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p.json @@ -1,6 +1,6 @@ { - "group_count": "1", - "group_list": + "server_count": "1", + "server_list": [ { "devices": [ diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json index e7cd0f2df..5120f0e87 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_0.json @@ -1,18 +1,18 @@ { - "server_count": "1", - "server_list": - [ - { - "devices": [ - { - "device_id": "0", - "device_ip": "192.1.2.8", - "rank_id": "0" - } - ], - "server_id": "10.155.111.118" - } - ], - "status": "completed", - "version":"1.0" +"server_count": "1", +"server_list": +[ + { + "devices": [ + { + "device_id": "0", + "device_ip": "192.1.2.8", + "rank_id": "0" + } + ], + "server_id": "10.155.111.118" + } +], +"status": "completed", +"version":"1.0" } \ No newline at end of file diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json index fea1c4649..9dde26beb 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_1.json @@ -1,6 +1,6 @@ { - "group_count": "1", - "group_list": + "server_count": "1", + "server_list": [ { "devices": [ diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json index 70bc18bc1..de665561e 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_2.json @@ -1,6 +1,6 @@ { - "group_count": "1", - "group_list": + "server_count": "1", + "server_list": [ { "devices": [ diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json index 72ffe7207..469e32252 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_3.json @@ -1,6 +1,6 @@ { - "group_count": "1", - "group_list": + "server_count": "1", + "server_list": [ { "devices": [ diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json index 8ce06df65..4c9272c5b 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_4.json @@ -1,6 +1,6 @@ { - "group_count": "1", - "group_list": + "server_count": "1", + "server_list": [ { "devices": [ diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json index 856384d15..cb7e81a93 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_5.json @@ -1,6 +1,6 @@ { - "group_count": "1", - "group_list": + "server_count": "1", + "server_list": [ { "devices": [ diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json index 6f0b88997..539e9d4aa 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_6.json @@ -1,6 +1,6 @@ { - "group_count": "1", - "group_list": + "server_count": "1", + "server_list": [ { "devices": [ diff --git a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json index beef41b4e..479df9ad5 100644 --- a/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json +++ b/TensorFlow/built-in/recommendation/WideDeep_ID2940_for_TensorFlow/config/1p_7.json @@ -1,6 +1,6 @@ { - "group_count": "1", - "group_list": + "server_count": "1", + "server_list": [ { "devices": [ -- Gitee From 9b49893324ee09b844cdad8aed3719ca9e1b7ac4 Mon Sep 17 00:00:00 2001 From: piggies <724022815@qq.com> Date: Tue, 12 Apr 2022 19:22:24 +0800 Subject: [PATCH 6/6] upate --- .../core/sharedqueue.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 TensorFlow2/built-in/cv/detection/YOLOv5_ID1719_for_TensorFlow2.X/core/sharedqueue.py diff --git a/TensorFlow2/built-in/cv/detection/YOLOv5_ID1719_for_TensorFlow2.X/core/sharedqueue.py b/TensorFlow2/built-in/cv/detection/YOLOv5_ID1719_for_TensorFlow2.X/core/sharedqueue.py new file mode 100644 index 000000000..cd8203f41 --- /dev/null +++ b/TensorFlow2/built-in/cv/detection/YOLOv5_ID1719_for_TensorFlow2.X/core/sharedqueue.py @@ -0,0 +1,99 @@ +import numpy as np + +class SafeQueue: + def __init__(self, size): + # space for close signal + size += 1 + import multiprocessing.sharedctypes + self.cvar = multiprocessing.Condition() + sary = multiprocessing.sharedctypes.RawArray('b', 8*size) + self.vals = np.frombuffer(sary, dtype=np.int64, count=size) + self.vals[:] = -1 + # tail is the next element to be read from the queue + self.tail = multiprocessing.sharedctypes.RawValue('l', 0) + # size is the current number of items in the queue. head = tail + size + self.size = multiprocessing.sharedctypes.RawValue('l', 0) + + def put(self, value): + assert (value >= 0) + with self.cvar: + assert (self.size.value < len(self.vals)) + + head = (self.tail.value + self.size.value) % len(self.vals) + self.vals[head] = value + self.size.value += 1 + self.cvar.notify() + + def get(self): + with self.cvar: + while True: + if self.size.value > 0: + ret = self.vals[self.tail.value] + self.vals[self.tail.value] = -1 + self.tail.value = (self.tail.value + 1) % len(self.vals) + self.size.value -= 1 + assert (ret >= 0) + return ret + self.cvar.wait() + + def get_size(self): + return self.size.value + +class BufferQueue: + def __init__(self, size, template): + import multiprocessing.sharedctypes + # records indexes of elements that are ready to be read or written into + self.read_queue = SafeQueue(size) + self.write_queue = SafeQueue(size) + self.write_index = -1 + self.arrays= [] + + for i in range(size): + data = [] + for n in range(len(template)): + nbytes = template[n].nbytes + dtype = template[n].dtype + size = template[n].size + shape = template[n].shape + sharedmem = multiprocessing.sharedctypes.RawArray('b', nbytes) + data.append(np.frombuffer(sharedmem, dtype=dtype, count=size).reshape(shape)) + self.arrays.append(data) + self.write_queue.put(i) + + class Guard: + def __init__(self, output, arys, idx_op): + self.output = output + self.arys = arys + self.idx_op = idx_op + + def __enter(self): + self.index = self.idx_op() + return self.arys[self.index] + + def __exit__(self, *args): + if self.output is not None: + self.output.put(self.index) + + def put(self, data): + with self.Guard(self.read_queue, self.arrays, self._put_idx) as ary: + for i in range(len(data)): + ary[i][:] = data[i] + + def _put_idx(self): + return self.write_queue.get() + + def get(self): + with self._get() as ary: + return ary + + def _get_idx(self): + return self.read_queue.get() + + def _get(self): + if self.write_index >= 0: + self.write_queue.put(self.write_index) + self.write_index = self._get_idx() + return self.Guard(None, self.arrays, lambda:self.write_index) + + def get_size(self): + return self.read_queue.get_size() \ No newline at end of file -- Gitee