From e7c09340a78f8dad30093e430d4adf70b714b9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:31:59 +0000 Subject: [PATCH 01/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20DIN=5FID0190=5Ffor?= =?UTF-8?q?=5FTensorFlow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../built-in/recommendation/DIN_ID0190_for_TensorFlow/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From 3eb4f20d48dccd11b9d198395fd779cad7f55020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:32:18 +0000 Subject: [PATCH 02/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../DIN_ID0190_for_TensorFlow/.gitattributes | 17 ++ .../DIN_ID0190_for_TensorFlow/.gitignore | 144 +++++++++++++ .../.readthedocs.yml | 5 + .../DIN_ID0190_for_TensorFlow/.travis.yml | 70 ++++++ .../DIN_ID0190_for_TensorFlow/CONTRIBUTING.md | 21 ++ .../DIN_ID0190_for_TensorFlow/LICENSE | 201 ++++++++++++++++++ .../DIN_ID0190_for_TensorFlow/README.md | 136 ++++++++++++ .../DIN_ID0190_for_TensorFlow/setup.cfg | 32 +++ .../DIN_ID0190_for_TensorFlow/setup.py | 58 +++++ 9 files changed, 684 insertions(+) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.gitattributes create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.gitignore create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.readthedocs.yml create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.travis.yml create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/CONTRIBUTING.md create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/LICENSE create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/README.md create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/setup.cfg create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/setup.py diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.gitattributes b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.gitattributes new file mode 100644 index 000000000..bdb0cabc8 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.gitattributes @@ -0,0 +1,17 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# Custom for Visual Studio +*.cs diff=csharp + +# Standard to msysgit +*.doc diff=astextplain +*.DOC diff=astextplain +*.docx diff=astextplain +*.DOCX diff=astextplain +*.dot diff=astextplain +*.DOT diff=astextplain +*.pdf diff=astextplain +*.PDF diff=astextplain +*.rtf diff=astextplain +*.RTF diff=astextplain diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.gitignore b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.gitignore new file mode 100644 index 000000000..499605f0d --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.gitignore @@ -0,0 +1,144 @@ +*.h5 +*.ipynb +.pytest_cache/ +.vscode/ +tests/unused/* +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +.idea/ +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask instance folder +instance/ + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# ========================= +# Operating System Files +# ========================= + +# OSX +# ========================= + +.DS_Store +.AppleDouble +.LSOverride + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# Windows +# ========================= + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.readthedocs.yml b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.readthedocs.yml new file mode 100644 index 000000000..1afb6e706 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.readthedocs.yml @@ -0,0 +1,5 @@ +build: + image: latest + +python: + version: 3.6 \ No newline at end of file diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.travis.yml b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.travis.yml new file mode 100644 index 000000000..4470dfa08 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.travis.yml @@ -0,0 +1,70 @@ +#sudo: required +#dist: trusty xenial +language: python + +python: + - "2.7" #time out + #- "3.4" + - "3.5" + - "3.6" + #- "3.7" + +env: +# - TF_VERSION=1.13.1 +# - TF_VERSION=1.12.2 + - TF_VERSION=1.4.0 + #Not Support- TF_VERSION=1.7.0 + #Not Support- TF_VERSION=1.7.1 + #Not Support- TF_VERSION=1.8.0 + #- TF_VERSION=1.8.0 + # - TF_VERSION=1.11.0 + #- TF_VERSION=1.6.0 + - TF_VERSION=2.0.0b1 + #- TF_VERSION=1.13.2 + - TF_VERSION=1.14.0 + +matrix: + allow_failures: + - python: "2.7" + env: TF_VERSION=1.6.0 # to speed up + - python: "2.7" + env: TF_VERSION=2.0.0b1 + - python: "3.4" + - python: "3.5" + - python: "3.7" + - env: TF_VERSION=1.5.0 #local is ok,but sometimes CI is failed + - env: TF_VERSION=1.7.0 + - env: TF_VERSION=1.7.1 + - env: TF_VERSION=1.8.0 + - env: TF_VERSION=1.12.0 # too slow + - env: TF_VERSION=1.13.1 # too slow + - env: TF_VERSION=1.13.2 # too slow + - env: TF_VERSION=1.14.0 # too slow + + fast_finish: true + +cache: pip +# command to install dependencies +install: + - pip install -q pytest-cov==2.4.0 #>=2.4.0,<2.6 + - pip install -q python-coveralls + - pip install -q codacy-coverage + - pip install -q tensorflow==$TF_VERSION + - pip install -q pandas + - pip install -q packaging + - pip install -e . +# command to run tests +script: + - pytest --cov=deepctr + +notifications: + recipients: + - weichenswc@163.com + + on_success: change + on_failure: change + +after_success: + - coveralls + - coverage xml + - python-codacy-coverage -r coverage.xml diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/CONTRIBUTING.md b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/CONTRIBUTING.md new file mode 100644 index 000000000..c7258d576 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/CONTRIBUTING.md @@ -0,0 +1,21 @@ +This project is under development and we need developers to participate in. +# Join us +If you + +- familiar with and interested in ctr prediction algorithms +- familiar with tensorflow +- have spare time to learn and develop +- familiar with git + +please send a brief introduction of your background and experience to weichenswc@163.com, welcome to join us! + +# Creating a pull request +1. **Become a collaborator**: Send an email with introduction and your github account name to weichenswc@163.com and waiting for invitation to become a collaborator. +2. **Fork&Dev**: Fork your own branch(`dev_yourname`) in `DeepCTR` from the `master` branch for development.If the `master` is updated during the development process, remember to merge and update to `dev_yourname` regularly. +3. **Testing**: Test logical correctness and effect when finishing the code development of the `dev_yourname` branch. +4. **Pre-release** : After testing contact weichenswc@163.com for pre-release integration, usually your branch `dev_yourname` will be merged into `release` branch by squash merge. +5. **Release a new version**: After confirming that the change is no longer needed, `release` branch will be merged into `master` and a new python package will be released on pypi. + +# Discussions + +https://github.com/shenweichen/DeepCTR/discussions \ No newline at end of file diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/LICENSE b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/LICENSE new file mode 100644 index 000000000..09dfe10e2 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017-present Weichen Shen + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/README.md b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/README.md new file mode 100644 index 000000000..ec42fd311 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/README.md @@ -0,0 +1,136 @@ +# DeepCTR + +[![Python Versions](https://img.shields.io/pypi/pyversions/deepctr.svg)](https://pypi.org/project/deepctr) +[![TensorFlow Versions](https://img.shields.io/badge/TensorFlow-1.4+/2.0+-blue.svg)](https://pypi.org/project/deepctr) +[![Downloads](https://pepy.tech/badge/deepctr)](https://pepy.tech/project/deepctr) +[![PyPI Version](https://img.shields.io/pypi/v/deepctr.svg)](https://pypi.org/project/deepctr) +[![GitHub Issues](https://img.shields.io/github/issues/shenweichen/deepctr.svg +)](https://github.com/shenweichen/deepctr/issues) + + + +[![Documentation Status](https://readthedocs.org/projects/deepctr-doc/badge/?version=latest)](https://deepctr-doc.readthedocs.io/) +![CI status](https://github.com/shenweichen/deepctr/workflows/CI/badge.svg) +[![codecov](https://codecov.io/gh/shenweichen/DeepCTR/branch/master/graph/badge.svg)](https://codecov.io/gh/shenweichen/DeepCTR) +[![Codacy Badge](https://api.codacy.com/project/badge/Grade/d4099734dc0e4bab91d332ead8c0bdd0)](https://www.codacy.com/gh/shenweichen/DeepCTR?utm_source=github.com&utm_medium=referral&utm_content=shenweichen/DeepCTR&utm_campaign=Badge_Grade) +[![Disscussion](https://img.shields.io/badge/chat-wechat-brightgreen?style=flat)](./README.md#DisscussionGroup) +[![License](https://img.shields.io/github/license/shenweichen/deepctr.svg)](https://github.com/shenweichen/deepctr/blob/master/LICENSE) + + + +DeepCTR is a **Easy-to-use**, **Modular** and **Extendible** package of deep-learning based CTR models along with lots of +core components layers which can be used to easily build custom models.You can use any complex model with `model.fit()` +，and `model.predict()` . + +- Provide `tf.keras.Model` like interfaces for **quick experiment**. [example](https://deepctr-doc.readthedocs.io/en/latest/Quick-Start.html#getting-started-4-steps-to-deepctr) +- Provide `tensorflow estimator` interface for **large scale data** and **distributed training**. [example](https://deepctr-doc.readthedocs.io/en/latest/Quick-Start.html#getting-started-4-steps-to-deepctr-estimator-with-tfrecord) +- It is compatible with both `tf 1.x` and `tf 2.x`. + +Some related projects: + +- DeepMatch: https://github.com/shenweichen/DeepMatch +- DeepCTR-Torch: https://github.com/shenweichen/DeepCTR-Torch + +Let's [**Get Started!**](https://deepctr-doc.readthedocs.io/en/latest/Quick-Start.html)([Chinese +Introduction](https://zhuanlan.zhihu.com/p/53231955)) and [welcome to join us!](./CONTRIBUTING.md) + +## Models List + +| Model | Paper | +| :------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Convolutional Click Prediction Model | [CIKM 2015][A Convolutional Click Prediction Model](http://ir.ia.ac.cn/bitstream/173211/12337/1/A%20Convolutional%20Click%20Prediction%20Model.pdf) | +| Factorization-supported Neural Network | [ECIR 2016][Deep Learning over Multi-field Categorical Data: A Case Study on User Response Prediction](https://arxiv.org/pdf/1601.02376.pdf) | +| Product-based Neural Network | [ICDM 2016][Product-based neural networks for user response prediction](https://arxiv.org/pdf/1611.00144.pdf) | +| Wide & Deep | [DLRS 2016][Wide & Deep Learning for Recommender Systems](https://arxiv.org/pdf/1606.07792.pdf) | +| DeepFM | [IJCAI 2017][DeepFM: A Factorization-Machine based Neural Network for CTR Prediction](http://www.ijcai.org/proceedings/2017/0239.pdf) | +| Piece-wise Linear Model | [arxiv 2017][Learning Piece-wise Linear Models from Large Scale Data for Ad Click Prediction](https://arxiv.org/abs/1704.05194) | +| Deep & Cross Network | [ADKDD 2017][Deep & Cross Network for Ad Click Predictions](https://arxiv.org/abs/1708.05123) | +| Attentional Factorization Machine | [IJCAI 2017][Attentional Factorization Machines: Learning the Weight of Feature Interactions via Attention Networks](http://www.ijcai.org/proceedings/2017/435) | +| Neural Factorization Machine | [SIGIR 2017][Neural Factorization Machines for Sparse Predictive Analytics](https://arxiv.org/pdf/1708.05027.pdf) | +| xDeepFM | [KDD 2018][xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems](https://arxiv.org/pdf/1803.05170.pdf) | +| Deep Interest Network | [KDD 2018][Deep Interest Network for Click-Through Rate Prediction](https://arxiv.org/pdf/1706.06978.pdf) | +| AutoInt | [CIKM 2019][AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks](https://arxiv.org/abs/1810.11921) | +| Deep Interest Evolution Network | [AAAI 2019][Deep Interest Evolution Network for Click-Through Rate Prediction](https://arxiv.org/pdf/1809.03672.pdf) | +| FwFM | [WWW 2018][Field-weighted Factorization Machines for Click-Through Rate Prediction in Display Advertising](https://arxiv.org/pdf/1806.03514.pdf) | +| ONN | [arxiv 2019][Operation-aware Neural Networks for User Response Prediction](https://arxiv.org/pdf/1904.12579.pdf) | +| FGCNN | [WWW 2019][Feature Generation by Convolutional Neural Network for Click-Through Rate Prediction ](https://arxiv.org/pdf/1904.04447) | +| Deep Session Interest Network | [IJCAI 2019][Deep Session Interest Network for Click-Through Rate Prediction ](https://arxiv.org/abs/1905.06482) | +| FiBiNET | [RecSys 2019][FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction](https://arxiv.org/pdf/1905.09433.pdf) | +| FLEN | [arxiv 2019][FLEN: Leveraging Field for Scalable CTR Prediction](https://arxiv.org/pdf/1911.04690.pdf) | +| BST | [DLP-KDD 2019][Behavior sequence transformer for e-commerce recommendation in Alibaba](https://arxiv.org/pdf/1905.06874.pdf) | +| IFM | [IJCAI 2019][An Input-aware Factorization Machine for Sparse Prediction](https://www.ijcai.org/Proceedings/2019/0203.pdf) | +| DCN V2 | [arxiv 2020][DCN V2: Improved Deep & Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/abs/2008.13535) | +| DIFM | [IJCAI 2020][A Dual Input-aware Factorization Machine for CTR Prediction](https://www.ijcai.org/Proceedings/2020/0434.pdf) | +| FEFM and DeepFEFM | [arxiv 2020][Field-Embedded Factorization Machines for Click-through rate prediction](https://arxiv.org/abs/2009.09931) | +| SharedBottom | [arxiv 2017][An Overview of Multi-Task Learning in Deep Neural Networks](https://arxiv.org/pdf/1706.05098.pdf) | +| ESMM | [SIGIR 2018][Entire Space Multi-Task Model: An Effective Approach for Estimating Post-Click Conversion Rate](https://arxiv.org/abs/1804.07931) | +| MMOE | [KDD 2018][Modeling Task Relationships in Multi-task Learning with Multi-gate Mixture-of-Experts](https://dl.acm.org/doi/abs/10.1145/3219819.3220007) | +| PLE | [RecSys 2020][Progressive Layered Extraction (PLE): A Novel Multi-Task Learning (MTL) Model for Personalized Recommendations](https://dl.acm.org/doi/10.1145/3383313.3412236) | +| EDCN | [KDD 2021][Enhancing Explicit and Implicit Feature Interactions via Information Sharing for Parallel Deep CTR Models](https://dlp-kdd.github.io/assets/pdf/DLP-KDD_2021_paper_12.pdf) | + +## Citation + +- Weichen Shen. (2017). DeepCTR: Easy-to-use,Modular and Extendible package of deep-learning based CTR + models. https://github.com/shenweichen/deepctr. + +If you find this code useful in your research, please cite it using the following BibTeX: + +```bibtex +@misc{shen2017deepctr, + author = {Weichen Shen}, + title = {DeepCTR: Easy-to-use,Modular and Extendible package of deep-learning based CTR models}, + year = {2017}, + publisher = {GitHub}, + journal = {GitHub Repository}, + howpublished = {\url{https://github.com/shenweichen/deepctr}}, +} +``` + +## DisscussionGroup + +- [Github Discussions](https://github.com/shenweichen/DeepCTR/discussions) +- Wechat Discussions + +|公众号：浅梦学习笔记|微信：deepctrbot|学习小组 [加入](https://t.zsxq.com/026UJEuzv) [主题集合](https://mp.weixin.qq.com/mp/appmsgalbum?__biz=MjM5MzY4NzE3MA==&action=getalbum&album_id=1361647041096843265&scene=126#wechat_redirect)| +|:--:|:--:|:--:| +| [![公众号](./docs/pics/code.png)](https://github.com/shenweichen/AlgoNotes)| [![微信](./docs/pics/deepctrbot.png)](https://github.com/shenweichen/AlgoNotes)|[![学习小组](./docs/pics/planet_github.png)](https://t.zsxq.com/026UJEuzv)| + +## Main contributors([welcome to join us!](./CONTRIBUTING.md)) + + + + + + + + + + + + +
+ ​ pic
+ ​ Shen Weichen ​ +

+ Alibaba Group

​ + 		+ pic
+ Zan Shuxun ​ +

Alibaba Group

​ + 		+ ​ pic
+ ​ Harshit Pande +

Amazon

​ + 		+ ​ pic
+ ​ Lai Mincai +

ByteDance

​ + 		+ ​ pic
+ ​ Li Zichao +

ByteDance

​ + 		+ ​ pic
+ Tan Tingyi +

Chongqing University
of Posts and
Telecommunications

​ +
diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/setup.cfg b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/setup.cfg new file mode 100644 index 000000000..b51f13b93 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/setup.cfg @@ -0,0 +1,32 @@ +[metadata] +desciption-file = README.md + +#[coverage:run] +#branch = True + +[coverage:report] +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise ValueError + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if False: + if __name__ == .__main__.: + +[coverage:run] +omit = + # omit anything in a .local directory anywhere + #*/.local/* + # omit everything in /usr + deepctr/contrib/* + # omit this single file + #utils/tirefire.py \ No newline at end of file diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/setup.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/setup.py new file mode 100644 index 000000000..3a52f4b8c --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/setup.py @@ -0,0 +1,58 @@ +from npu_bridge.npu_init import * +import sys + +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() + +REQUIRED_PACKAGES = [ + 'requests', + 'h5py==3.7.0; python_version>="3.9"', + 'h5py==2.10.0; python_version<"3.9"' +] + +setuptools.setup( + name="deepctr", + version="0.9.3", + author="Weichen Shen", + author_email="weichenswc@163.com", + description="Easy-to-use,Modular and Extendible package of deep learning based CTR(Click Through Rate) prediction models with tensorflow 1.x and 2.x .", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/shenweichen/deepctr", + download_url='https://github.com/shenweichen/deepctr/tags', + packages=setuptools.find_packages( + exclude=["tests", "tests.models", "tests.layers"]), + python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*", # '>=3.4', # 3.4.6 + install_requires=REQUIRED_PACKAGES, + extras_require={ + "cpu": ["tensorflow>=1.4.0,!=1.7.*,!=1.8.*"], + "gpu": ["tensorflow-gpu>=1.4.0,!=1.7.*,!=1.8.*"], + }, + entry_points={ + }, + classifiers=( + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ), + license="Apache-2.0", + keywords=['ctr', 'click through rate', + 'deep learning', 'tensorflow', 'tensor', 'keras'], +) + -- Gitee From 7269a9eace359f1d577a33fbf460abb50cd6b2d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:32:34 +0000 Subject: [PATCH 03/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20.github?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../recommendation/DIN_ID0190_for_TensorFlow/.github/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From 229edb95395c6d421c09944807538c706df60bc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:32:57 +0000 Subject: [PATCH 04/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20ISSUE=5FTEMPLATE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From eea3af29cea5f28b8175cecc2092dc4a5a5332fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:33:10 +0000 Subject: [PATCH 05/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../.github/ISSUE_TEMPLATE/bug_report.md | 26 +++++++++++++++++++ .../.github/ISSUE_TEMPLATE/feature_request.md | 20 ++++++++++++++ .../.github/ISSUE_TEMPLATE/question.md | 20 ++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/bug_report.md create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/feature_request.md create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/question.md diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/bug_report.md b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..a99cfe419 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,26 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug(问题描述)** +A clear and concise description of what the bug is.Better with standalone code to reproduce the issue. + +**To Reproduce(复现步骤)** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Operating environment(运行环境):** + - python version [e.g. 3.6, 3.7] + - tensorflow version [e.g. 1.4.0, 1.15.0, 2.10.0] + - deepctr version [e.g. 0.9.2,] + +**Additional context** +Add any other context about the problem here. diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/feature_request.md b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..4a057ccce --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement&feature request +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/question.md b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 000000000..8b7f819d2 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,20 @@ +--- +name: Question +about: Ask any question ~ +title: '' +labels: question +assignees: '' + +--- +Please refer to the [FAQ](https://deepctr-doc.readthedocs.io/en/latest/FAQ.html) in doc and search for the [related issues](https://github.com/shenweichen/DeepCTR/issues) before you ask the question. + +**Describe the question(问题描述)** +A clear and concise description of what the question is. + +**Additional context** +Add any other context about the problem here. + +**Operating environment(运行环境):** + - python version [e.g. 3.6] + - tensorflow version [e.g. 1.4.0, 1.15.0, 2.10.0] + - deepctr version [e.g. 0.9.2,] -- Gitee From 7dbd59dfed72c238f6c09cf040c7ae93aa26c629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:33:25 +0000 Subject: [PATCH 06/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20workflows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DIN_ID0190_for_TensorFlow/.github/workflows/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From 243566c8d6ce82bf04a67f69e2606490ae6d191b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:33:43 +0000 Subject: [PATCH 07/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../.github/workflows/ci.yml | 94 ++++++++++++++++++ .../.github/workflows/ci2.yml | 96 +++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/ci.yml create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/ci2.yml diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/ci.yml b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/ci.yml new file mode 100644 index 000000000..3001b2e02 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/ci.yml @@ -0,0 +1,94 @@ +name: CI_TF2 + +on: + push: + path: + - 'deepctr/*' + - 'tests/*' + pull_request: + path: + - 'deepctr/*' + - 'tests/*' + +jobs: + build: + + runs-on: ubuntu-latest + timeout-minutes: 180 + strategy: + matrix: + python-version: [ 3.6,3.7,3.8, 3.9,3.10.7 ] + tf-version: [ 2.6.0,2.7.0,2.8.0,2.9.0,2.10.0 ] + + exclude: + - python-version: 3.7 + tf-version: 1.4.0 + - python-version: 3.7 + tf-version: 1.15.0 + - python-version: 3.8 + tf-version: 1.4.0 + - python-version: 3.8 + tf-version: 1.14.0 + - python-version: 3.8 + tf-version: 1.15.0 + - python-version: 3.6 + tf-version: 2.7.0 + - python-version: 3.6 + tf-version: 2.8.0 + - python-version: 3.6 + tf-version: 2.9.0 + - python-version: 3.6 + tf-version: 2.10.0 + - python-version: 3.9 + tf-version: 1.4.0 + - python-version: 3.9 + tf-version: 1.15.0 + - python-version: 3.9 + tf-version: 2.2.0 + - python-version: 3.9 + tf-version: 2.5.0 + - python-version: 3.9 + tf-version: 2.6.0 + - python-version: 3.9 + tf-version: 2.7.0 + - python-version: 3.10.7 + tf-version: 1.4.0 + - python-version: 3.10.7 + tf-version: 1.15.0 + - python-version: 3.10.7 + tf-version: 2.2.0 + - python-version: 3.10.7 + tf-version: 2.5.0 + - python-version: 3.10.7 + tf-version: 2.6.0 + - python-version: 3.10.7 + tf-version: 2.7.0 + steps: + + - uses: actions/checkout@v3 + + - name: Setup python environment + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip3 install -q tensorflow==${{ matrix.tf-version }} + pip install -q protobuf==3.19.0 + pip install -q requests + pip install -e . + - name: Test with pytest + timeout-minutes: 180 + run: | + pip install -q pytest + pip install -q pytest-cov + pip install -q python-coveralls + pytest --cov=deepctr --cov-report=xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3.1.0 + with: + token: ${{secrets.CODECOV_TOKEN}} + file: ./coverage.xml + flags: pytest + name: py${{ matrix.python-version }}-tf${{ matrix.tf-version }} diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/ci2.yml b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/ci2.yml new file mode 100644 index 000000000..40785e137 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/.github/workflows/ci2.yml @@ -0,0 +1,96 @@ +name: CI_TF1 + +on: + push: + path: + - 'deepctr/*' + - 'tests/*' + pull_request: + path: + - 'deepctr/*' + - 'tests/*' + +jobs: + build: + + runs-on: ubuntu-latest + timeout-minutes: 360 + strategy: + matrix: + python-version: [ 3.6,3.7 ] + tf-version: [ 1.15.0 ] + + exclude: + - python-version: 3.7 + tf-version: 1.4.0 + - python-version: 3.7 + tf-version: 1.12.0 + - python-version: 3.7 + tf-version: 1.15.0 + - python-version: 3.8 + tf-version: 1.4.0 + - python-version: 3.8 + tf-version: 1.14.0 + - python-version: 3.8 + tf-version: 1.15.0 + - python-version: 3.6 + tf-version: 2.7.0 + - python-version: 3.6 + tf-version: 2.8.0 + - python-version: 3.6 + tf-version: 2.9.0 + - python-version: 3.6 + tf-version: 2.10.0 + - python-version: 3.9 + tf-version: 1.4.0 + - python-version: 3.9 + tf-version: 1.15.0 + - python-version: 3.9 + tf-version: 2.2.0 + - python-version: 3.9 + tf-version: 2.5.0 + - python-version: 3.9 + tf-version: 2.6.0 + - python-version: 3.9 + tf-version: 2.7.0 + - python-version: 3.10.7 + tf-version: 1.4.0 + - python-version: 3.10.7 + tf-version: 1.15.0 + - python-version: 3.10.7 + tf-version: 2.2.0 + - python-version: 3.10.7 + tf-version: 2.5.0 + - python-version: 3.10.7 + tf-version: 2.6.0 + - python-version: 3.10.7 + tf-version: 2.7.0 + steps: + + - uses: actions/checkout@v3 + + - name: Setup python environment + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip3 install -q tensorflow==${{ matrix.tf-version }} + pip install -q protobuf==3.19.0 + pip install -q requests + pip install -e . + - name: Test with pytest + timeout-minutes: 360 + run: | + pip install -q pytest + pip install -q pytest-cov + pip install -q python-coveralls + pytest --cov=deepctr --cov-report=xml + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3.1.0 + with: + token: ${{secrets.CODECOV_TOKEN}} + file: ./coverage.xml + flags: pytest + name: py${{ matrix.python-version }}-tf${{ matrix.tf-version }} -- Gitee From 4f350a03a30ede63530a35ecd35e8360758ace25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:34:04 +0000 Subject: [PATCH 08/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20deepctr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../recommendation/DIN_ID0190_for_TensorFlow/deepctr/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From 8f3be22e730f3ecb4bba7889f222d806dd13789e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:34:20 +0000 Subject: [PATCH 09/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../deepctr/__init__.py | 5 + .../deepctr/feature_column.py | 216 ++++++++++++++++++ .../deepctr/inputs.py | 157 +++++++++++++ .../deepctr/utils.py | 48 ++++ 4 files changed, 426 insertions(+) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__init__.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/feature_column.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/inputs.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/utils.py diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__init__.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__init__.py new file mode 100644 index 000000000..8cb950715 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__init__.py @@ -0,0 +1,5 @@ +from .utils import check_version + +__version__ = '0.9.3' +check_version(__version__) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/feature_column.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/feature_column.py new file mode 100644 index 000000000..c3fe13702 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/feature_column.py @@ -0,0 +1,216 @@ +from npu_bridge.npu_init import * +import tensorflow as tf +from collections import namedtuple, OrderedDict +from copy import copy +from itertools import chain + +from tensorflow.python.keras.initializers import RandomNormal, Zeros +from tensorflow.python.keras.layers import Input, Lambda + +from .inputs import create_embedding_matrix, embedding_lookup, get_dense_input, varlen_embedding_lookup, \ + get_varlen_pooling_list, mergeDict +from .layers import Linear +from .layers.utils import concat_func + +DEFAULT_GROUP_NAME = "default_group" + + +class SparseFeat(namedtuple('SparseFeat', + ['name', 'vocabulary_size', 'embedding_dim', 'use_hash', 'vocabulary_path', 'dtype', 'embeddings_initializer', + 'embedding_name', + 'group_name', 'trainable'])): + __slots__ = () + + def __new__(cls, name, vocabulary_size, embedding_dim=4, use_hash=False, vocabulary_path=None, dtype="int32", embeddings_initializer=None, + embedding_name=None, + group_name=DEFAULT_GROUP_NAME, trainable=True): + + if embedding_dim == "auto": + embedding_dim = 6 * int(pow(vocabulary_size, 0.25)) + if embeddings_initializer is None: + embeddings_initializer = RandomNormal(mean=0.0, stddev=0.0001, seed=2020) + + if embedding_name is None: + embedding_name = name + + return super(SparseFeat, cls).__new__(cls, name, vocabulary_size, embedding_dim, use_hash, vocabulary_path, dtype, + embeddings_initializer, + embedding_name, group_name, trainable) + + def __hash__(self): + return self.name.__hash__() + + +class VarLenSparseFeat(namedtuple('VarLenSparseFeat', + ['sparsefeat', 'maxlen', 'combiner', 'length_name', 'weight_name', 'weight_norm'])): + __slots__ = () + + def __new__(cls, sparsefeat, maxlen, combiner="mean", length_name=None, weight_name=None, weight_norm=True): + return super(VarLenSparseFeat, cls).__new__(cls, sparsefeat, maxlen, combiner, length_name, weight_name, + weight_norm) + + @property + def name(self): + return self.sparsefeat.name + + @property + def vocabulary_size(self): + return self.sparsefeat.vocabulary_size + + @property + def embedding_dim(self): + return self.sparsefeat.embedding_dim + + @property + def use_hash(self): + return self.sparsefeat.use_hash + + @property + def vocabulary_path(self): + return self.sparsefeat.vocabulary_path + + @property + def dtype(self): + return self.sparsefeat.dtype + + @property + def embeddings_initializer(self): + return self.sparsefeat.embeddings_initializer + + @property + def embedding_name(self): + return self.sparsefeat.embedding_name + + @property + def group_name(self): + return self.sparsefeat.group_name + + @property + def trainable(self): + return self.sparsefeat.trainable + + def __hash__(self): + return self.name.__hash__() + + +class DenseFeat(namedtuple('DenseFeat', ['name', 'dimension', 'dtype', 'transform_fn'])): + """ Dense feature + Args: + name: feature name. + dimension: dimension of the feature, default = 1. + dtype: dtype of the feature, default="float32". + transform_fn: If not `None` , a function that can be used to transform + values of the feature. the function takes the input Tensor as its + argument, and returns the output Tensor. + (e.g. lambda x: (x - 3.0) / 4.2). + """ + __slots__ = () + + def __new__(cls, name, dimension=1, dtype="float32", transform_fn=None): + return super(DenseFeat, cls).__new__(cls, name, dimension, dtype, transform_fn) + + def __hash__(self): + return self.name.__hash__() + + # def __eq__(self, other): + # if self.name == other.name: + # return True + # return False + + # def __repr__(self): + # return 'DenseFeat:'+self.name + + +def get_feature_names(feature_columns): + features = build_input_features(feature_columns) + return list(features.keys()) + + +def build_input_features(feature_columns, prefix=''): + input_features = OrderedDict() + for fc in feature_columns: + if isinstance(fc, SparseFeat): + input_features[fc.name] = Input( + shape=(1,), name=prefix + fc.name, dtype=fc.dtype) + elif isinstance(fc, DenseFeat): + input_features[fc.name] = Input( + shape=(fc.dimension,), name=prefix + fc.name, dtype=fc.dtype) + elif isinstance(fc, VarLenSparseFeat): + input_features[fc.name] = Input(shape=(fc.maxlen,), name=prefix + fc.name, + dtype=fc.dtype) + if fc.weight_name is not None: + input_features[fc.weight_name] = Input(shape=(fc.maxlen, 1), name=prefix + fc.weight_name, + dtype="float32") + if fc.length_name is not None: + input_features[fc.length_name] = Input((1,), name=prefix + fc.length_name, dtype='int32') + + else: + raise TypeError("Invalid feature column type,got", type(fc)) + + return input_features + + +def get_linear_logit(features, feature_columns, units=1, use_bias=False, seed=1024, prefix='linear', + l2_reg=0, sparse_feat_refine_weight=None): + linear_feature_columns = copy(feature_columns) + for i in range(len(linear_feature_columns)): + if isinstance(linear_feature_columns[i], SparseFeat): + linear_feature_columns[i] = linear_feature_columns[i]._replace(embedding_dim=1, + embeddings_initializer=Zeros()) + if isinstance(linear_feature_columns[i], VarLenSparseFeat): + linear_feature_columns[i] = linear_feature_columns[i]._replace( + sparsefeat=linear_feature_columns[i].sparsefeat._replace(embedding_dim=1, + embeddings_initializer=Zeros())) + + linear_emb_list = [input_from_feature_columns(features, linear_feature_columns, l2_reg, seed, + prefix=prefix + str(i))[0] for i in range(units)] + _, dense_input_list = input_from_feature_columns(features, linear_feature_columns, l2_reg, seed, prefix=prefix) + + linear_logit_list = [] + for i in range(units): + + if len(linear_emb_list[i]) > 0 and len(dense_input_list) > 0: + sparse_input = concat_func(linear_emb_list[i]) + dense_input = concat_func(dense_input_list) + if sparse_feat_refine_weight is not None: + sparse_input = Lambda(lambda x: x[0] * tf.expand_dims(x[1], axis=1))( + [sparse_input, sparse_feat_refine_weight]) + linear_logit = Linear(l2_reg, mode=2, use_bias=use_bias, seed=seed)([sparse_input, dense_input]) + elif len(linear_emb_list[i]) > 0: + sparse_input = concat_func(linear_emb_list[i]) + if sparse_feat_refine_weight is not None: + sparse_input = Lambda(lambda x: x[0] * tf.expand_dims(x[1], axis=1))( + [sparse_input, sparse_feat_refine_weight]) + linear_logit = Linear(l2_reg, mode=0, use_bias=use_bias, seed=seed)(sparse_input) + elif len(dense_input_list) > 0: + dense_input = concat_func(dense_input_list) + linear_logit = Linear(l2_reg, mode=1, use_bias=use_bias, seed=seed)(dense_input) + else: #empty feature_columns + return Lambda(lambda x: tf.constant([[0.0]]))(list(features.values())[0]) + linear_logit_list.append(linear_logit) + + return concat_func(linear_logit_list) + + +def input_from_feature_columns(features, feature_columns, l2_reg, seed, prefix='', seq_mask_zero=True, + support_dense=True, support_group=False): + sparse_feature_columns = list( + filter(lambda x: isinstance(x, SparseFeat), feature_columns)) if feature_columns else [] + varlen_sparse_feature_columns = list( + filter(lambda x: isinstance(x, VarLenSparseFeat), feature_columns)) if feature_columns else [] + + embedding_matrix_dict = create_embedding_matrix(feature_columns, l2_reg, seed, prefix=prefix, + seq_mask_zero=seq_mask_zero) + group_sparse_embedding_dict = embedding_lookup(embedding_matrix_dict, features, sparse_feature_columns) + dense_value_list = get_dense_input(features, feature_columns) + if not support_dense and len(dense_value_list) > 0: + raise ValueError("DenseFeat is not supported in dnn_feature_columns") + + sequence_embed_dict = varlen_embedding_lookup(embedding_matrix_dict, features, varlen_sparse_feature_columns) + group_varlen_sparse_embedding_dict = get_varlen_pooling_list(sequence_embed_dict, features, + varlen_sparse_feature_columns) + group_embedding_dict = mergeDict(group_sparse_embedding_dict, group_varlen_sparse_embedding_dict) + if not support_group: + group_embedding_dict = list(chain.from_iterable(group_embedding_dict.values())) + return group_embedding_dict, dense_value_list + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/inputs.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/inputs.py new file mode 100644 index 000000000..f529070e1 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/inputs.py @@ -0,0 +1,157 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen,weichenswc@163.com + +""" +from npu_bridge.npu_init import * + +from collections import defaultdict +from itertools import chain + +from tensorflow.python.keras.layers import Embedding, Lambda +from tensorflow.python.keras.regularizers import l2 + +from .layers.sequence import SequencePoolingLayer, WeightedSequenceLayer +from .layers.utils import Hash + + +def get_inputs_list(inputs): + return list(chain(*list(map(lambda x: x.values(), filter(lambda x: x is not None, inputs))))) + + +def create_embedding_dict(sparse_feature_columns, varlen_sparse_feature_columns, seed, l2_reg, + prefix='sparse_', seq_mask_zero=True): + sparse_embedding = {} + for feat in sparse_feature_columns: + emb = Embedding(feat.vocabulary_size, feat.embedding_dim, + embeddings_initializer=feat.embeddings_initializer, + embeddings_regularizer=l2(l2_reg), + name=prefix + '_emb_' + feat.embedding_name) + emb.trainable = feat.trainable + sparse_embedding[feat.embedding_name] = emb + + if varlen_sparse_feature_columns and len(varlen_sparse_feature_columns) > 0: + for feat in varlen_sparse_feature_columns: + # if feat.name not in sparse_embedding: + emb = Embedding(feat.vocabulary_size, feat.embedding_dim, + embeddings_initializer=feat.embeddings_initializer, + embeddings_regularizer=l2( + l2_reg), + name=prefix + '_seq_emb_' + feat.name, + mask_zero=seq_mask_zero) + emb.trainable = feat.trainable + sparse_embedding[feat.embedding_name] = emb + return sparse_embedding + + +def get_embedding_vec_list(embedding_dict, input_dict, sparse_feature_columns, return_feat_list=(), mask_feat_list=()): + embedding_vec_list = [] + for fg in sparse_feature_columns: + feat_name = fg.name + if len(return_feat_list) == 0 or feat_name in return_feat_list: + if fg.use_hash: + lookup_idx = Hash(fg.vocabulary_size, mask_zero=(feat_name in mask_feat_list), vocabulary_path=fg.vocabulary_path)(input_dict[feat_name]) + else: + lookup_idx = input_dict[feat_name] + + embedding_vec_list.append(embedding_dict[feat_name](lookup_idx)) + + return embedding_vec_list + + +def create_embedding_matrix(feature_columns, l2_reg, seed, prefix="", seq_mask_zero=True): + from . import feature_column as fc_lib + + sparse_feature_columns = list( + filter(lambda x: isinstance(x, fc_lib.SparseFeat), feature_columns)) if feature_columns else [] + varlen_sparse_feature_columns = list( + filter(lambda x: isinstance(x, fc_lib.VarLenSparseFeat), feature_columns)) if feature_columns else [] + sparse_emb_dict = create_embedding_dict(sparse_feature_columns, varlen_sparse_feature_columns, seed, + l2_reg, prefix=prefix + 'sparse', seq_mask_zero=seq_mask_zero) + return sparse_emb_dict + + +def embedding_lookup(sparse_embedding_dict, sparse_input_dict, sparse_feature_columns, return_feat_list=(), + mask_feat_list=(), to_list=False): + group_embedding_dict = defaultdict(list) + for fc in sparse_feature_columns: + feature_name = fc.name + embedding_name = fc.embedding_name + if (len(return_feat_list) == 0 or feature_name in return_feat_list): + if fc.use_hash: + lookup_idx = Hash(fc.vocabulary_size, mask_zero=(feature_name in mask_feat_list), vocabulary_path=fc.vocabulary_path)( + sparse_input_dict[feature_name]) + else: + lookup_idx = sparse_input_dict[feature_name] + + group_embedding_dict[fc.group_name].append(sparse_embedding_dict[embedding_name](lookup_idx)) + if to_list: + return list(chain.from_iterable(group_embedding_dict.values())) + return group_embedding_dict + + +def varlen_embedding_lookup(embedding_dict, sequence_input_dict, varlen_sparse_feature_columns): + varlen_embedding_vec_dict = {} + for fc in varlen_sparse_feature_columns: + feature_name = fc.name + embedding_name = fc.embedding_name + if fc.use_hash: + lookup_idx = Hash(fc.vocabulary_size, mask_zero=True, vocabulary_path=fc.vocabulary_path)(sequence_input_dict[feature_name]) + else: + lookup_idx = sequence_input_dict[feature_name] + varlen_embedding_vec_dict[feature_name] = embedding_dict[embedding_name](lookup_idx) + return varlen_embedding_vec_dict + + +def get_varlen_pooling_list(embedding_dict, features, varlen_sparse_feature_columns, to_list=False): + pooling_vec_list = defaultdict(list) + for fc in varlen_sparse_feature_columns: + feature_name = fc.name + combiner = fc.combiner + feature_length_name = fc.length_name + if feature_length_name is not None: + if fc.weight_name is not None: + seq_input = WeightedSequenceLayer(weight_normalization=fc.weight_norm)( + [embedding_dict[feature_name], features[feature_length_name], features[fc.weight_name]]) + else: + seq_input = embedding_dict[feature_name] + vec = SequencePoolingLayer(combiner, supports_masking=False)( + [seq_input, features[feature_length_name]]) + else: + if fc.weight_name is not None: + seq_input = WeightedSequenceLayer(weight_normalization=fc.weight_norm, supports_masking=True)( + [embedding_dict[feature_name], features[fc.weight_name]]) + else: + seq_input = embedding_dict[feature_name] + vec = SequencePoolingLayer(combiner, supports_masking=True)( + seq_input) + pooling_vec_list[fc.group_name].append(vec) + if to_list: + return chain.from_iterable(pooling_vec_list.values()) + return pooling_vec_list + + +def get_dense_input(features, feature_columns): + from . import feature_column as fc_lib + dense_feature_columns = list( + filter(lambda x: isinstance(x, fc_lib.DenseFeat), feature_columns)) if feature_columns else [] + dense_input_list = [] + for fc in dense_feature_columns: + if fc.transform_fn is None: + dense_input_list.append(features[fc.name]) + else: + transform_result = Lambda(fc.transform_fn)(features[fc.name]) + dense_input_list.append(transform_result) + return dense_input_list + + +def mergeDict(a, b): + c = defaultdict(list) + for k, v in a.items(): + c[k].extend(v) + for k, v in b.items(): + c[k].extend(v) + return c + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/utils.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/utils.py new file mode 100644 index 000000000..7d462f622 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/utils.py @@ -0,0 +1,48 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen,weichenswc@163.com + +""" +from npu_bridge.npu_init import * + +import json +import logging +from threading import Thread + +import requests + +try: + from packaging.version import parse +except ImportError: + from pip._vendor.packaging.version import parse + + +def check_version(version): + """Return version of package on pypi.python.org using json.""" + + def check(version): + try: + url_pattern = 'https://pypi.python.org/pypi/deepctr/json' + req = requests.get(url_pattern) + latest_version = parse('0') + version = parse(version) + if req.status_code == requests.codes.ok: + j = json.loads(req.text.encode('utf-8')) + releases = j.get('releases', []) + for release in releases: + ver = parse(release) + if ver.is_prerelease or ver.is_postrelease: + continue + latest_version = max(latest_version, ver) + if latest_version > version: + logging.warning( + '\nDeepCTR version {0} detected. Your version is {1}.\nUse `pip install -U deepctr` to upgrade.Changelog: https://github.com/shenweichen/DeepCTR/releases/tag/v{0}'.format( + latest_version, version)) + except: + print("Please check the latest version manually on https://pypi.org/project/deepctr/#history") + return + + Thread(target=check, args=(version,)).start() + -- Gitee From 617a456fa626dd5035df0a5f155459384db89be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:34:35 +0000 Subject: [PATCH 10/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20=5F=5Fpycache=5F=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DIN_ID0190_for_TensorFlow/deepctr/__pycache__/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From 1baf253723a358da14dcc87a1863f92d6edc122c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:34:50 +0000 Subject: [PATCH 11/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../deepctr/__pycache__/__init__.cpython-37.pyc | Bin 0 -> 239 bytes .../__pycache__/feature_column.cpython-37.pyc | Bin 0 -> 8068 bytes .../deepctr/__pycache__/inputs.cpython-37.pyc | Bin 0 -> 4884 bytes .../deepctr/__pycache__/utils.cpython-37.pyc | Bin 0 -> 1523 bytes 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/__init__.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/feature_column.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/inputs.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/utils.cpython-37.pyc diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/__init__.cpython-37.pyc b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bdaee857649e49943d66666169d6f733468f829d GIT binary patch literal 239 zcmZ?b<>g`k0-g5Yln@~O7{q}ACLqHBh>K-_L<&O+V-7!@w#4k_%jQreG{o?WpGYd--BSZc4f>M2#)YJm!kRaXM#Nv|FqWHXm(s&~S zBVz+o0~13FBSQlt{S=^z7)TcJAu#nd#~IAw^N7MAK56tKHRXB#NSB$E#>%do9a0qKr2d*A6=NwwqHu z!x4L?N7X$NMKpnADL~9YfOAYvVt`<=z#ei4f*g`lfFQ@5`>+U-93mvQN8u#UGue$ZtI<#onut5`}t1Z&PzJy7dnP*NILHqJEm=RN_I&;7yNRkVplpj!K_UX>NJ>NNFpONjRztCB<7bRWt&vwq)=RlY3r4JQe;nk-K zuX;MK@)fb$*4fg0lDxzQ(VCdV{O? zmD-yxY5i&}VLzzX)Wqz%9gj!-p6?~4PXzab$M3Wo5g7DF(0hV=uCe8|yWlN<>UMe1 zsRyFt`bp0AL=d7n_i?w^kCOa7x3kIJm)H!YmQNNN!gC|f@j9Cx=k0FG>9~<-?V8bP<=MozQeZP*p=HQl}+IV}<<)c#lNu1&;-|lq47Sd!7*UL>C;V>pgZH zEP>YW!$c>&N@gAf4R^EeyW)uxws*axGTMCJ?j%MZd%op{TN9i;H`+>aJbKdel7(T1 z5X#Yx+OCgrL{c5rJ5rJ@K?{~VZy(hxLsAqA7b0`$uU~|l1Rn=g+dX}kdcW$H+tJxwo$VTW#cNN$BxI#j@ zSb3nlV228}jKMs`3hENWk!ys-g;9o{-;{5X7FWO^R#9BS71CG=+dVxYlGYb}4H}2m z%5f6YaXJC-`$U%==V9OVvlc8@=m!y&E1A3V!QJ@1zE1_c zkEl`c11f$C#e~U{Y0)qI3ltM3`?@sI9M8iCT*|$1r--iS^P!dgC0GD`+3JXQ8Al|?&F++(& zq_9c8 z`dY8la?*6E?74P2>qtGj282_)T}u&Hm8Mp`#T z8a2H=Ce1n~?FgowNHA&o6v5PVYfSL@V}eJJ2kbnNShB4tVyWrfF|p^4i5+0{i6oLC zPmxGXcg7@s;|kMRa3aA39#aHU(??^17mmHEJabMYj$mqvIBL2#Chq&k#0@ue;%qyS zd_u@6@~P=3WAZOdmoF{3E>0kmg~ix?0uCulT3Oh(u%pLGeQ>$SD@TLj~be1K02; zPXRoIv=$SmuxjhvlqWHKR_QDwk~-on7w?eMPrkDI0+4OVM=Nu&ObfsFL@T^0DS9HD zzd2}^PpiW=Kq3Zf2i=?2khFrP6>WKg9+#~w0g%iIr z>IaSLhE1H3P7^{uZd%_)wP;+)w;2|UQ0Ps)uP}fDP+r0Z{#vvpgZ|4 z8m^H1;aB-Bhq(!#Iqp+gE?i9+p2>VoJssb?7sx3qtN(xa9)jiy-WBD~M`UK?|Ii?XcYqBe&b|(wK`Pi!l)+C6eKTgnS9~1o=sE z1H*kFL?9BKMy~1NF4gCI!fPUvQEM zenv@=12s0zDG?H6;@y;n=CCNgj|Aa!@W#e1WynCY94 z5x+}~bS@(sC%m5T0+|Wl#vE^&jyO#PX||yJnYe=@$)_%2l6QMOugft!sbPxAxjvGk zq1dDfsd7@YKgXkeE=>eU^@PkOK-EbTKH@WKpULzN(P)}XXO2C~N9lN3X`dR`&Xlj5 z4J_@(E{%1u%j4W~H!`CUG$lcy+L(Dzt}vNBs!e}}+VICH6eLsA6QYW9!eC{#pw7us zXLG6r%E0MC|0VF1GkQVtE~v{$#tI*!Y#cjCr1SCDXo{IwAb*uJu0Ewh1yI^m0oevx z)?^Dl1x&OQ0e-`5JJ+A*_Dr}6CUSJfzP4AQoIH)QSB{ZIk8%WQJh!cjD9ZEvKKnU~ z3X;1B?n-0;sFj%LYE+C%aTPsh;zIZ<$TnY;;+YraL*)=ChT569+$86rP=B>1lh%;e z7ePGOWFC2_cHf}4|A>B4Z+9RNQIM1=sm=zqj1|n z+CPx^wR>&oc3bUGhF$m)(F;U`*s9&Nc(;2Li))q4gqm$18GcbFF%>Gv?uAE^%+DwS zlm3pNZ;OKb4RMMJ@>G*blTL;Ef~jVYY>#3RyOK>T{k%1Llo=3vVazf| z9~DCT^+)OVuPM!EhIymxn;KBN5!tV$X_c(gh(|VN1}zaJHR#h=*uDqPF#XJyjNd7HTpj!56v~fY60DQ+)A)V&i6=rK-E6YM zBri=VJgRV$`UOuS?MGM-?353y{iu!k$%EhqO`lGU^q7!t`$zOh2qwWzF5=iG3_T7U x-07#D+q94}Ph(&sX0a7z72daMVKd6At`WAXth`ud^ViQW8#(i=!pxe&{ujz0F?|34 literal 0 HcmV?d00001 diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/inputs.cpython-37.pyc b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/__pycache__/inputs.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c108bbfb7be6595348b1be3e44968a6a05517ab9 GIT binary patch literal 4884 zcmb7I&2!tv702R>AVf*DtPd-3T-kA)Fij*|c0Eq2x=yU9ZR$}wk*4lo+@cT{q$mL- zy#Q_61NI`%B%O|P=%Hshht9Oe{tZ3$FW74@%}j5-_0->6fTTWbcS;Qw`}ub9-uwOD zTYWk?>1z1h`f`hZ{)VRggBru1fyx~`$zM@$&C@vJdc?NrZA5x(cm|`o8JV%=S*mVD zc3kpGvEwC!T3um0#jheEJ#l&hWSS z89syB48P20`5bCz`8#}ppF?eyFY-4qe~w?_Z}Ri#ndfiu3+P$k-{DJmpW_#|wA#B* zS6ug#Ub@>6AG=HV-49#!-LSp1h2n$9c_VpTzjN*8_2qgecHM9Bp<0be_CAUU9yWqr zl=4C*42OQiB`x&a?v^f9z}R zfZEres#QqzP_Vi*iO#OU@AGMNnc`fSHgDAO;NC+FIJS)E)Rw8q3FWU6{z zOjEOpf;JRqP}EFm(mujOLmF|=m3E^Q;S=cN^MjlaAg529aTZGx(xJx0Jc%hK-9RLv zzg@Sf7r%1xC^P*Djg~gB`|FvuuKj9lP)fC|w8!x1c#PD{j02XLoGnnlwV-kRxixT5 zw=>7bliAe2rag4;YZ(*Yq@|1<0A%~7_@=*+F-X1rN~xxj>Y%i^_1u1`+}}X14Ncmc zp5=$}j$bq6Jl?qJL7iGzSkyE92qxblIZylqqU7XgXP>v?TtJCtPK^g>Gg(drFc<7Zp){hf ztv0BY<$Pg}Mi`_$5&HE`)Qj7ReCtslFwK9(u$hD*mv*%3*my`4it|*wM+0`y?S^d*gB`IeX-K*Xh)P@HB2BG|5SDEF zq-w>6O(KFoEqev9`{T56skepHtP1DtkWD3I@w`S)~1<;I|#2M`7IRRjxy`V<8W@ynC~M+tod_0pbG zK;2PLcNNr8AAlGzT3*x+T>u{M%q;*vg}`8mKOo1LIILv$;p7YL1x~V+hl}Ghpio_# z5@gq0!9@7t0u@VC)TlT?0&Y`ufJqfQ-luUDm?s;e6ZS$ep{RH=Eh1t!l&Mu zqWc7Ch!08JG74ErJ2@lF=DUrNuINT@)MaI8e{~Xd=Bu<`IoQa{WOckWXOnxFl+br- zx}zu}Cy7~B0Zp6-ZOkz72}bLr75WigN49-{22ul~|An&a=L0h{LEaWhSR17-)WtQ2>sc%}LlPOUCD8LN&Ftqr0*4~@<;@hQgYwXPXSXpxYO*k+DPdpkQFu4U4=q9F_cf~9NqrLJ=jrD@*%!Y#R#rqjhZ<~ zmH$YmKvSaFQv3kpUbUOwcl=>|KH@N+(S)-@2p&0&SElq!AzK<)coL5b6u1kg8p@~q zK7zpH0~8wgl_EF&QO#9u<+3UgtqPkjKsyVdBGvjS=GRqbj|zd5GA;sc@)X6fh|W$$ zbVSsHJfc%nH;|W8K6YC{+JxIVVmmVq5$yqFfK{3L1~Peo8?Y^U;SiRZW#!}K3NXAi zjPojb{U5ua1_7M z&OJm00a9ko7C)gKK0<-ZxspCK`7lMV9@$h;wsI1$Iuo*`t$gUIrAK;W}8BvVkIf za1dMaKXD(un?U;l-h}#5)Iy!iv|TgCec+0X+M+D^KJV0hU!JAQ`;KVwX1GjMy6njb zSR@MT#4U+Bw=7dQPJ!VhkI@%!@}wggQRgu(_xLYLdwC!6I$4hB9~z0e+X=F}cllK_ zk6a%UK{q^aI$yjzj6XTuC>E*|L%p-C*(>;==R-SZY20JEe+uHk^+FNX z@w2_h4}-WEdfuM^61A|x_m4r^FM6Cs6lPW#alvA&m!T2pCiEld;v*0MguFvjEYSo{ z&?SWS6Pe+;hG+s1YkY9RWW{s;HRB>Ja=*Cr%P1a13G+cKM`aq6BLJEQ1yB4+qor%-;#0khvo?V^D#>AL7Z^_iSIypx$ zM*|b?+?qA1HG5C%b27(uif%7bBeyBs1Mvf3-CNG_xYd)g6o+B>pLlCMq>PoZOF@oM=ws+EWT+^eaa+@C8L4=vZ(laDHZ;ky=mZ`3Fcpv zY3ZlAkWrTTyJv8u_H^OP!mrANN0bFey(mvuRwRf13T2YYUiC^BMkt_Qmh}=YL4+$@ zp^TF7H9#4keODv(p~m8YFMG_-BFThYgBnJ81sFy;=PNHY7m)O@|=*9(Q%FF`3>>AVEpTwuvZLOD6VfFAAIp>e=khRYQ^sEFcNTWH!rL1 zetZ96`}6jry@UO|_WqimN~W2BR|COozCVO3#b+Q8CfLLtcHwK{+qhBB8(?jM#e}bk zZJ3Sk*5_7OgC|)LN11pU)aDjZwv0H?txOoiaf@&RE>{-t&LwCcfX4OXucKLrdwR## zzwW&QCZRKi4EGNI7{*pZIo&QT;%-;%=m1~wlqM|D(=<;dZ@?zL143=c`VRzYsGW01 z8*eTC9AFHVJTEKa+hJL7`8DSS@7#l$40gdAyxOF~w4c>h+ENGRnr`nZx TAB2frqPc7kmk_+IIW_(U!auVx literal 0 HcmV?d00001 -- Gitee From abd9cb392fe0c581c87516effce036ecba981b6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:35:15 +0000 Subject: [PATCH 12/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20contrib?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DIN_ID0190_for_TensorFlow/deepctr/contrib/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From b1b801c37dde4085f9425906347513e938e2f696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:35:29 +0000 Subject: [PATCH 13/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../deepctr/contrib/__init__.py | 0 .../deepctr/contrib/rnn.py | 1154 +++++++++++++ .../deepctr/contrib/rnn_v2.py | 1454 +++++++++++++++++ .../deepctr/contrib/utils.py | 380 +++++ 4 files changed, 2988 insertions(+) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__init__.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/rnn.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/rnn_v2.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/utils.py diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__init__.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/rnn.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/rnn.py new file mode 100644 index 000000000..2dcdcc504 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/rnn.py @@ -0,0 +1,1154 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +# + +# Licensed under the Apache License, Version 2.0 (the "License"); + +# you may not use this file except in compliance with the License. + +# You may obtain a copy of the License at + +# + +# http://www.apache.org/licenses/LICENSE-2.0 + +# + +# Unless required by applicable law or agreed to in writing, software + +# distributed under the License is distributed on an "AS IS" BASIS, + +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +# See the License for the specific language governing permissions and + +# limitations under the License. + +# ============================================================================== + + +"""RNN helpers for TensorFlow models. +@@bidirectional_dynamic_rnn +@@dynamic_rnn +@@raw_rnn +@@static_rnn +@@static_state_saving_rnn +@@static_bidirectional_rnn +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from npu_bridge.npu_init import * + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import rnn_cell_impl +from tensorflow.python.ops import tensor_array_ops +from tensorflow.python.ops import variable_scope as vs +from tensorflow.python.util import nest +import tensorflow as tf + + +def _like_rnncell_(cell): + """Checks that a given object is an RNNCell by using duck typing.""" + + conditions = [hasattr(cell, "output_size"), hasattr(cell, "state_size"), + + hasattr(cell, "zero_state"), callable(cell)] + + return all(conditions) + + +# pylint: disable=protected-access + +_concat = rnn_cell_impl._concat +try: + _like_rnncell = rnn_cell_impl._like_rnncell +except Exception as e: + _like_rnncell = _like_rnncell_ + + +# pylint: enable=protected-access + + +def _transpose_batch_time(x): + """Transpose the batch and time dimensions of a Tensor. + Retains as much of the static shape information as possible. + Args: + x: A tensor of rank 2 or higher. + Returns: + x transposed along the first two dimensions. + Raises: + ValueError: if `x` is rank 1 or lower. + """ + + x_static_shape = x.get_shape() + + if x_static_shape.ndims is not None and x_static_shape.ndims < 2: + raise ValueError( + + "Expected input tensor %s to have rank at least 2, but saw shape: %s" % + + (x, x_static_shape)) + + x_rank = array_ops.rank(x) + + x_t = array_ops.transpose( + + x, array_ops.concat( + + ([1, 0], math_ops.range(2, x_rank)), axis=0)) + + x_t.set_shape( + + tensor_shape.TensorShape([ + + x_static_shape[1].value, x_static_shape[0].value + + ]).concatenate(x_static_shape[2:])) + + return x_t + + +def _best_effort_input_batch_size(flat_input): + """Get static input batch size if available, with fallback to the dynamic one. + Args: + flat_input: An iterable of time major input Tensors of shape [max_time, + batch_size, ...]. All inputs should have compatible batch sizes. + Returns: + The batch size in Python integer if available, or a scalar Tensor otherwise. + Raises: + ValueError: if there is any input with an invalid shape. + """ + + for input_ in flat_input: + + shape = input_.shape + + if shape.ndims is None: + continue + + if shape.ndims < 2: + raise ValueError( + + "Expected input tensor %s to have rank at least 2" % input_) + + batch_size = shape[1].value + + if batch_size is not None: + return batch_size + + # Fallback to the dynamic batch size of the first input. + + return array_ops.shape(flat_input[0])[1] + + +def _infer_state_dtype(explicit_dtype, state): + """Infer the dtype of an RNN state. + Args: + explicit_dtype: explicitly declared dtype or None. + state: RNN's hidden state. Must be a Tensor or a nested iterable containing + Tensors. + Returns: + dtype: inferred dtype of hidden state. + Raises: + ValueError: if `state` has heterogeneous dtypes or is empty. + """ + + if explicit_dtype is not None: + + return explicit_dtype + + elif nest.is_sequence(state): + + inferred_dtypes = [element.dtype for element in nest.flatten(state)] + + if not inferred_dtypes: + raise ValueError("Unable to infer dtype from empty state.") + + all_same = all([x == inferred_dtypes[0] for x in inferred_dtypes]) + + if not all_same: + raise ValueError( + + "State has tensors of different inferred_dtypes. Unable to infer a " + + "single representative dtype.") + + return inferred_dtypes[0] + + else: + + return state.dtype + + +# pylint: disable=unused-argument + +def _rnn_step( + + time, sequence_length, min_sequence_length, max_sequence_length, + + zero_output, state, call_cell, state_size, skip_conditionals=False): + """Calculate one step of a dynamic RNN minibatch. + Returns an (output, state) pair conditioned on the sequence_lengths. + When skip_conditionals=False, the pseudocode is something like: + if t >= max_sequence_length: + return (zero_output, state) + if t < min_sequence_length: + return call_cell() + # Selectively output zeros or output, old state or new state depending + # on if we've finished calculating each row. + new_output, new_state = call_cell() + final_output = np.vstack([ + zero_output if time >= sequence_lengths[r] else new_output_r + for r, new_output_r in enumerate(new_output) + ]) + final_state = np.vstack([ + state[r] if time >= sequence_lengths[r] else new_state_r + for r, new_state_r in enumerate(new_state) + ]) + return (final_output, final_state) + Args: + time: Python int, the current time step + sequence_length: int32 `Tensor` vector of size [batch_size] + min_sequence_length: int32 `Tensor` scalar, min of sequence_length + max_sequence_length: int32 `Tensor` scalar, max of sequence_length + zero_output: `Tensor` vector of shape [output_size] + state: Either a single `Tensor` matrix of shape `[batch_size, state_size]`, + or a list/tuple of such tensors. + call_cell: lambda returning tuple of (new_output, new_state) where + new_output is a `Tensor` matrix of shape `[batch_size, output_size]`. + new_state is a `Tensor` matrix of shape `[batch_size, state_size]`. + state_size: The `cell.state_size` associated with the state. + skip_conditionals: Python bool, whether to skip using the conditional + calculations. This is useful for `dynamic_rnn`, where the input tensor + matches `max_sequence_length`, and using conditionals just slows + everything down. + Returns: + A tuple of (`final_output`, `final_state`) as given by the pseudocode above: + final_output is a `Tensor` matrix of shape [batch_size, output_size] + final_state is either a single `Tensor` matrix, or a tuple of such + matrices (matching length and shapes of input `state`). + Raises: + ValueError: If the cell returns a state tuple whose length does not match + that returned by `state_size`. + """ + + # Convert state to a list for ease of use + + flat_state = nest.flatten(state) + + flat_zero_output = nest.flatten(zero_output) + + def _copy_one_through(output, new_output): + + # If the state contains a scalar value we simply pass it through. + + if output.shape.ndims == 0: + return new_output + + copy_cond = (time >= sequence_length) + + with ops.colocate_with(new_output): + return array_ops.where(copy_cond, output, new_output) + + def _copy_some_through(flat_new_output, flat_new_state): + + # Use broadcasting select to determine which values should get + + # the previous state & zero output, and which values should get + + # a calculated state & output. + + flat_new_output = [ + + _copy_one_through(zero_output, new_output) + + for zero_output, new_output in zip(flat_zero_output, flat_new_output)] + + flat_new_state = [ + + _copy_one_through(state, new_state) + + for state, new_state in zip(flat_state, flat_new_state)] + + return flat_new_output + flat_new_state + + def _maybe_copy_some_through(): + + """Run RNN step. Pass through either no or some past state.""" + + new_output, new_state = call_cell() + + nest.assert_same_structure(state, new_state) + + flat_new_state = nest.flatten(new_state) + + flat_new_output = nest.flatten(new_output) + + return control_flow_ops.cond( + + # if t < min_seq_len: calculate and return everything + + time < min_sequence_length, lambda: flat_new_output + flat_new_state, + + # else copy some of it through + + lambda: _copy_some_through(flat_new_output, flat_new_state)) + + # TODO(ebrevdo): skipping these conditionals may cause a slowdown, + + # but benefits from removing cond() and its gradient. We should + + # profile with and without this switch here. + + if skip_conditionals: + + # Instead of using conditionals, perform the selective copy at all time + + # steps. This is faster when max_seq_len is equal to the number of unrolls + + # (which is typical for dynamic_rnn). + + new_output, new_state = call_cell() + + nest.assert_same_structure(state, new_state) + + new_state = nest.flatten(new_state) + + new_output = nest.flatten(new_output) + + final_output_and_state = _copy_some_through(new_output, new_state) + + else: + + empty_update = lambda: flat_zero_output + flat_state + + final_output_and_state = control_flow_ops.cond( + + # if t >= max_seq_len: copy all state through, output zeros + + time >= max_sequence_length, empty_update, + + # otherwise calculation is required: copy some or all of it through + + _maybe_copy_some_through) + + if len(final_output_and_state) != len(flat_zero_output) + len(flat_state): + raise ValueError("Internal error: state and output were not concatenated " + + "correctly.") + + final_output = final_output_and_state[:len(flat_zero_output)] + + final_state = final_output_and_state[len(flat_zero_output):] + + for output, flat_output in zip(final_output, flat_zero_output): + output.set_shape(flat_output.get_shape()) + + for substate, flat_substate in zip(final_state, flat_state): + substate.set_shape(flat_substate.get_shape()) + + final_output = nest.pack_sequence_as( + + structure=zero_output, flat_sequence=final_output) + + final_state = nest.pack_sequence_as( + + structure=state, flat_sequence=final_state) + + return final_output, final_state + + +def _reverse_seq(input_seq, lengths): + """Reverse a list of Tensors up to specified lengths. + Args: + input_seq: Sequence of seq_len tensors of dimension (batch_size, n_features) + or nested tuples of tensors. + lengths: A `Tensor` of dimension batch_size, containing lengths for each + sequence in the batch. If "None" is specified, simply reverses + the list. + Returns: + time-reversed sequence + """ + + if lengths is None: + return list(reversed(input_seq)) + + flat_input_seq = tuple(nest.flatten(input_) for input_ in input_seq) + + flat_results = [[] for _ in range(len(input_seq))] + + for sequence in zip(*flat_input_seq): + + input_shape = tensor_shape.unknown_shape( + + ndims=sequence[0].get_shape().ndims) + + for input_ in sequence: + input_shape.merge_with(input_.get_shape()) + + input_.set_shape(input_shape) + + # Join into (time, batch_size, depth) + + s_joined = array_ops.stack(sequence) + + # Reverse along dimension 0 + + s_reversed = array_ops.reverse_sequence(s_joined, lengths, 0, 1) + + # Split again into list + + result = array_ops.unstack(s_reversed) + + for r, flat_result in zip(result, flat_results): + r.set_shape(input_shape) + + flat_result.append(r) + + results = [nest.pack_sequence_as(structure=input_, flat_sequence=flat_result) + + for input_, flat_result in zip(input_seq, flat_results)] + + return results + + +# +# def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, +# +# initial_state_fw=None, initial_state_bw=None, +# +# dtype=None, parallel_iterations=None, +# +# swap_memory=False, time_major=False, scope=None): +# +# """Creates a dynamic version of bidirectional recurrent neural network. +# +# +# +# Takes input and builds independent forward and backward RNNs. The input_size +# +# of forward and backward cell must match. The initial state for both directions +# +# is zero by default (but can be set optionally) and no intermediate states are +# +# ever returned -- the network is fully unrolled for the given (passed in) +# +# length(s) of the sequence(s) or completely unrolled if length(s) is not +# +# given. +# +# +# +# Args: +# +# cell_fw: An instance of RNNCell, to be used for forward direction. +# +# cell_bw: An instance of RNNCell, to be used for backward direction. +# +# inputs: The RNN inputs. +# +# If time_major == False (default), this must be a tensor of shape: +# +# `[batch_size, max_time, ...]`, or a nested tuple of such elements. +# +# If time_major == True, this must be a tensor of shape: +# +# `[max_time, batch_size, ...]`, or a nested tuple of such elements. +# +# sequence_length: (optional) An int32/int64 vector, size `[batch_size]`, +# +# containing the actual lengths for each of the sequences in the batch. +# +# If not provided, all batch entries are assumed to be full sequences; and +# +# time reversal is applied from time `0` to `max_time` for each sequence. +# +# initial_state_fw: (optional) An initial state for the forward RNN. +# +# This must be a tensor of appropriate type and shape +# +# `[batch_size, cell_fw.state_size]`. +# +# If `cell_fw.state_size` is a tuple, this should be a tuple of +# +# tensors having shapes `[batch_size, s] for s in cell_fw.state_size`. +# +# initial_state_bw: (optional) Same as for `initial_state_fw`, but using +# +# the corresponding properties of `cell_bw`. +# +# dtype: (optional) The data type for the initial states and expected output. +# +# Required if initial_states are not provided or RNN states have a +# +# heterogeneous dtype. +# +# parallel_iterations: (Default: 32). The number of iterations to run in +# +# parallel. Those operations which do not have any temporal dependency +# +# and can be run in parallel, will be. This parameter trades off +# +# time for space. Values >> 1 use more memory but take less time, +# +# while smaller values use less memory but computations take longer. +# +# swap_memory: Transparently swap the tensors produced in forward inference +# +# but needed for back prop from GPU to CPU. This allows training RNNs +# +# which would typically not fit on a single GPU, with very minimal (or no) +# +# performance penalty. +# +# time_major: The shape format of the `inputs` and `outputs` Tensors. +# +# If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`. +# +# If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`. +# +# Using `time_major = True` is a bit more efficient because it avoids +# +# transposes at the beginning and end of the RNN calculation. However, +# +# most TensorFlow data is batch-major, so by default this function +# +# accepts input and emits output in batch-major form. +# +# scope: VariableScope for the created subgraph; defaults to +# +# "bidirectional_rnn" +# +# +# +# Returns: +# +# A tuple (outputs, output_states) where: +# +# outputs: A tuple (output_fw, output_bw) containing the forward and +# +# the backward rnn output `Tensor`. +# +# If time_major == False (default), +# +# output_fw will be a `Tensor` shaped: +# +# `[batch_size, max_time, cell_fw.output_size]` +# +# and output_bw will be a `Tensor` shaped: +# +# `[batch_size, max_time, cell_bw.output_size]`. +# +# If time_major == True, +# +# output_fw will be a `Tensor` shaped: +# +# `[max_time, batch_size, cell_fw.output_size]` +# +# and output_bw will be a `Tensor` shaped: +# +# `[max_time, batch_size, cell_bw.output_size]`. +# +# It returns a tuple instead of a single concatenated `Tensor`, unlike +# +# in the `bidirectional_rnn`. If the concatenated one is preferred, +# +# the forward and backward outputs can be concatenated as +# +# `tf.concat(outputs, 2)`. +# +# output_states: A tuple (output_state_fw, output_state_bw) containing +# +# the forward and the backward final states of bidirectional rnn. +# +# +# +# Raises: +# +# TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`. +# +# """ +# +# +# +# if not _like_rnncell(cell_fw): +# +# raise TypeError("cell_fw must be an instance of RNNCell") +# +# if not _like_rnncell(cell_bw): +# +# raise TypeError("cell_bw must be an instance of RNNCell") +# +# +# +# with vs.variable_scope(scope or "bidirectional_rnn"): +# +# # Forward direction +# +# with vs.variable_scope("fw") as fw_scope: +# +# output_fw, output_state_fw = dynamic_rnn( +# +# cell=cell_fw, inputs=inputs, sequence_length=sequence_length, +# +# initial_state=initial_state_fw, dtype=dtype, +# +# parallel_iterations=parallel_iterations, swap_memory=swap_memory, +# +# time_major=time_major, scope=fw_scope) +# +# +# +# # Backward direction +# +# if not time_major: +# +# time_dim = 1 +# +# batch_dim = 0 +# +# else: +# +# time_dim = 0 +# +# batch_dim = 1 +# +# +# +# def _reverse(input_, seq_lengths, seq_dim, batch_dim): +# +# if seq_lengths is not None: +# +# return array_ops.reverse_sequence( +# +# input=input_, seq_lengths=seq_lengths, +# +# seq_dim=seq_dim, batch_dim=batch_dim) +# +# else: +# +# return array_ops.reverse(input_, axis=[seq_dim]) +# +# +# +# with vs.variable_scope("bw") as bw_scope: +# +# inputs_reverse = _reverse( +# +# inputs, seq_lengths=sequence_length, +# +# seq_dim=time_dim, batch_dim=batch_dim) +# +# tmp, output_state_bw = dynamic_rnn( +# +# cell=cell_bw, inputs=inputs_reverse, sequence_length=sequence_length, +# +# initial_state=initial_state_bw, dtype=dtype, +# +# parallel_iterations=parallel_iterations, swap_memory=swap_memory, +# +# time_major=time_major, scope=bw_scope) +# +# +# +# output_bw = _reverse( +# +# tmp, seq_lengths=sequence_length, +# +# seq_dim=time_dim, batch_dim=batch_dim) +# +# +# +# outputs = (output_fw, output_bw) +# +# output_states = (output_state_fw, output_state_bw) +# +# +# +# return (outputs, output_states) +# + + +def dynamic_rnn(cell, inputs, att_scores=None, sequence_length=None, initial_state=None, + + dtype=None, parallel_iterations=None, swap_memory=False, + + time_major=False, scope=None): + """Creates a recurrent neural network specified by RNNCell `cell`. + Performs fully dynamic unrolling of `inputs`. + Example: + ```python + # create a BasicRNNCell + rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size) + # 'outputs' is a tensor of shape [batch_size, max_time, cell_state_size] + # defining initial state + initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32) + # 'state' is a tensor of shape [batch_size, cell_state_size] + outputs, state = tf.nn.dynamic_rnn(rnn_cell, input_data, + initial_state=initial_state, + dtype=tf.float32) + ``` + ```python + # create 2 LSTMCells + rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [128, 256]] + # create a RNN cell composed sequentially of a number of RNNCells + multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers) + # 'outputs' is a tensor of shape [batch_size, max_time, 256] + # 'state' is a N-tuple where N is the number of LSTMCells containing a + # tf.contrib.rnn.LSTMStateTuple for each cell + outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell, + inputs=data, + dtype=tf.float32) + ``` + Args: + cell: An instance of RNNCell. + inputs: The RNN inputs. + If `time_major == False` (default), this must be a `Tensor` of shape: + `[batch_size, max_time, ...]`, or a nested tuple of such + elements. + If `time_major == True`, this must be a `Tensor` of shape: + `[max_time, batch_size, ...]`, or a nested tuple of such + elements. + This may also be a (possibly nested) tuple of Tensors satisfying + this property. The first two dimensions must match across all the inputs, + but otherwise the ranks and other shape components may differ. + In this case, input to `cell` at each time-step will replicate the + structure of these tuples, except for the time dimension (from which the + time is taken). + The input to `cell` at each time step will be a `Tensor` or (possibly + nested) tuple of Tensors each with dimensions `[batch_size, ...]`. + sequence_length: (optional) An int32/int64 vector sized `[batch_size]`. + Used to copy-through state and zero-out outputs when past a batch + element's sequence length. So it's more for correctness than performance. + initial_state: (optional) An initial state for the RNN. + If `cell.state_size` is an integer, this must be + a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`. + If `cell.state_size` is a tuple, this should be a tuple of + tensors having shapes `[batch_size, s] for s in cell.state_size`. + dtype: (optional) The data type for the initial state and expected output. + Required if initial_state is not provided or RNN state has a heterogeneous + dtype. + parallel_iterations: (Default: 32). The number of iterations to run in + parallel. Those operations which do not have any temporal dependency + and can be run in parallel, will be. This parameter trades off + time for space. Values >> 1 use more memory but take less time, + while smaller values use less memory but computations take longer. + swap_memory: Transparently swap the tensors produced in forward inference + but needed for back prop from GPU to CPU. This allows training RNNs + which would typically not fit on a single GPU, with very minimal (or no) + performance penalty. + time_major: The shape format of the `inputs` and `outputs` Tensors. + If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`. + If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`. + Using `time_major = True` is a bit more efficient because it avoids + transposes at the beginning and end of the RNN calculation. However, + most TensorFlow data is batch-major, so by default this function + accepts input and emits output in batch-major form. + scope: VariableScope for the created subgraph; defaults to "rnn". + Returns: + A pair (outputs, state) where: + outputs: The RNN output `Tensor`. + If time_major == False (default), this will be a `Tensor` shaped: + `[batch_size, max_time, cell.output_size]`. + If time_major == True, this will be a `Tensor` shaped: + `[max_time, batch_size, cell.output_size]`. + Note, if `cell.output_size` is a (possibly nested) tuple of integers + or `TensorShape` objects, then `outputs` will be a tuple having the + same structure as `cell.output_size`, containing Tensors having shapes + corresponding to the shape data in `cell.output_size`. + state: The final state. If `cell.state_size` is an int, this + will be shaped `[batch_size, cell.state_size]`. If it is a + `TensorShape`, this will be shaped `[batch_size] + cell.state_size`. + If it is a (possibly nested) tuple of ints or `TensorShape`, this will + be a tuple having the corresponding shapes. If cells are `LSTMCells` + `state` will be a tuple containing a `LSTMStateTuple` for each cell. + Raises: + TypeError: If `cell` is not an instance of RNNCell. + ValueError: If inputs is None or an empty list. + """ + + if not _like_rnncell(cell): + raise TypeError("cell must be an instance of RNNCell") + + # By default, time_major==False and inputs are batch-major: shaped + + # [batch, time, depth] + + # For internal calculations, we transpose to [time, batch, depth] + + flat_input = nest.flatten(inputs) + + if not time_major: + # (B,T,D) => (T,B,D) + + flat_input = [ops.convert_to_tensor(input_) for input_ in flat_input] + + flat_input = tuple(_transpose_batch_time(input_) for input_ in flat_input) + + parallel_iterations = parallel_iterations or 32 + + if sequence_length is not None: + + sequence_length = math_ops.to_int32(sequence_length) + + if sequence_length.get_shape().ndims not in (None, 1): + raise ValueError( + + "sequence_length must be a vector of length batch_size, " + + "but saw shape: %s" % sequence_length.get_shape()) + + sequence_length = array_ops.identity( # Just to find it in the graph. + + sequence_length, name="sequence_length") + + # Create a new scope in which the caching device is either + + # determined by the parent scope, or is set to place the cached + + # Variable using the same placement as for the rest of the RNN. + + with vs.variable_scope(scope or "rnn",reuse=tf.AUTO_REUSE) as varscope:#TODO:user defined reuse + + if varscope.caching_device is None: + varscope.set_caching_device(lambda op: op.device) + + batch_size = _best_effort_input_batch_size(flat_input) + + if initial_state is not None: + + state = initial_state + + else: + + if not dtype: + raise ValueError("If there is no initial_state, you must give a dtype.") + + state = cell.zero_state(batch_size, dtype) + + def _assert_has_shape(x, shape): + + x_shape = array_ops.shape(x) + + packed_shape = array_ops.stack(shape) + + return control_flow_ops.Assert( + + math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)), + + ["Expected shape for Tensor %s is " % x.name, + + packed_shape, " but saw shape: ", x_shape]) + + if sequence_length is not None: + # Perform some shape validation + + with ops.control_dependencies( + + [_assert_has_shape(sequence_length, [batch_size])]): + sequence_length = array_ops.identity( + + sequence_length, name="CheckSeqLen") + + inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input) + + (outputs, final_state) = _dynamic_rnn_loop( + + cell, + + inputs, + + state, + + parallel_iterations=parallel_iterations, + + swap_memory=swap_memory, + + att_scores=att_scores, + + sequence_length=sequence_length, + + dtype=dtype) + + # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth]. + + # If we are performing batch-major calculations, transpose output back + + # to shape [batch, time, depth] + + if not time_major: + # (T,B,D) => (B,T,D) + + outputs = nest.map_structure(_transpose_batch_time, outputs) + + return (outputs, final_state) + + +def _dynamic_rnn_loop(cell, + + inputs, + + initial_state, + + parallel_iterations, + + swap_memory, + + att_scores=None, + + sequence_length=None, + + dtype=None): + """Internal implementation of Dynamic RNN. + Args: + cell: An instance of RNNCell. + inputs: A `Tensor` of shape [time, batch_size, input_size], or a nested + tuple of such elements. + initial_state: A `Tensor` of shape `[batch_size, state_size]`, or if + `cell.state_size` is a tuple, then this should be a tuple of + tensors having shapes `[batch_size, s] for s in cell.state_size`. + parallel_iterations: Positive Python int. + swap_memory: A Python boolean + sequence_length: (optional) An `int32` `Tensor` of shape [batch_size]. + dtype: (optional) Expected dtype of output. If not specified, inferred from + initial_state. + Returns: + Tuple `(final_outputs, final_state)`. + final_outputs: + A `Tensor` of shape `[time, batch_size, cell.output_size]`. If + `cell.output_size` is a (possibly nested) tuple of ints or `TensorShape` + objects, then this returns a (possibly nsted) tuple of Tensors matching + the corresponding shapes. + final_state: + A `Tensor`, or possibly nested tuple of Tensors, matching in length + and shapes to `initial_state`. + Raises: + ValueError: If the input depth cannot be inferred via shape inference + from the inputs. + """ + + state = initial_state + + assert isinstance(parallel_iterations, int), "parallel_iterations must be int" + + state_size = cell.state_size + + flat_input = nest.flatten(inputs) + + flat_output_size = nest.flatten(cell.output_size) + + # Construct an initial output + + input_shape = array_ops.shape(flat_input[0]) + + time_steps = input_shape[0] + + batch_size = _best_effort_input_batch_size(flat_input) + + inputs_got_shape = tuple(input_.get_shape().with_rank_at_least(3) + + for input_ in flat_input) + + const_time_steps, const_batch_size = inputs_got_shape[0].as_list()[:2] + + for shape in inputs_got_shape: + + if not shape[2:].is_fully_defined(): + raise ValueError( + + "Input size (depth of inputs) must be accessible via shape inference," + + " but saw value None.") + + got_time_steps = shape[0].value + + got_batch_size = shape[1].value + + if const_time_steps != got_time_steps: + raise ValueError( + + "Time steps is not the same for all the elements in the input in a " + + "batch.") + + if const_batch_size != got_batch_size: + raise ValueError( + + "Batch_size is not the same for all the elements in the input.") + + # Prepare dynamic conditional copying of state & output + + def _create_zero_arrays(size): + + size = _concat(batch_size, size) + + return array_ops.zeros( + + array_ops.stack(size), _infer_state_dtype(dtype, state)) + + flat_zero_output = tuple(_create_zero_arrays(output) + + for output in flat_output_size) + + zero_output = nest.pack_sequence_as(structure=cell.output_size, + + flat_sequence=flat_zero_output) + + if sequence_length is not None: + min_sequence_length = math_ops.reduce_min(sequence_length) + + max_sequence_length = math_ops.reduce_max(sequence_length) + + time = array_ops.constant(0, dtype=dtypes.int32, name="time") + + with ops.name_scope("dynamic_rnn") as scope: + + base_name = scope + + def _create_ta(name, dtype): + + return tensor_array_ops.TensorArray(dtype=dtype, + + size=time_steps, + + tensor_array_name=base_name + name) + + output_ta = tuple(_create_ta("output_%d" % i, + + _infer_state_dtype(dtype, state)) + + for i in range(len(flat_output_size))) + + input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype) + + for i in range(len(flat_input))) + + input_ta = tuple(ta.unstack(input_) + + for ta, input_ in zip(input_ta, flat_input)) + + def _time_step(time, output_ta_t, state, att_scores=None): + + """Take a time step of the dynamic RNN. + Args: + time: int32 scalar Tensor. + output_ta_t: List of `TensorArray`s that represent the output. + state: nested tuple of vector tensors that represent the state. + Returns: + The tuple (time + 1, output_ta_t with updated flow, new_state). + """ + + input_t = tuple(ta.read(time) for ta in input_ta) + + # Restore some shape information + + for input_, shape in zip(input_t, inputs_got_shape): + input_.set_shape(shape[1:]) + + input_t = nest.pack_sequence_as(structure=inputs, flat_sequence=input_t) + + if att_scores is not None: + + att_score = att_scores[:, time, :] + + call_cell = lambda: cell(input_t, state, att_score) + + else: + + call_cell = lambda: cell(input_t, state) + + if sequence_length is not None: + + (output, new_state) = _rnn_step( + + time=time, + + sequence_length=sequence_length, + + min_sequence_length=min_sequence_length, + + max_sequence_length=max_sequence_length, + + zero_output=zero_output, + + state=state, + + call_cell=call_cell, + + state_size=state_size, + + skip_conditionals=True) + + else: + + (output, new_state) = call_cell() + + # Pack state if using state tuples + + output = nest.flatten(output) + + output_ta_t = tuple( + + ta.write(time, out) for ta, out in zip(output_ta_t, output)) + + if att_scores is not None: + + return (time + 1, output_ta_t, new_state, att_scores) + + else: + + return (time + 1, output_ta_t, new_state) + + if att_scores is not None: + + _, output_final_ta, final_state, _ = control_flow_ops.while_loop( + + cond=lambda time, *_: time < time_steps, + + body=_time_step, + + loop_vars=(time, output_ta, state, att_scores), + + parallel_iterations=parallel_iterations, + + swap_memory=swap_memory) + + else: + + _, output_final_ta, final_state = control_flow_ops.while_loop( + + cond=lambda time, *_: time < time_steps, + + body=_time_step, + + loop_vars=(time, output_ta, state), + + parallel_iterations=parallel_iterations, + + swap_memory=swap_memory) + + # Unpack final output if not using output tuples. + + final_outputs = tuple(ta.stack() for ta in output_final_ta) + + # Restore some shape information + + for output, output_size in zip(final_outputs, flat_output_size): + shape = _concat( + + [const_time_steps, const_batch_size], output_size, static=True) + + output.set_shape(shape) + + final_outputs = nest.pack_sequence_as( + + structure=cell.output_size, flat_sequence=final_outputs) + + return (final_outputs, final_state) diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/rnn_v2.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/rnn_v2.py new file mode 100644 index 000000000..cc55737dd --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/rnn_v2.py @@ -0,0 +1,1454 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +# + +# Licensed under the Apache License, Version 2.0 (the "License"); + +# you may not use this file except in compliance with the License. + +# You may obtain a copy of the License at + +# + +# http://www.apache.org/licenses/LICENSE-2.0 + +# + +# Unless required by applicable law or agreed to in writing, software + +# distributed under the License is distributed on an "AS IS" BASIS, + +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +# See the License for the specific language governing permissions and + +# limitations under the License. + +# ============================================================================== + + +"""RNN helpers for TensorFlow models. + + + + + +@@bidirectional_dynamic_rnn + +@@dynamic_rnn + +@@raw_rnn + +@@static_rnn + +@@static_state_saving_rnn + +@@static_bidirectional_rnn + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from npu_bridge.npu_init import * + +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import rnn_cell_impl +from tensorflow.python.ops import tensor_array_ops +from tensorflow.python.ops import variable_scope as vs +from tensorflow.python.util import nest +import tensorflow as tf + + +def _like_rnncell_(cell): + """Checks that a given object is an RNNCell by using duck typing.""" + + conditions = [hasattr(cell, "output_size"), hasattr(cell, "state_size"), + + hasattr(cell, "zero_state"), callable(cell)] + + return all(conditions) + + +# pylint: disable=protected-access + +_concat = rnn_cell_impl._concat +try: + _like_rnncell = rnn_cell_impl._like_rnncell +except: + _like_rnncell = _like_rnncell_ + + +# pylint: enable=protected-access + + +def _transpose_batch_time(x): + """Transpose the batch and time dimensions of a Tensor. + + + + Retains as much of the static shape information as possible. + + + + Args: + + x: A tensor of rank 2 or higher. + + + + Returns: + + x transposed along the first two dimensions. + + + + Raises: + + ValueError: if `x` is rank 1 or lower. + + """ + + x_static_shape = x.get_shape() + + if x_static_shape.ndims is not None and x_static_shape.ndims < 2: + raise ValueError( + + "Expected input tensor %s to have rank at least 2, but saw shape: %s" % + + (x, x_static_shape)) + + x_rank = array_ops.rank(x) + + x_t = array_ops.transpose( + + x, array_ops.concat( + + ([1, 0], math_ops.range(2, x_rank)), axis=0)) + + x_t.set_shape( + + tensor_shape.TensorShape([ + + x_static_shape[1], x_static_shape[0] + + ]).concatenate(x_static_shape[2:])) + + return x_t + + +def _best_effort_input_batch_size(flat_input): + """Get static input batch size if available, with fallback to the dynamic one. + + + + Args: + + flat_input: An iterable of time major input Tensors of shape [max_time, + + batch_size, ...]. All inputs should have compatible batch sizes. + + + + Returns: + + The batch size in Python integer if available, or a scalar Tensor otherwise. + + + + Raises: + + ValueError: if there is any input with an invalid shape. + + """ + + for input_ in flat_input: + + shape = input_.shape + + if shape.ndims is None: + continue + + if shape.ndims < 2: + raise ValueError( + + "Expected input tensor %s to have rank at least 2" % input_) + + batch_size = shape[1] + + if batch_size is not None: + return batch_size + + # Fallback to the dynamic batch size of the first input. + + return array_ops.shape(flat_input[0])[1] + + +def _infer_state_dtype(explicit_dtype, state): + """Infer the dtype of an RNN state. + + + + Args: + + explicit_dtype: explicitly declared dtype or None. + + state: RNN's hidden state. Must be a Tensor or a nested iterable containing + + Tensors. + + + + Returns: + + dtype: inferred dtype of hidden state. + + + + Raises: + + ValueError: if `state` has heterogeneous dtypes or is empty. + + """ + + if explicit_dtype is not None: + + return explicit_dtype + + elif nest.is_sequence(state): + + inferred_dtypes = [element.dtype for element in nest.flatten(state)] + + if not inferred_dtypes: + raise ValueError("Unable to infer dtype from empty state.") + + all_same = all([x == inferred_dtypes[0] for x in inferred_dtypes]) + + if not all_same: + raise ValueError( + + "State has tensors of different inferred_dtypes. Unable to infer a " + + "single representative dtype.") + + return inferred_dtypes[0] + + else: + + return state.dtype + + +# pylint: disable=unused-argument + +def _rnn_step( + + time, sequence_length, min_sequence_length, max_sequence_length, + + zero_output, state, call_cell, state_size, skip_conditionals=False): + """Calculate one step of a dynamic RNN minibatch. + + + + Returns an (output, state) pair conditioned on the sequence_lengths. + + When skip_conditionals=False, the pseudocode is something like: + + + + if t >= max_sequence_length: + + return (zero_output, state) + + if t < min_sequence_length: + + return call_cell() + + + + # Selectively output zeros or output, old state or new state depending + + # on if we've finished calculating each row. + + new_output, new_state = call_cell() + + final_output = np.vstack([ + + zero_output if time >= sequence_lengths[r] else new_output_r + + for r, new_output_r in enumerate(new_output) + + ]) + + final_state = np.vstack([ + + state[r] if time >= sequence_lengths[r] else new_state_r + + for r, new_state_r in enumerate(new_state) + + ]) + + return (final_output, final_state) + + + + Args: + + time: Python int, the current time step + + sequence_length: int32 `Tensor` vector of size [batch_size] + + min_sequence_length: int32 `Tensor` scalar, min of sequence_length + + max_sequence_length: int32 `Tensor` scalar, max of sequence_length + + zero_output: `Tensor` vector of shape [output_size] + + state: Either a single `Tensor` matrix of shape `[batch_size, state_size]`, + + or a list/tuple of such tensors. + + call_cell: lambda returning tuple of (new_output, new_state) where + + new_output is a `Tensor` matrix of shape `[batch_size, output_size]`. + + new_state is a `Tensor` matrix of shape `[batch_size, state_size]`. + + state_size: The `cell.state_size` associated with the state. + + skip_conditionals: Python bool, whether to skip using the conditional + + calculations. This is useful for `dynamic_rnn`, where the input tensor + + matches `max_sequence_length`, and using conditionals just slows + + everything down. + + + + Returns: + + A tuple of (`final_output`, `final_state`) as given by the pseudocode above: + + final_output is a `Tensor` matrix of shape [batch_size, output_size] + + final_state is either a single `Tensor` matrix, or a tuple of such + + matrices (matching length and shapes of input `state`). + + + + Raises: + + ValueError: If the cell returns a state tuple whose length does not match + + that returned by `state_size`. + + """ + + # Convert state to a list for ease of use + + flat_state = nest.flatten(state) + + flat_zero_output = nest.flatten(zero_output) + + def _copy_one_through(output, new_output): + + # If the state contains a scalar value we simply pass it through. + + if output.shape.ndims == 0: + return new_output + + copy_cond = (time >= sequence_length) + + with ops.colocate_with(new_output): + return array_ops.where(copy_cond, output, new_output) + + def _copy_some_through(flat_new_output, flat_new_state): + + # Use broadcasting select to determine which values should get + + # the previous state & zero output, and which values should get + + # a calculated state & output. + + flat_new_output = [ + + _copy_one_through(zero_output, new_output) + + for zero_output, new_output in zip(flat_zero_output, flat_new_output)] + + flat_new_state = [ + + _copy_one_through(state, new_state) + + for state, new_state in zip(flat_state, flat_new_state)] + + return flat_new_output + flat_new_state + + def _maybe_copy_some_through(): + + """Run RNN step. Pass through either no or some past state.""" + + new_output, new_state = call_cell() + + nest.assert_same_structure(state, new_state) + + flat_new_state = nest.flatten(new_state) + + flat_new_output = nest.flatten(new_output) + + return control_flow_ops.cond( + + # if t < min_seq_len: calculate and return everything + + time < min_sequence_length, lambda: flat_new_output + flat_new_state, + + # else copy some of it through + + lambda: _copy_some_through(flat_new_output, flat_new_state)) + + # TODO(ebrevdo): skipping these conditionals may cause a slowdown, + + # but benefits from removing cond() and its gradient. We should + + # profile with and without this switch here. + + if skip_conditionals: + + # Instead of using conditionals, perform the selective copy at all time + + # steps. This is faster when max_seq_len is equal to the number of unrolls + + # (which is typical for dynamic_rnn). + + new_output, new_state = call_cell() + + nest.assert_same_structure(state, new_state) + + new_state = nest.flatten(new_state) + + new_output = nest.flatten(new_output) + + final_output_and_state = _copy_some_through(new_output, new_state) + + else: + + empty_update = lambda: flat_zero_output + flat_state + + final_output_and_state = control_flow_ops.cond( + + # if t >= max_seq_len: copy all state through, output zeros + + time >= max_sequence_length, empty_update, + + # otherwise calculation is required: copy some or all of it through + + _maybe_copy_some_through) + + if len(final_output_and_state) != len(flat_zero_output) + len(flat_state): + raise ValueError("Internal error: state and output were not concatenated " + + "correctly.") + + final_output = final_output_and_state[:len(flat_zero_output)] + + final_state = final_output_and_state[len(flat_zero_output):] + + for output, flat_output in zip(final_output, flat_zero_output): + output.set_shape(flat_output.get_shape()) + + for substate, flat_substate in zip(final_state, flat_state): + substate.set_shape(flat_substate.get_shape()) + + final_output = nest.pack_sequence_as( + + structure=zero_output, flat_sequence=final_output) + + final_state = nest.pack_sequence_as( + + structure=state, flat_sequence=final_state) + + return final_output, final_state + + +def _reverse_seq(input_seq, lengths): + """Reverse a list of Tensors up to specified lengths. + + + + Args: + + input_seq: Sequence of seq_len tensors of dimension (batch_size, n_features) + + or nested tuples of tensors. + + lengths: A `Tensor` of dimension batch_size, containing lengths for each + + sequence in the batch. If "None" is specified, simply reverses + + the list. + + + + Returns: + + time-reversed sequence + + """ + + if lengths is None: + return list(reversed(input_seq)) + + flat_input_seq = tuple(nest.flatten(input_) for input_ in input_seq) + + flat_results = [[] for _ in range(len(input_seq))] + + for sequence in zip(*flat_input_seq): + + input_shape = tensor_shape.unknown_shape( + + ndims=sequence[0].get_shape().ndims) + + for input_ in sequence: + input_shape.merge_with(input_.get_shape()) + + input_.set_shape(input_shape) + + # Join into (time, batch_size, depth) + + s_joined = array_ops.stack(sequence) + + # Reverse along dimension 0 + + s_reversed = array_ops.reverse_sequence(s_joined, lengths, 0, 1) + + # Split again into list + + result = array_ops.unstack(s_reversed) + + for r, flat_result in zip(result, flat_results): + r.set_shape(input_shape) + + flat_result.append(r) + + results = [nest.pack_sequence_as(structure=input_, flat_sequence=flat_result) + + for input_, flat_result in zip(input_seq, flat_results)] + + return results + + +# +# def bidirectional_dynamic_rnn(cell_fw, cell_bw, inputs, sequence_length=None, +# +# initial_state_fw=None, initial_state_bw=None, +# +# dtype=None, parallel_iterations=None, +# +# swap_memory=False, time_major=False, scope=None): +# +# """Creates a dynamic version of bidirectional recurrent neural network. +# +# +# +# Takes input and builds independent forward and backward RNNs. The input_size +# +# of forward and backward cell must match. The initial state for both directions +# +# is zero by default (but can be set optionally) and no intermediate states are +# +# ever returned -- the network is fully unrolled for the given (passed in) +# +# length(s) of the sequence(s) or completely unrolled if length(s) is not +# +# given. +# +# +# +# Args: +# +# cell_fw: An instance of RNNCell, to be used for forward direction. +# +# cell_bw: An instance of RNNCell, to be used for backward direction. +# +# inputs: The RNN inputs. +# +# If time_major == False (default), this must be a tensor of shape: +# +# `[batch_size, max_time, ...]`, or a nested tuple of such elements. +# +# If time_major == True, this must be a tensor of shape: +# +# `[max_time, batch_size, ...]`, or a nested tuple of such elements. +# +# sequence_length: (optional) An int32/int64 vector, size `[batch_size]`, +# +# containing the actual lengths for each of the sequences in the batch. +# +# If not provided, all batch entries are assumed to be full sequences; and +# +# time reversal is applied from time `0` to `max_time` for each sequence. +# +# initial_state_fw: (optional) An initial state for the forward RNN. +# +# This must be a tensor of appropriate type and shape +# +# `[batch_size, cell_fw.state_size]`. +# +# If `cell_fw.state_size` is a tuple, this should be a tuple of +# +# tensors having shapes `[batch_size, s] for s in cell_fw.state_size`. +# +# initial_state_bw: (optional) Same as for `initial_state_fw`, but using +# +# the corresponding properties of `cell_bw`. +# +# dtype: (optional) The data type for the initial states and expected output. +# +# Required if initial_states are not provided or RNN states have a +# +# heterogeneous dtype. +# +# parallel_iterations: (Default: 32). The number of iterations to run in +# +# parallel. Those operations which do not have any temporal dependency +# +# and can be run in parallel, will be. This parameter trades off +# +# time for space. Values >> 1 use more memory but take less time, +# +# while smaller values use less memory but computations take longer. +# +# swap_memory: Transparently swap the tensors produced in forward inference +# +# but needed for back prop from GPU to CPU. This allows training RNNs +# +# which would typically not fit on a single GPU, with very minimal (or no) +# +# performance penalty. +# +# time_major: The shape format of the `inputs` and `outputs` Tensors. +# +# If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`. +# +# If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`. +# +# Using `time_major = True` is a bit more efficient because it avoids +# +# transposes at the beginning and end of the RNN calculation. However, +# +# most TensorFlow data is batch-major, so by default this function +# +# accepts input and emits output in batch-major form. +# +# scope: VariableScope for the created subgraph; defaults to +# +# "bidirectional_rnn" +# +# +# +# Returns: +# +# A tuple (outputs, output_states) where: +# +# outputs: A tuple (output_fw, output_bw) containing the forward and +# +# the backward rnn output `Tensor`. +# +# If time_major == False (default), +# +# output_fw will be a `Tensor` shaped: +# +# `[batch_size, max_time, cell_fw.output_size]` +# +# and output_bw will be a `Tensor` shaped: +# +# `[batch_size, max_time, cell_bw.output_size]`. +# +# If time_major == True, +# +# output_fw will be a `Tensor` shaped: +# +# `[max_time, batch_size, cell_fw.output_size]` +# +# and output_bw will be a `Tensor` shaped: +# +# `[max_time, batch_size, cell_bw.output_size]`. +# +# It returns a tuple instead of a single concatenated `Tensor`, unlike +# +# in the `bidirectional_rnn`. If the concatenated one is preferred, +# +# the forward and backward outputs can be concatenated as +# +# `tf.concat(outputs, 2)`. +# +# output_states: A tuple (output_state_fw, output_state_bw) containing +# +# the forward and the backward final states of bidirectional rnn. +# +# +# +# Raises: +# +# TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`. +# +# """ +# +# +# +# if not _like_rnncell(cell_fw): +# +# raise TypeError("cell_fw must be an instance of RNNCell") +# +# if not _like_rnncell(cell_bw): +# +# raise TypeError("cell_bw must be an instance of RNNCell") +# +# +# +# with vs.variable_scope(scope or "bidirectional_rnn"): +# +# # Forward direction +# +# with vs.variable_scope("fw") as fw_scope: +# +# output_fw, output_state_fw = dynamic_rnn( +# +# cell=cell_fw, inputs=inputs, sequence_length=sequence_length, +# +# initial_state=initial_state_fw, dtype=dtype, +# +# parallel_iterations=parallel_iterations, swap_memory=swap_memory, +# +# time_major=time_major, scope=fw_scope) +# +# +# +# # Backward direction +# +# if not time_major: +# +# time_dim = 1 +# +# batch_dim = 0 +# +# else: +# +# time_dim = 0 +# +# batch_dim = 1 +# +# +# +# def _reverse(input_, seq_lengths, seq_dim, batch_dim): +# +# if seq_lengths is not None: +# +# return array_ops.reverse_sequence( +# +# input=input_, seq_lengths=seq_lengths, +# +# seq_dim=seq_dim, batch_dim=batch_dim) +# +# else: +# +# return array_ops.reverse(input_, axis=[seq_dim]) +# +# +# +# with vs.variable_scope("bw") as bw_scope: +# +# inputs_reverse = _reverse( +# +# inputs, seq_lengths=sequence_length, +# +# seq_dim=time_dim, batch_dim=batch_dim) +# +# tmp, output_state_bw = dynamic_rnn( +# +# cell=cell_bw, inputs=inputs_reverse, sequence_length=sequence_length, +# +# initial_state=initial_state_bw, dtype=dtype, +# +# parallel_iterations=parallel_iterations, swap_memory=swap_memory, +# +# time_major=time_major, scope=bw_scope) +# +# +# +# output_bw = _reverse( +# +# tmp, seq_lengths=sequence_length, +# +# seq_dim=time_dim, batch_dim=batch_dim) +# +# +# +# outputs = (output_fw, output_bw) +# +# output_states = (output_state_fw, output_state_bw) +# +# +# +# return (outputs, output_states) +# + + +def dynamic_rnn(cell, inputs, att_scores=None, sequence_length=None, initial_state=None, + + dtype=None, parallel_iterations=None, swap_memory=False, + + time_major=False, scope=None): + """Creates a recurrent neural network specified by RNNCell `cell`. + + + + Performs fully dynamic unrolling of `inputs`. + + + + Example: + + + + ```python + + # create a BasicRNNCell + + rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size) + + + + # 'outputs' is a tensor of shape [batch_size, max_time, cell_state_size] + + + + # defining initial state + + initial_state = rnn_cell.zero_state(batch_size, dtype=tf.float32) + + + + # 'state' is a tensor of shape [batch_size, cell_state_size] + + outputs, state = tf.nn.dynamic_rnn(rnn_cell, input_data, + + initial_state=initial_state, + + dtype=tf.float32) + + ``` + + + + ```python + + # create 2 LSTMCells + + rnn_layers = [tf.nn.rnn_cell.LSTMCell(size) for size in [128, 256]] + + + + # create a RNN cell composed sequentially of a number of RNNCells + + multi_rnn_cell = tf.nn.rnn_cell.MultiRNNCell(rnn_layers) + + + + # 'outputs' is a tensor of shape [batch_size, max_time, 256] + + # 'state' is a N-tuple where N is the number of LSTMCells containing a + + # tf.contrib.rnn.LSTMStateTuple for each cell + + outputs, state = tf.nn.dynamic_rnn(cell=multi_rnn_cell, + + inputs=data, + + dtype=tf.float32) + + ``` + + + + + + Args: + + cell: An instance of RNNCell. + + inputs: The RNN inputs. + + If `time_major == False` (default), this must be a `Tensor` of shape: + + `[batch_size, max_time, ...]`, or a nested tuple of such + + elements. + + If `time_major == True`, this must be a `Tensor` of shape: + + `[max_time, batch_size, ...]`, or a nested tuple of such + + elements. + + This may also be a (possibly nested) tuple of Tensors satisfying + + this property. The first two dimensions must match across all the inputs, + + but otherwise the ranks and other shape components may differ. + + In this case, input to `cell` at each time-step will replicate the + + structure of these tuples, except for the time dimension (from which the + + time is taken). + + The input to `cell` at each time step will be a `Tensor` or (possibly + + nested) tuple of Tensors each with dimensions `[batch_size, ...]`. + + sequence_length: (optional) An int32/int64 vector sized `[batch_size]`. + + Used to copy-through state and zero-out outputs when past a batch + + element's sequence length. So it's more for correctness than performance. + + initial_state: (optional) An initial state for the RNN. + + If `cell.state_size` is an integer, this must be + + a `Tensor` of appropriate type and shape `[batch_size, cell.state_size]`. + + If `cell.state_size` is a tuple, this should be a tuple of + + tensors having shapes `[batch_size, s] for s in cell.state_size`. + + dtype: (optional) The data type for the initial state and expected output. + + Required if initial_state is not provided or RNN state has a heterogeneous + + dtype. + + parallel_iterations: (Default: 32). The number of iterations to run in + + parallel. Those operations which do not have any temporal dependency + + and can be run in parallel, will be. This parameter trades off + + time for space. Values >> 1 use more memory but take less time, + + while smaller values use less memory but computations take longer. + + swap_memory: Transparently swap the tensors produced in forward inference + + but needed for back prop from GPU to CPU. This allows training RNNs + + which would typically not fit on a single GPU, with very minimal (or no) + + performance penalty. + + time_major: The shape format of the `inputs` and `outputs` Tensors. + + If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`. + + If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`. + + Using `time_major = True` is a bit more efficient because it avoids + + transposes at the beginning and end of the RNN calculation. However, + + most TensorFlow data is batch-major, so by default this function + + accepts input and emits output in batch-major form. + + scope: VariableScope for the created subgraph; defaults to "rnn". + + + + Returns: + + A pair (outputs, state) where: + + + + outputs: The RNN output `Tensor`. + + + + If time_major == False (default), this will be a `Tensor` shaped: + + `[batch_size, max_time, cell.output_size]`. + + + + If time_major == True, this will be a `Tensor` shaped: + + `[max_time, batch_size, cell.output_size]`. + + + + Note, if `cell.output_size` is a (possibly nested) tuple of integers + + or `TensorShape` objects, then `outputs` will be a tuple having the + + same structure as `cell.output_size`, containing Tensors having shapes + + corresponding to the shape data in `cell.output_size`. + + + + state: The final state. If `cell.state_size` is an int, this + + will be shaped `[batch_size, cell.state_size]`. If it is a + + `TensorShape`, this will be shaped `[batch_size] + cell.state_size`. + + If it is a (possibly nested) tuple of ints or `TensorShape`, this will + + be a tuple having the corresponding shapes. If cells are `LSTMCells` + + `state` will be a tuple containing a `LSTMStateTuple` for each cell. + + + + Raises: + + TypeError: If `cell` is not an instance of RNNCell. + + ValueError: If inputs is None or an empty list. + + """ + + if not _like_rnncell(cell): + raise TypeError("cell must be an instance of RNNCell") + + # By default, time_major==False and inputs are batch-major: shaped + + # [batch, time, depth] + + # For internal calculations, we transpose to [time, batch, depth] + + flat_input = nest.flatten(inputs) + + if not time_major: + # (B,T,D) => (T,B,D) + + flat_input = [ops.convert_to_tensor(input_) for input_ in flat_input] + + flat_input = tuple(_transpose_batch_time(input_) for input_ in flat_input) + + parallel_iterations = parallel_iterations or 32 + + if sequence_length is not None: + + sequence_length = math_ops.to_int32(sequence_length) + + if sequence_length.get_shape().ndims not in (None, 1): + raise ValueError( + + "sequence_length must be a vector of length batch_size, " + + "but saw shape: %s" % sequence_length.get_shape()) + + sequence_length = array_ops.identity( # Just to find it in the graph. + + sequence_length, name="sequence_length") + + # Create a new scope in which the caching device is either + + # determined by the parent scope, or is set to place the cached + + # Variable using the same placement as for the rest of the RNN. + + try: + resue = tf.AUTO_REUSE + except: + resue = tf.compat.v1.AUTO_REUSE + + with vs.variable_scope(scope or "rnn",reuse=resue) as varscope:#TODO:user defined reuse + + if varscope.caching_device is None: + varscope.set_caching_device(lambda op: op.device) + + batch_size = _best_effort_input_batch_size(flat_input) + + if initial_state is not None: + + state = initial_state + + else: + + if not dtype: + raise ValueError("If there is no initial_state, you must give a dtype.") + + state = cell.zero_state(batch_size, dtype) + + def _assert_has_shape(x, shape): + + x_shape = array_ops.shape(x) + + packed_shape = array_ops.stack(shape) + + return control_flow_ops.Assert( + + math_ops.reduce_all(math_ops.equal(x_shape, packed_shape)), + + ["Expected shape for Tensor %s is " % x.name, + + packed_shape, " but saw shape: ", x_shape]) + + if sequence_length is not None: + # Perform some shape validation + + with ops.control_dependencies( + + [_assert_has_shape(sequence_length, [batch_size])]): + sequence_length = array_ops.identity( + + sequence_length, name="CheckSeqLen") + + inputs = nest.pack_sequence_as(structure=inputs, flat_sequence=flat_input) + + (outputs, final_state) = _dynamic_rnn_loop( + + cell, + + inputs, + + state, + + parallel_iterations=parallel_iterations, + + swap_memory=swap_memory, + + att_scores=att_scores, + + sequence_length=sequence_length, + + dtype=dtype) + + # Outputs of _dynamic_rnn_loop are always shaped [time, batch, depth]. + + # If we are performing batch-major calculations, transpose output back + + # to shape [batch, time, depth] + + if not time_major: + # (T,B,D) => (B,T,D) + + outputs = nest.map_structure(_transpose_batch_time, outputs) + + return (outputs, final_state) + + +def _dynamic_rnn_loop(cell, + + inputs, + + initial_state, + + parallel_iterations, + + swap_memory, + + att_scores=None, + + sequence_length=None, + + dtype=None): + """Internal implementation of Dynamic RNN. + + + + Args: + + cell: An instance of RNNCell. + + inputs: A `Tensor` of shape [time, batch_size, input_size], or a nested + + tuple of such elements. + + initial_state: A `Tensor` of shape `[batch_size, state_size]`, or if + + `cell.state_size` is a tuple, then this should be a tuple of + + tensors having shapes `[batch_size, s] for s in cell.state_size`. + + parallel_iterations: Positive Python int. + + swap_memory: A Python boolean + + sequence_length: (optional) An `int32` `Tensor` of shape [batch_size]. + + dtype: (optional) Expected dtype of output. If not specified, inferred from + + initial_state. + + + + Returns: + + Tuple `(final_outputs, final_state)`. + + final_outputs: + + A `Tensor` of shape `[time, batch_size, cell.output_size]`. If + + `cell.output_size` is a (possibly nested) tuple of ints or `TensorShape` + + objects, then this returns a (possibly nsted) tuple of Tensors matching + + the corresponding shapes. + + final_state: + + A `Tensor`, or possibly nested tuple of Tensors, matching in length + + and shapes to `initial_state`. + + + + Raises: + + ValueError: If the input depth cannot be inferred via shape inference + + from the inputs. + + """ + + state = initial_state + + assert isinstance(parallel_iterations, int), "parallel_iterations must be int" + + state_size = cell.state_size + + flat_input = nest.flatten(inputs) + + flat_output_size = nest.flatten(cell.output_size) + + # Construct an initial output + + input_shape = array_ops.shape(flat_input[0]) + + time_steps = input_shape[0] + + batch_size = _best_effort_input_batch_size(flat_input) + + inputs_got_shape = tuple(input_.get_shape().with_rank_at_least(3) + + for input_ in flat_input) + + const_time_steps, const_batch_size = inputs_got_shape[0].as_list()[:2] + + for shape in inputs_got_shape: + + if not shape[2:].is_fully_defined(): + raise ValueError( + + "Input size (depth of inputs) must be accessible via shape inference," + + " but saw value None.") + + got_time_steps = shape[0] + + got_batch_size = shape[1] + + if const_time_steps != got_time_steps: + raise ValueError( + + "Time steps is not the same for all the elements in the input in a " + + "batch.") + + if const_batch_size != got_batch_size: + raise ValueError( + + "Batch_size is not the same for all the elements in the input.") + + # Prepare dynamic conditional copying of state & output + + def _create_zero_arrays(size): + + size = _concat(batch_size, size) + + return array_ops.zeros( + + array_ops.stack(size), _infer_state_dtype(dtype, state)) + + flat_zero_output = tuple(_create_zero_arrays(output) + + for output in flat_output_size) + + zero_output = nest.pack_sequence_as(structure=cell.output_size, + + flat_sequence=flat_zero_output) + + if sequence_length is not None: + min_sequence_length = math_ops.reduce_min(sequence_length) + + max_sequence_length = math_ops.reduce_max(sequence_length) + + time = array_ops.constant(0, dtype=dtypes.int32, name="time") + + with ops.name_scope("dynamic_rnn") as scope: + + base_name = scope + + def _create_ta(name, dtype): + + return tensor_array_ops.TensorArray(dtype=dtype, + + size=time_steps, + + tensor_array_name=base_name + name) + + output_ta = tuple(_create_ta("output_%d" % i, + + _infer_state_dtype(dtype, state)) + + for i in range(len(flat_output_size))) + + input_ta = tuple(_create_ta("input_%d" % i, flat_input[i].dtype) + + for i in range(len(flat_input))) + + input_ta = tuple(ta.unstack(input_) + + for ta, input_ in zip(input_ta, flat_input)) + + def _time_step(time, output_ta_t, state, att_scores=None): + + """Take a time step of the dynamic RNN. + + + + Args: + + time: int32 scalar Tensor. + + output_ta_t: List of `TensorArray`s that represent the output. + + state: nested tuple of vector tensors that represent the state. + + + + Returns: + + The tuple (time + 1, output_ta_t with updated flow, new_state). + + """ + + input_t = tuple(ta.read(time) for ta in input_ta) + + # Restore some shape information + + for input_, shape in zip(input_t, inputs_got_shape): + input_.set_shape(shape[1:]) + + input_t = nest.pack_sequence_as(structure=inputs, flat_sequence=input_t) + + if att_scores is not None: + + att_score = att_scores[:, time, :] + + call_cell = lambda: cell(input_t, state, att_score) + + else: + + call_cell = lambda: cell(input_t, state) + + if sequence_length is not None: + + (output, new_state) = _rnn_step( + + time=time, + + sequence_length=sequence_length, + + min_sequence_length=min_sequence_length, + + max_sequence_length=max_sequence_length, + + zero_output=zero_output, + + state=state, + + call_cell=call_cell, + + state_size=state_size, + + skip_conditionals=True) + + else: + + (output, new_state) = call_cell() + + # Pack state if using state tuples + + output = nest.flatten(output) + + output_ta_t = tuple( + + ta.write(time, out) for ta, out in zip(output_ta_t, output)) + + if att_scores is not None: + + return (time + 1, output_ta_t, new_state, att_scores) + + else: + + return (time + 1, output_ta_t, new_state) + + if att_scores is not None: + + _, output_final_ta, final_state, _ = control_flow_ops.while_loop( + + cond=lambda time, *_: time < time_steps, + + body=_time_step, + + loop_vars=(time, output_ta, state, att_scores), + + parallel_iterations=parallel_iterations, + + swap_memory=swap_memory) + + else: + + _, output_final_ta, final_state = control_flow_ops.while_loop( + + cond=lambda time, *_: time < time_steps, + + body=_time_step, + + loop_vars=(time, output_ta, state), + + parallel_iterations=parallel_iterations, + + swap_memory=swap_memory) + + # Unpack final output if not using output tuples. + + final_outputs = tuple(ta.stack() for ta in output_final_ta) + + # Restore some shape information + + for output, output_size in zip(final_outputs, flat_output_size): + shape = _concat( + + [const_time_steps, const_batch_size], output_size, static=True) + + output.set_shape(shape) + + final_outputs = nest.pack_sequence_as( + + structure=cell.output_size, flat_sequence=final_outputs) + + return (final_outputs, final_state) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/utils.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/utils.py new file mode 100644 index 000000000..52161993f --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/utils.py @@ -0,0 +1,380 @@ +from npu_bridge.npu_init import * +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import init_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import nn_ops +from tensorflow.python.ops import variable_scope as vs +from tensorflow.python.ops.rnn_cell import * +from tensorflow.python.util import nest + +_BIAS_VARIABLE_NAME = "bias" + +_WEIGHTS_VARIABLE_NAME = "kernel" + + +class _Linear_(object): + """Linear map: sum_i(args[i] * W[i]), where W[i] is a variable. + + + + Args: + + args: a 2D Tensor or a list of 2D, batch x n, Tensors. + + output_size: int, second dimension of weight variable. + + dtype: data type for variables. + + build_bias: boolean, whether to build a bias variable. + + bias_initializer: starting value to initialize the bias + + (default is all zeros). + + kernel_initializer: starting value to initialize the weight. + + + + Raises: + + ValueError: if inputs_shape is wrong. + + """ + + def __init__(self, + + args, + + output_size, + + build_bias, + + bias_initializer=None, + + kernel_initializer=None): + + self._build_bias = build_bias + + if args is None or (nest.is_sequence(args) and not args): + raise ValueError("`args` must be specified") + + if not nest.is_sequence(args): + + args = [args] + + self._is_sequence = False + + else: + + self._is_sequence = True + + # Calculate the total size of arguments on dimension 1. + + total_arg_size = 0 + + shapes = [a.get_shape() for a in args] + + for shape in shapes: + + if shape.ndims != 2: + raise ValueError( + "linear is expecting 2D arguments: %s" % shapes) + + if shape[1] is None: + + raise ValueError("linear expects shape[1] to be provided for shape %s, " + + "but saw %s" % (shape, shape[1])) + + else: + + total_arg_size += int(shape[1])#.value + + dtype = [a.dtype for a in args][0] + + scope = vs.get_variable_scope() + + with vs.variable_scope(scope) as outer_scope: + + self._weights = vs.get_variable( + + _WEIGHTS_VARIABLE_NAME, [total_arg_size, output_size], + + dtype=dtype, + + initializer=kernel_initializer) + + if build_bias: + + with vs.variable_scope(outer_scope) as inner_scope: + + inner_scope.set_partitioner(None) + + if bias_initializer is None: + bias_initializer = init_ops.constant_initializer( + 0.0, dtype=dtype) + + self._biases = vs.get_variable( + + _BIAS_VARIABLE_NAME, [output_size], + + dtype=dtype, + + initializer=bias_initializer) + + def __call__(self, args): + + if not self._is_sequence: + args = [args] + + if len(args) == 1: + + res = math_ops.matmul(args[0], self._weights) + + else: + + res = math_ops.matmul(array_ops.concat(args, 1), self._weights) + + if self._build_bias: + res = nn_ops.bias_add(res, self._biases) + + return res + + +try: + from tensorflow.python.ops.rnn_cell_impl import _Linear +except: + _Linear = _Linear_ + + +class QAAttGRUCell(RNNCell): + """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078). + + Args: + + num_units: int, The number of units in the GRU cell. + + activation: Nonlinearity to use. Default: `tanh`. + + reuse: (optional) Python boolean describing whether to reuse variables + + in an existing scope. If not `True`, and the existing scope already has + + the given variables, an error is raised. + + kernel_initializer: (optional) The initializer to use for the weight and + + projection matrices. + + bias_initializer: (optional) The initializer to use for the bias. + + """ + + def __init__(self, + + num_units, + + activation=None, + + reuse=None, + + kernel_initializer=None, + + bias_initializer=None): + + super(QAAttGRUCell, self).__init__(_reuse=reuse) + + self._num_units = num_units + + self._activation = activation or math_ops.tanh + + self._kernel_initializer = kernel_initializer + + self._bias_initializer = bias_initializer + + self._gate_linear = None + + self._candidate_linear = None + + @property + def state_size(self): + + return self._num_units + + @property + def output_size(self): + + return self._num_units + + def __call__(self, inputs, state, att_score): + + return self.call(inputs, state, att_score) + + def call(self, inputs, state, att_score=None): + """Gated recurrent unit (GRU) with nunits cells.""" + + if self._gate_linear is None: + + bias_ones = self._bias_initializer + + if self._bias_initializer is None: + bias_ones = init_ops.constant_initializer( + 1.0, dtype=inputs.dtype) + + with vs.variable_scope("gates"): # Reset gate and update gate. + + self._gate_linear = _Linear( + + [inputs, state], + + 2 * self._num_units, + + True, + + bias_initializer=bias_ones, + + kernel_initializer=self._kernel_initializer) + + value = math_ops.sigmoid(self._gate_linear([inputs, state])) + + r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1) + + r_state = r * state + + if self._candidate_linear is None: + with vs.variable_scope("candidate"): + self._candidate_linear = _Linear( + + [inputs, r_state], + + self._num_units, + + True, + + bias_initializer=self._bias_initializer, + + kernel_initializer=self._kernel_initializer) + + c = self._activation(self._candidate_linear([inputs, r_state])) + + new_h = (1. - att_score) * state + att_score * c + + return new_h, new_h + + +class VecAttGRUCell(RNNCell): + """Gated Recurrent Unit cell (cf. http://arxiv.org/abs/1406.1078). + + Args: + + num_units: int, The number of units in the GRU cell. + + activation: Nonlinearity to use. Default: `tanh`. + + reuse: (optional) Python boolean describing whether to reuse variables + + in an existing scope. If not `True`, and the existing scope already has + + the given variables, an error is raised. + + kernel_initializer: (optional) The initializer to use for the weight and + + projection matrices. + + bias_initializer: (optional) The initializer to use for the bias. + + """ + + def __init__(self, + + num_units, + + activation=None, + + reuse=None, + + kernel_initializer=None, + + bias_initializer=None): + + super(VecAttGRUCell, self).__init__(_reuse=reuse) + + self._num_units = num_units + + self._activation = activation or math_ops.tanh + + self._kernel_initializer = kernel_initializer + + self._bias_initializer = bias_initializer + + self._gate_linear = None + + self._candidate_linear = None + + @property + def state_size(self): + + return self._num_units + + @property + def output_size(self): + + return self._num_units + + def __call__(self, inputs, state, att_score): + + return self.call(inputs, state, att_score) + + def call(self, inputs, state, att_score=None): + """Gated recurrent unit (GRU) with nunits cells.""" + + if self._gate_linear is None: + + bias_ones = self._bias_initializer + + if self._bias_initializer is None: + bias_ones = init_ops.constant_initializer( + 1.0, dtype=inputs.dtype) + + with vs.variable_scope("gates"): # Reset gate and update gate. + + self._gate_linear = _Linear( + + [inputs, state], + + 2 * self._num_units, + + True, + + bias_initializer=bias_ones, + + kernel_initializer=self._kernel_initializer) + + value = math_ops.sigmoid(self._gate_linear([inputs, state])) + + r, u = array_ops.split(value=value, num_or_size_splits=2, axis=1) + + r_state = r * state + + if self._candidate_linear is None: + with vs.variable_scope("candidate"): + self._candidate_linear = _Linear( + + [inputs, r_state], + + self._num_units, + + True, + + bias_initializer=self._bias_initializer, + + kernel_initializer=self._kernel_initializer) + + c = self._activation(self._candidate_linear([inputs, r_state])) + + u = (1.0 - att_score) * u + + new_h = u * state + (1 - u) * c + + return new_h, new_h + -- Gitee From 860727546f7434f42748e759b71167b9b9a6df56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:35:44 +0000 Subject: [PATCH 14/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20=5F=5Fpycache=5F=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From 6058ae89160316d63198ae34420cc4619459560d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:36:28 +0000 Subject: [PATCH 15/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../contrib/__pycache__/__init__.cpython-37.pyc | Bin 0 -> 168 bytes .../contrib/__pycache__/rnn.cpython-37.pyc | Bin 0 -> 23488 bytes .../contrib/__pycache__/utils.cpython-37.pyc | Bin 0 -> 7296 bytes 3 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/__init__.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/rnn.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/utils.cpython-37.pyc diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/__init__.cpython-37.pyc b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a18a66a3e96dfc5972fc798a6432b26ce1350e23 GIT binary patch literal 168 zcmZ?b<>g`k0-N^W6cGIwM8E(ekl_Ht#VkM~g&~+hlhJP_LlHFfueS(oX@ZNG>VT oPtMOPDauUJkB`sH%PfhH*DI*J#bJ}1pHiBWY6r6FGY~TX0DrnCx&QzG literal 0 HcmV?d00001 diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/rnn.cpython-37.pyc b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/contrib/__pycache__/rnn.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..738c8bc1e99780ca8ace89ac61631e22ff9a5192 GIT binary patch literal 23488 zcmch9U2GgzmR?nLb+fzKBwG|k{aChZ{437Llr4L_9*refBiUomdbA!%9((A~G>ctD zvdLz5tEyTeiRR8^<$%;Cn)SXU*<_HA$O5~QAp5Whf&h8h$2q| z;YHIhJY&J|OwVeY8`gql^1Oia!a_mLZJgT+wwxE+#oN4JYL_;~7RKb-7|zQJWjQb7 zys}V{^GdtAF}^S^=T)577HV=nj`N9y2|2Ifd~#t@&L?m_wJ;^;lQ^$09Fy~@_VmW_ zh2wHwZ_g~8@Q!)YpBoFa{^Z)pg_B!Ve4ko4^@-7W$u|7c>($`Dna08y)7UQiv%K!r zz52qL&(3^m{KV*3pBmnAZwAjl<(=?m@qN}i>7By&)82F58Skmj&4p*Yv)-p_) zzWUBPOD(Sz_|2%*?YQlRx7BesTFpk#>EPb*ufV;puZNKvrH%9te))~iz1Qlj3^yAd zfjxi8MV>R`DR(LCw)=RlwbAPa5iXUz*1c8;U~oR!3tFA1vE1(nc(`JI6Ni!)ZT0*R zXN7K$U$sak)(BVKo{!%ZHwfIVhPqjAb~{nfZ8w%NZ+)%2;YO?ad;$P9ntr=Y7}|Kc zu6xUOGI`GpTJBOC6Kr;647=lp(Tyf(Q~oR*i}+l_C;TQ3kud-jKCtB5(%%Jqn*-y4 zxnl2_ku@+Mm{=)$`^vRdzquYd(W)CcuCvm*=Xad$(i)bV|F<1#!9Qw%eReT)@ek z728BWTm`DU7AYb8wGo`cEPnIhh1Kqce<8fT`Q}^SdE?@17gl=x3)g+WckTAgS2o-* z@`FaF*Kb^W_2TQVe*4uoUVH1}Yp-6s;Gsn`3N8rlTT2&!6z=5;Jln!>qc-svb@5;7?{h} zEU^0rW>na>2gZT9W8thgFrva*$+NxUiWQCRmpyA>9azhjSM*AcaJEwz6!t4ob$@(d zdiH5!t%mO+zgMgSbAJNAEYJMg)@k&3V4B9DfK?lFU%=wu4%|-I>xMqo&UcpFsJV*e z@|>u(;X57<9nx~>beFNNivQ-T4*qWXk=w!zH*_}ocnA&HgJLE}DA{Rsz_lP%^uTr) zCTwBRR0mgsmGF{ux_P?4IMcyPzoo zu-pp5$cgTEbI|MO-B#$QkN?dT>2VprE&pS(K7P|KpIxaachF_B30v^Dqw!7I1 z=O*LIiVxB!@5e>Z??xC`Gw1@+P;3*hxRTB|E`c(eZWN63ZV`7^{J0V({RCxR9#`vi zOZ9w@6OPB#bAAWHZqAO)&3JNC_zU7tfl%6P$Y6!dMkG9>4yW1U30%_j(hyo{kWPLN zx5BeH7~^GY#E*mPnQJ&iz)j>MTRHEV&d@T_G*Kn>gLh2ohdyyHP_31GL<6S`aK0 zP{J4Fr44rtRHZ0Kt&1>~VwBG|Twoi`&TEEn6f25AJnziU&)=DMu0nju^C9S|+i!b{ z=$hS)9$1WHWz)gVsF^g+GPhF}S5xjdA8kdeU{35({)!LqhL8YwT_=R7b(5{v>4L?A z`{1Lmz*1}qyi)vc>4^(ku^Y5HpvIP`_(FAcBX|mAu>I=SWwL;Tr#Pw@7X_Wcv%KW+ z@G=kNAkca>TcWrmh-?J&xD{6wDHA7$3I6HE61K^PzYIa9msb-$Ti?Ue;VcfucooE6 z1?kHN#9ubcc5o4Qnv@;Udq((XRPb!s8SGtS8PXlC%6nj$2iSwjF5#zWQDTcd!y7w|KC!zLZ($6Of7!ha z^8cW-j5Vf^6a_?Ny6nut6eDQb-|V$p%~sTq_LrRGcYDk6{3gf(vQT#wI5$$Za8!Sn zxaq$fLWX-D)B)MV=baz+AsUx_P?q8X%39ZGm@}1eAsXN zAP3a)@Ysqap12DLkc(g}fb9dA?uy^>yZum26C4X32_1i<7j0>>O+^~lnQMEC>pP9Z z3Rd_|RSC2|w(i=ulu81@Ule$uwpjD8#$$fl$2NOd*8ZmYcDofu0WNg0# zA%j^eok%Ur{M+p=xHPo`EYoJ{CjjR|Q&5aTko0oVl_N;qKlzr$x_Kb$_g7=Ir(1${=X zjIFO~%~-Zs#ujKpI+Sckh0AF1UXwzJe>3K?ua;}5KzTE~&TEpN;#qn`;XqI=~*VZAt*7*q}lJGH1d zs68;F;vU2ij)e#2Nn^bb{8d!)DilA1npahKsc`O04)6rqdR6&7wfRCcHkiZ}3;X>n z;H&Q(8`Ss9gJb)Z1MJwQareJ|Vr;&IR#WJmtxloU^v?0YG+G@`T76;OoyJ~otW^j0 zVA30ZWbI7y7v?!7zorJ0gQ;Z;;;-h$P>8O%?Pecqk3G3VMb+<#1ay*Zoz~_CWRC0x zLt=&o%sFMEoL6i!=k(lG;6O|%djqly+bBhALT8P(-&u)P^+x>3Dkb20tJg>$0mEK? z4+sCeJl6~TzSnKStfhnrVYEc6w3BFXC^Z?IbL3pP3{3(Wo2F(xN}@*xeuVwrIVV=r zFu)@Gq>HyX%iq!i#7SF>%5!t_;PcKch$&dH_k0L{)w9Dr6hZgCS_E{|NJikf>hv<77=jU9 z!2zIQyX=}i)JL*fpmOy?KG`Yipzbahvz$mcvjiXS>jUk{CYRMz+JXZv0%(c!N zYQ)*4oYtslx7o+83z;n#C1Yu99l|UOtxoj%MQ2fM<%`ZeP!tdfi%;~?&oVvij_zR; zr5^4@X{hH()zZ~4Ya|_wV4BBvS3`KuW<{~J)h0%};g zeWlodp&Yc-$kNAR4j)?VsQwl+D*@PZAtJ))Le%dmYa^sJuD4XhKu*Rgmz=h{vE;ef zG$|>_Z0UnJ+Eo;;Xu!EoJzk^H$&tRO_8AJE41OLTiwRh&HyP$@btX?}TqJ#E*Dg7? zA%_>~p_$LFF2Vy5cAG6Qsi(|4k=7JYX%~m(b4uw;-EJF3Ayh&x9aJica_z`sH#wo6 zA}SSVaG%UOfC*w9|NEi8+;59Jb1~d~PFim%8`#=0$5X-_LJZ?;+C9(@r{zoS0%?{*$@tyfQP=Ns&p?t{iNsV7(@g zdrq)S)Tr%S&?-iysHbXS7WLFM<{oQMe4t!nBt*?`u!Z%Ws35t&O3#}f%IgAvPB$Vn zIq1Zvs@N5j3nZ)Ih5R)q4cVAfEJcTZ8z%!}%@9Y571Rf?1oxoJuNUD}wDt?MFaM|x zh0unF(W3Y901lKN88;31?_tKwSvL|bnr5=Hs>g5^Pc*xi@#1Ap+b-h4E8`oj`=t{MDo){^i0!8wQ5b;_*mO7ekFu8(XITENJ!v9BsXya z`-2t~oc6Sp_KIK;WkHE^<{__wbYP*uK@U5rv}do=I)fWu*=n%M_=}br2#V}AXK#P! zWb{9P4x`usA&!8kFF&Po;noS^_J8($~8OS{3RvZz%d9s$iUd zfUn>OJY={lzpZqDE8hgh{s-q570M(wTn8{p$giu`y&*fWvkbK|m7YDqpKlw6nKM>xaMb|cR0?tOk3}9<*>h`gq6mI{wYf|S5fj@ zNL>KUmXLE(vy^F&0F_{?q-*$BxE8*JgYgC_Y_e9LojGfl6N;%=6X4pi3}CeWhfW%6>&c^7NZ=?3_8Lc zU5F5lY&7X5_~KnOj#5^&ocZsHH0VO#lY(?3Xl?GG0%Q8dp(2AziN5}vu5nP5ZX<%a`DYj=7wCow?iXUS>78i0hQ zVTOo8FCEQ0&}N^bkN-KU+hKwu7^X}+*n)y9Sk26gbUHK&XrhiZG^qN$qMwEHNF!5# z%JeMCM=Eb2Ka_rOtiv%a{!1vaKQ^%TmZ+>Igj6Poa4z2ISMi9FnyKjgP33L6_BVUM zm7i0bt0A_ZZD?31m2x{>(l*tj*uH=d89A)uAgW(^$6PUvD0~Wv*lZkvLG-)7m5dJG zzVn#TQ?I_#$4#Y*Gw_wdLs*!JAn;JQ3OsbLGKT;*0F_bCc$E6^ft!!fEch`F3C=#{ zxl&I9;!F!WP-(w?I6fi3|C1DWYwlQV6VAAttY%yk@(o_#FrV-s;-BhDJkjs0cVK*~ zn6vB#GO4%SHfw7_vKd4%ND zBeGsXb3_Kr8uCSEp`zOOu9}lnSnWF0i5atsD<#|sw$ZZ5U6g;^;~47~@!dgB%*%OT zhV=+yT;hV-qT8ts%Hb6)#;YQ(tC6k5_&|O`96}uTu6rej=7Kkd8066R)iYT^a%zrnqcP2Wg2F36{VwAds4vhWdgBrxTwKJhI&=5Wl z2){7lcA42f0hfMZP+NgjwnFO_wnn5NI<6ps@E=naY-Fv?g66B<3<3jYbdvTPLKr1% zF&6qLdM6&0!5vdO^+EXyGyEwm;#1M-!4$@x8cYr93>;`%cnn*|ta|~WqH6)9Hmr0P z;x2J|cKm(-t*YZA=OkFqBqL4rBQQBDl$PbtmgN(L*n}Jl7u;%nT$nk2^^8hKn zD0CR=C*Iw3A!4;7WpQz_Cqa9Wk^Pxj=UN}ZAU`m?E zO?^e_VE{7)P>eolH?(U?Xzwy6gH$*dp5zo>Y5G;59SNhJt@g2|=*oHL&LxnY$1Erz zpenkV7dGAMMxP8#^YYNIuiVGzse>2CIjUpOOdMPcRKuB-?BQH#Mv9m=hTzB& z+@?6!T^uHF7auS*_U(Qp_jaL$n9Ic=mBz)>E z8R!XmLYha3GUe*7fJ$BsT1#ZIdA7Ypi~Y7ZQ5iInkQV1DPkaT6W82HaDEi98qNK;= zuSKL!Ca}EnOUP7?6Zc+()N?DFxY88_$`el#QNNSB4wcD8s(vuBnHF{VvLm6;MduvE zg^PusV`K@7u#u*VEGjL1F_l9}(34Cz94pCW5;LdZvc?c;j)_`l5aS^quuj9!0b6!@ z6v*2_-$!>}53&rJyxja7K_*Wr zx0dW8A}~4#8)A_$6P5XqqYxS&Ab_UJ2)WP?jWe~AI!Lk&T5*Ux; zrjty479<_qfbAzl5S~XyQ`7H7BJ4O9c9%EVBwObGYKsxIT!+$(62e{gJ98P#x6#Fu zAjQ^)qyvFeGN3Y&4xDuTQm~Z`IAR@T=8{U6T9*h54TVgEx)Q(=^)^WsKrTa1rOVGr zsv?837Z93#^Nl1TOTq6Ab$KTjwELK19&14t>??`mQiQ22GB@v6AmMbHK*+dRN4YUw zm7%g~X)@cQ(2Z$P}U469u$THX|hiL`zMIA}E>a(B^vS$V77r zNV?__Ps!7Y>$yt;f$zkqKXW|Z7;V?$PQ^>d{B_h{f@S@0mc%2T7mB%ur z&!O9rqUFI((+LR#-fMYifWEndDEeYz zo;#eHm(Et8Oh)LrP|DjriXxa&C?x@yz;(5`Tymg0YH644@L5X|cK|1StCMz~^e0a- zXh)H$`UQPcKzLn&TN1&U@fyLc*rjEl(qE8!ww3lqS=a(+@ zT_&_K1!NC|8^l@QNUFGms#V)u2b9`7 z=en8GE+>N^kKkTU5<{`KruvQ2gB1T`4%kNZPLvvYA|MvETDa)w9XlarkxPhrs8lg5 ztKK-m#Z*6%&MC4-Ya5`!X!VeEL3+DRK6`SHd95Yu{YZ}=OIgp5RtExrzNC!ECE!;? zIdZ9yQ_OUsCBNyCNO8x#*KK*(GNn}p;C7@HL9{ISE3J;KCwT{+o&OTaK zjTU~Zbt4()x+Jx3Br7OYSC_z^stW5CwobLFac;_lhvn2?S_$0V>UR?`K*e3>Ian0W zseXH&P;?hBQkPMCF~^Hv8Jg^71zLO63T7s*l;qEO;z$E8uzBK6AQ zJtm_X+0YaOhw^t6ji)Tb@ku>;oBN7`W&`Dk_DO@@=tfj8lf5#B*2 zY#|#oT5nOsg{dMXa99~EbH~Z}JDZN$v-JjzRw=D9)QJVvp-yJmC~Y7Q;rYBRs%6IT zW}kK^$`cB`BoC8hMZ!}ONwEXKJB+84HiWJe1xY+i5gJ(Su{sL(&Pc6Y3PLjJR8$>V zBN->53+WxA0gR~3Y@sVPONHJ;v@xRJJI=QrtJLKC&$RE6s|*f5w6eLO)A{9euxve* z)|w(-U_?3?5JzldYEW($Q{RH}=Ey_^7mJBDfPG0(O5XXP9=w%08Wv@POg#-+W(=h# z-G(x%i{6s86FTP(C^{~N9c|HzW|2Lr40kR=^I2pG+xqdlD2)JbyZY!Zx;_84X1ea{AIidxW)aNTt*E z`-!Ebt+@=liSY&_SH(TlZY2CPq7+fW&&bfRQ80}z$%1vBNm@$js^qPx zD$Od)9jLnvsGM;T=8xMBcF=Opmg*T*iZ+JSer_Fcjj0i+evw~?kc$L4oeu+ZNVT|5 zda9u}1A%{r|LaJS+kR;z+K)UOk4yRXE&uL^aA)M>g2niFVwmI%sdzl6<;PW2)3B-> zAuyHE@H!`TCY~6om5wW^=*i-v>aEjeti4)8aLnl_}07eWD~(`==%*tmGLx-VTZ3tJzg-)UPXA!h|d<4^{Iu{ z!W5B6_+0iC=7HpVHo@!c72UkCWc5@3XFn>y!(f|Tg~(= zgf^)Q1{Zj}j4FNg^EI}W)R0}?7*o4`7_e0KFg1+lK>afa(U2isV+j?ANEWdRlZYpl z@ee5^x-=KbA>~<#I`VqSW_?o4Do<7mh(;p4at6`I!+*7MsZg_me~g}+WH$aW-`@!wJMC3lZXT;VhC#Br>^YuN59nGRt%i=^PJBa%QiUz@TMTv+tEWsZ2-F zQiW#^r=rRpDj87#Rh1GaW3QyrQ65(Iq{aUD0m^&55(|n{I>_dydle*`e8fZ#B#;z& zXTNq}gujX=yfIytgllV)I}>P8ma|C|JxxgI(u6Ku8XrvJ_b>1aB9(t&-u=ZVEVU}` zP9TME+^ezJDykx(&-5l(dNnBPG?}T$UYo)^ODH{>d}KYW@728GdIeF-b5VW&*j^3u z9>>|VoK>T-wd0-@&8(g9ro8&2!cIM!-9PCaLqxNRVyI&eEyOX4D*A{NLKHE1)828G zeqnS&!!tt&BE~GsNn#G7tJBh=geOiU_lZlDxC5My0jIOLV?<~6#&AcnUU}Vfc5RlB z?N)ZHyW_jH-O1hhaz)t#flb=!h!3#gtZRD%;5qbE1th|Ua7Vs3!&&P!P3 zQ#+@lr&*@9Vhv97ogu(mZE$M;nE|kC8edp=Pd( zIdIHbk*Z7aWYCtpuH&$&JMga?@)*v?Q9H{P>>|x zrqPKZ`NFp+B=#KVq6Qtd!ScGb4A{!tNJ{h1S{(K>GNp+I@&6ZRm%6_$_eJc}c6m~*P%Nd`RX4Ndh0xy2Pw@>=wK zs@A!5siZ^f)m)Pu^ z9`tK%_#kg|&iax{0F&<|v3Hu$DS3@Z18>EU*P;A2i|F5?1C@nmp|m5X(unqzv(6H` zNWU=R(+|WMC#ebu9>dK^{!`}ZTsl2^?$pa?oDqo2tRwM!t5+FlbL%!kf{fdV=Fe3i zV#PZ{cQllFxsb?bj4Z#^>fwhDdChNM{vPQ?B)&3pzD`f`>AKJyqa}76pYS;xP^Q9E zFjOf-=00A}gu))IO=PRgd$DPd6!rf~#APEqG(2o;d;SSaH{$VhS|m4;5XTKO3!6nWhb(MSe-mnY z2hCB2@~8MxYB4kSgC-9Dg@6eWTX~ZvxXvdmrIFM8QYtMQ@{iy2K?rtdaH3KtSvEC5 zR=#IGVhx9N16h)Sp?``d@iFCjXs8!%(VWHr%(qV7z;l(a^TIV5kIylaehGo?hS=W9 z91t4ICJ|Q?bVcs}$pdz0NkCGh@xaP6 z^mavi8ejBo0K-BfNg`_t)BiR4Q8*AYC8ZKU{{KmlZx2C1)IJ6Z)>=U;2O$qZQ5HsO zC?FK-R8H?dLsvklfI;Ia!<;}+gzVDGpF#kjj*2vuAxp7_cWNafE%9*#8yrO!Y*~>j zS+Q!Q;R#oA!{Jo^Xt^?F^Ef^stD+(p6HuH@)&lJnwMk=co{LN@z?dkyX8?}9Qykdv zTi_Kho=rnWa@(JSa$8tqjS*h{0u#f8*TP2X<}dioc6<7Jxd0Q2RY*493$cv6d~XbI zawv7``bxnot1Ej|>EYk-Ee?Cty~|ea#qthgLsMhg^c&yo5ScUr3RDKmJhwID0{@w8d)Q98_^}_LnJD52(2^=e=bT^ zL4?ds{W=i=Kr3?T&Yr&8CZD0tu5>NC5(Oi>68$;@=}<6-6S<8UE4BX|=LF~VYtTj1 ztMLf9s#{itDu{+6&Y3UN&bm%1UgRIhPk4?Ls}ko)?R(huYhl%}EO62oCK7fe1&G3buZg16^1kmy@`?1KQfUp4xL(*Pasj_0f=U} zkj%!r5)=26^2LlPfe{_ryu=~-B6?L0$=~_9U~5<95R8;30wbLuk1uuOZWjCn4~HZr zOWo!YkZ!|;c%6-|@ol7^DTDr!BB9*JuzOcqR?_aGm-xYYsM;#aM z2S|5Fa{al~co$_K@2?!S-oFbNb%Dg(nDW`Zk0(?i7hkX=J~JsdlKL+_lk}~QBwaNH zaga$h#ylUB8NI`^@8f{S#G9r?KBVnGz}2{}bh*Y#H%ZJFyaUa7T8EgsP-8Fz0V(p-#?Ge~~{|afrv@NSErBI$jkeiTQ8} z!%XzOz+t18dH9qE?&xvi%11?McJ6NCy`*^A10rZqKz%DNieEuFo2yjy)Z14NSHw_! z#8pYb5o0Gf;6N|pFn2~+LKq?V6JAZ7$eFWQv@l&Qtv3s5@Z-JzkvB!z=q5x9F}$p=j>KJYnz-o6ACQ zSD8DeRJcFo)j#7w%ny;7uk)wqc7Mq;N`v5E@^DB1h;$SwAlxGolT07{Jsf@?#2EfQ zPKM)DoIVUW0=6iGH8~l6kJ(v*qo_h{7#Wx(3w%yqlKqahzXft{BTu zg#Lu^)T=;G9BqrY3oBM2`3YGE+-t>fkq}Gp1&5wJ`eZ+9wS}n=$dH&TK?wes4>Kg7 z$oKE^;mW&`Uki4OO>!QJ`O;r w$$xwg#L<7{Lb+BlEwgIY&Dm1<(Ee3p(mX_XKERxThOgqR_R)3t2VM>u9dIlk!IHlJXb4TXFSgfPZ%%wGmnj0X{U_Vi{Wgm zTq|qr4r@cw&o9-=AF{Uo0Yi-$UP6tMe+V_^@IK4ScrUB>!~D=KR+;-8V`mjD(N03- zJR!Utx6_q)v4eIH4fU26Z4Gp*-5zMAhn@($jnH>xz0>tk$87sDO3aPGlZkcT7i~YR zQwPJ#fN~kPd=bKDHI1{H&b6Asv)th36Rl?Q47Z-JT88Two}Jk4&7kdj!u85JW2x@6 zyza6ido4FO;faR)D7fpKbnf6$Ipu6``NCH(P9Po68IEx=kITWef*Q-I>Nr$=8Kusy zI_rL0c7%gJ&j|w=Ih{=;pK><5sJ`Vqa@wZ`MI@S{h8(BUi@LqYmBFsR>;&!Tlq3Cm zr_CK6w9s16X;UNHe$d#8CI^z%=Fv_U^|%*#4yn#2`W_B<+{s2S2)RopU3NA)ozV9% z63jBfM4hPPsCN{p(HhDfO{tz_VZ7^Z;TB2Pp?yMeM{&!-l~*$x_24Gm}LI48Kj z>Gi@$Su+eBw9%23QTyp4f5RS9i;Pxe%?qSIUX$B2u4_VcFxX%dql1m5E4MtD2ur#x zI_<`w|7x9rkggfr6slKo%eOSfVk2fQi{WASZTUi^?dh?$uZd%^CXYo1*B)!1Xv0)_ zA=VmNU*kIJm~D$wU6Y%AEA_@$TV;>6e)iFNoLyyiZ}#ojZf1D)z9AyCWyjeCB7F&^(rajhPV}0vE?G8ekJU;D%n@3qc7iIVCeb&$8 znTzxL+OI{w5M`VBs1RGR-q7P*oZ-c%CPrC|jo)bEf+|@+$&3ABT-=-CY+ut@Y|>be z|3Xx1&hnW!$4gK3zNy+Oqa_V3nS+)NVI=H5w#GgHoD4P25QVeV-6QuXQtmme9s*;- zcVySE2b+P<-&hwgHmj4M3qi_E+_!Mcd58v!+1+Aai?uzr&wi^vFmJKn5#r<;K0=39 zwW25b;^{>5L=hE~_@cVB)oJ-la{JM%=U+c}_RLbF+gn=o{qEKEwO3l6jC|q3G2FAK z&;I1}PfwpabN=j^(`T1BO4K817YcfQD9pqQ?ATHT9p)6i&~7TY~O zgqKBE`LOFQUI;sNFO(Od3~PK&rFV~o3W>-E{YRL;3L*dpEYm}{M~Khk@^do8?xjJ| zw48L*G(S3XSH-LEbVcW3zdHIUuoI-PA(L5&hPs4e5g^tVf*dZH$C?A%+R-E!C5PQBH&&i)Dy!hs zmz9~slzzCGnB?6_VFJ&S{CL($`B@ZA<__XsQi?i}7rJOl?FfmbR#qk%^-OJ@Q9GsT z2@X7X3EG&%z#!90hDi_H@)V{hS0Px@(ib#a&+C?Mo0e(nCFprg*YbK%H}yGlUZ2x# zrQ5o$$`w@!6D3U4()Ha#2i(3+E>0JD8-c0<(i)VQHJL1?02#sW9&22?uZvTWzGnbl z^`lh6+}Hbhdx6rT%$|jGZ69EorsJ6vw<8;1MrnB}8`Jh~RK;QRpb(@|5J#aWM(DRi zhBOPLqty$ALq%v54qdm8 z5|V;~>6T`}So7urw9>R`(y|r3TCG@Ax9zoj*G=-S+v@Nhc#jL0cOUe;aFCOAUEZm? zt~i61#cL$S3#v2~hy`R&fJzD{`v+#8>G|t|QW3}5!P?aEI=aZi>5?}grijzbr)*PG zW;s%6$ryuH^~blj3_TD^YZP?ft8M z7*b$8-Z$mDfId}Ti z#WSa0J3r(GE0a8{4YJllE5Jy_lh(mAkhTE|1BRq>kfG=Y2BP{JQ49}A`_Q9MUv{dU zb_%UQv_oLplm4RPtfmxY*|~@K-?}%dDSYHFJ106_s_lgp=iQxX3#@Fwqa5zbI<^U7 zLlcZjRU1($IpRVXpw|ZWV_TpaDtgiTjZLTBiJW`uqUYZ`g(6(dW4fHX2p;>8YSKOi ziy+oSQ1>Tj=K%_K*M;SNj;2-oAx=kbmm-Qu?tu+$>`~Ys z4jMH%r1poYeWd6%5EO3ed?-+d83R!FVN%6X(39jwT_*X7ZZc|+;v|i&0zv!-SQL6v z0xibKK)CUH+>$nNR-6M|7je^X_qoY99r2YBRUqO)_f_1Ij18zI_NKu~;xgXxQ0O>D zBm$!(k4+QPphL_b@vg`LGj^Y!>TO)+KkZNa7@hoSqN5{I9ZhV(^&#ldlXe6(Gcruf z_Q+S@H~Yq5hKWgFT`|*Du+l>VT!Q9?b8Q5wS?>sc0*)0D6GZ<7cRK5XHcoed(MSlQ zbGX$={sDz-jP1}C_n>Kgcx#ikM_ZhX^rnIHgw`~6W4&+nvve+)XE>Lz zW=Zw`XWE82ogI!1`m=FvKyT;^W5hk|tu*cpJOILVD7u~HV-sQM}i@)_|W3F5fo1rkIy z#RU?VNzfvuxR8}Wqtyv`$}%#leenimUx5HKQE~GQB~(fL0wT#N>H+>K#e4LwiA3uq zS&UKnoK{aVZGYR{de-G=2mdQ>c@BaZ#N+assTC3EI&|V}y1t;#;|G?f6*WpVk!B83 zcMnZPOzQVlTomdBaUQLT*GVjs_!$X;mBa?z10bUvK_?&;hbcv&5hXJNIHGNS2uXw# zSg5zCxC$4-*3WQDS}*%K)53WVzf!rFDQ0X_VQ(E-qiYz#{|(+UbK9?fPh7qyF8`av zMO;LXAY3ouAzTs^q$x7dW+zri5dNeBF^h$%NczsGg~FL->Jw)QK;7<<>3+UrQgLabr=!z7eOp!`2pR{ZLQLcTv#?G~ zVs$hEefNM({}q+MhyMQ`DvlLHv?3MgyH92?S<28t+$-1j?^g=99zp{dPkHyx}jBN=Mh^xk&vR zW;5(;bsMNlS(; literal 0 HcmV?d00001 -- Gitee From 4ce7fee6ee039b116d7a125e4bab19b9defe0d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:36:47 +0000 Subject: [PATCH 16/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20estimator?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DIN_ID0190_for_TensorFlow/deepctr/estimator/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From 10e9348af36603e5aefa3c1a33c3b3190f04e7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:37:05 +0000 Subject: [PATCH 17/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../deepctr/estimator/__init__.py | 1 + .../deepctr/estimator/feature_column.py | 54 +++++ .../deepctr/estimator/inputs.py | 54 +++++ .../deepctr/estimator/utils.py | 218 ++++++++++++++++++ 4 files changed, 327 insertions(+) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/__init__.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/feature_column.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/inputs.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/utils.py diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/__init__.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/__init__.py new file mode 100644 index 000000000..aed4fa323 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/__init__.py @@ -0,0 +1 @@ +from .models import * diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/feature_column.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/feature_column.py new file mode 100644 index 000000000..2432c617c --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/feature_column.py @@ -0,0 +1,54 @@ +from npu_bridge.npu_init import * +import tensorflow as tf +from tensorflow.python.feature_column.feature_column import _EmbeddingColumn + +from .utils import LINEAR_SCOPE_NAME, variable_scope, get_collection, get_GraphKeys, input_layer, get_losses + + +def linear_model(features, linear_feature_columns): + if tf.__version__ >= '2.0.0': + linear_logits = tf.compat.v1.feature_column.linear_model(features, linear_feature_columns) + else: + linear_logits = tf.feature_column.linear_model(features, linear_feature_columns) + return linear_logits + + +def get_linear_logit(features, linear_feature_columns, l2_reg_linear=0): + with variable_scope(LINEAR_SCOPE_NAME): + if not linear_feature_columns: + linear_logits = tf.Variable([[0.0]], name='bias_weights') + else: + + linear_logits = linear_model(features, linear_feature_columns) + + if l2_reg_linear > 0: + for var in get_collection(get_GraphKeys().TRAINABLE_VARIABLES, LINEAR_SCOPE_NAME)[:-1]: + get_losses().add_loss(l2_reg_linear * tf.nn.l2_loss(var, name=var.name.split(":")[0] + "_l2loss"), + get_GraphKeys().REGULARIZATION_LOSSES) + return linear_logits + + +def input_from_feature_columns(features, feature_columns, l2_reg_embedding=0.0): + dense_value_list = [] + sparse_emb_list = [] + for feat in feature_columns: + if is_embedding(feat): + sparse_emb = tf.expand_dims(input_layer(features, [feat]), axis=1) + sparse_emb_list.append(sparse_emb) + if l2_reg_embedding > 0: + get_losses().add_loss(l2_reg_embedding * tf.nn.l2_loss(sparse_emb, name=feat.name + "_l2loss"), + get_GraphKeys().REGULARIZATION_LOSSES) + + else: + dense_value_list.append(input_layer(features, [feat])) + + return sparse_emb_list, dense_value_list + + +def is_embedding(feature_column): + try: + from tensorflow.python.feature_column.feature_column_v2 import EmbeddingColumn + except ImportError: + EmbeddingColumn = _EmbeddingColumn + return isinstance(feature_column, (_EmbeddingColumn, EmbeddingColumn)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/inputs.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/inputs.py new file mode 100644 index 000000000..f2ec76db7 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/inputs.py @@ -0,0 +1,54 @@ +from npu_bridge.npu_init import * +import tensorflow as tf + + +def input_fn_pandas(df, features, label=None, batch_size=256, num_epochs=1, shuffle=False, queue_capacity_factor=10, + num_threads=1): + if label is not None: + y = df[label] + else: + y = None + if tf.__version__ >= "2.0.0": + return tf.compat.v1.estimator.inputs.pandas_input_fn(df[features], y, batch_size=batch_size, + num_epochs=num_epochs, + shuffle=shuffle, + queue_capacity=batch_size * queue_capacity_factor, + num_threads=num_threads) + + return tf.estimator.inputs.pandas_input_fn(df[features], y, batch_size=batch_size, num_epochs=num_epochs, + shuffle=shuffle, queue_capacity=batch_size * queue_capacity_factor, + num_threads=num_threads) + + +def input_fn_tfrecord(filenames, feature_description, label=None, batch_size=256, num_epochs=1, num_parallel_calls=8, + shuffle_factor=10, prefetch_factor=1, + ): + def _parse_examples(serial_exmp): + try: + features = tf.parse_single_example(serial_exmp, features=feature_description) + except AttributeError: + features = tf.io.parse_single_example(serial_exmp, features=feature_description) + if label is not None: + labels = features.pop(label) + return features, labels + return features + + def input_fn(): + dataset = tf.data.TFRecordDataset(filenames) + dataset = dataset.map(_parse_examples, num_parallel_calls=num_parallel_calls) + if shuffle_factor > 0: + dataset = dataset.shuffle(buffer_size=batch_size * shuffle_factor) + + dataset = dataset.repeat(num_epochs).batch(batch_size, drop_remainder=True) + + if prefetch_factor > 0: + dataset = dataset.prefetch(buffer_size=batch_size * prefetch_factor) + try: + iterator = dataset.make_one_shot_iterator() + except AttributeError: + iterator = tf.compat.v1.data.make_one_shot_iterator(dataset) + + return iterator.get_next() + + return input_fn + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/utils.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/utils.py new file mode 100644 index 000000000..3de97026f --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/utils.py @@ -0,0 +1,218 @@ +from npu_bridge.npu_init import * +import tensorflow as tf +from tensorflow.python.estimator.canned.head import _Head +from tensorflow.python.estimator.canned.optimizers import get_optimizer_instance + +LINEAR_SCOPE_NAME = 'linear' +DNN_SCOPE_NAME = 'dnn' + + +def _summary_key(head_name, val): + return '%s/%s' % (val, head_name) if head_name else val + + +class Head(_Head): + + def __init__(self, task, + name=None): + self._task = task + self._name = name + + @property + def name(self): + return self._name + + @property + def logits_dimension(self): + return 1 + + def _eval_metric_ops(self, + labels, + logits, + predictions, + unweighted_loss, + weights=None): + + labels = to_float(labels) + predictions = to_float(predictions) + + # with name_scope(None, 'metrics', (labels, logits, predictions, + # unweighted_loss, weights)): + metrics = get_metrics() + losses = get_losses() + + metric_ops = { + _summary_key(self._name, "prediction/mean"): metrics.mean(predictions, weights=weights), + _summary_key(self._name, "label/mean"): metrics.mean(labels, weights=weights), + } + + summary_scalar("prediction/mean", metric_ops[_summary_key(self._name, "prediction/mean")][1]) + summary_scalar("label/mean", metric_ops[_summary_key(self._name, "label/mean")][1]) + + + mean_loss = losses.compute_weighted_loss( + unweighted_loss, weights=1.0, reduction=losses.Reduction.MEAN) + + if self._task == "binary": + metric_ops[_summary_key(self._name, "LogLoss")] = metrics.mean(mean_loss, weights=weights, ) + summary_scalar("LogLoss", mean_loss) + + metric_ops[_summary_key(self._name, "AUC")] = metrics.auc(labels, predictions, weights=weights) + summary_scalar("AUC", metric_ops[_summary_key(self._name, "AUC")][1]) + else: + + metric_ops[_summary_key(self._name, "MSE")] = metrics.mean_squared_error(labels, predictions, + weights=weights) + summary_scalar("MSE", mean_loss) + + metric_ops[_summary_key(self._name, "MAE")] = metrics.mean_absolute_error(labels, predictions, + weights=weights) + summary_scalar("MAE", metric_ops[_summary_key(self._name, "MAE")][1]) + + return metric_ops + + def create_loss(self, features, mode, logits, labels): + del mode, features # Unused for this head. + losses = get_losses() + if self._task == "binary": + loss = losses.sigmoid_cross_entropy(labels, logits, reduction=losses.Reduction.NONE) + else: + loss = losses.mean_squared_error(labels, logits, reduction=losses.Reduction.NONE) + return loss + + def create_estimator_spec( + self, features, mode, logits, labels=None, train_op_fn=None, training_chief_hooks=None): + # with name_scope('head'): + logits = tf.reshape(logits, [-1, 1]) + if self._task == 'binary': + pred = tf.sigmoid(logits) + else: + pred = logits + + predictions = {"pred": pred, "logits": logits} + export_outputs = {"predict": tf.estimator.export.PredictOutput(predictions)} + if mode == tf.estimator.ModeKeys.PREDICT: + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + export_outputs=export_outputs) + + labels = tf.reshape(labels, [-1, 1]) + + unweighted_loss = self.create_loss(features, mode, logits, labels) + + losses = get_losses() + loss = losses.compute_weighted_loss( + unweighted_loss, weights=1.0, reduction=losses.Reduction.SUM) + reg_loss = losses.get_regularization_loss() + + training_loss = loss + reg_loss + + eval_metric_ops = self._eval_metric_ops(labels, logits, pred, unweighted_loss) + + return tf.estimator.EstimatorSpec( + mode=mode, + predictions=predictions, + loss=training_loss, + train_op=train_op_fn(training_loss), + eval_metric_ops=eval_metric_ops, + training_chief_hooks=training_chief_hooks) + + +def deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, training_chief_hooks): + linear_optimizer = get_optimizer_instance(linear_optimizer, 0.005) + dnn_optimizer = get_optimizer_instance(dnn_optimizer, 0.01) + train_op_fn = get_train_op_fn(linear_optimizer, dnn_optimizer) + + head = Head(task) + return head.create_estimator_spec(features=features, + mode=mode, + labels=labels, + train_op_fn=train_op_fn, + logits=logits, training_chief_hooks=training_chief_hooks) + + +def get_train_op_fn(linear_optimizer, dnn_optimizer): + def _train_op_fn(loss): + train_ops = [] + try: + global_step = tf.train.get_global_step() + except AttributeError: + global_step = tf.compat.v1.train.get_global_step() + linear_var_list = get_collection(get_GraphKeys().TRAINABLE_VARIABLES, LINEAR_SCOPE_NAME) + dnn_var_list = get_collection(get_GraphKeys().TRAINABLE_VARIABLES, DNN_SCOPE_NAME) + + if len(dnn_var_list) > 0: + train_ops.append( + dnn_optimizer.minimize( + loss, + var_list=dnn_var_list)) + if len(linear_var_list) > 0: + train_ops.append( + linear_optimizer.minimize( + loss, + var_list=linear_var_list)) + + train_op = tf.group(*train_ops) + with tf.control_dependencies([train_op]): + try: + return tf.assign_add(global_step, 1).op + except AttributeError: + return tf.compat.v1.assign_add(global_step, 1).op + + return _train_op_fn + + +def variable_scope(name_or_scope): + try: + return tf.variable_scope(name_or_scope) + except AttributeError: + return tf.compat.v1.variable_scope(name_or_scope) + +def get_collection(key, scope=None): + try: + return tf.get_collection(key, scope=scope) + except AttributeError: + return tf.compat.v1.get_collection(key, scope=scope) + + +def get_GraphKeys(): + try: + return tf.GraphKeys + except AttributeError: + return tf.compat.v1.GraphKeys + + +def get_losses(): + try: + return tf.compat.v1.losses + except AttributeError: + return tf.losses + + +def input_layer(features, feature_columns): + try: + return tf.feature_column.input_layer(features, feature_columns) + except AttributeError: + return tf.compat.v1.feature_column.input_layer(features, feature_columns) + + +def get_metrics(): + try: + return tf.compat.v1.metrics + except AttributeError: + return tf.metrics + + +def to_float(x, name="ToFloat"): + try: + return tf.to_float(x, name) + except AttributeError: + return tf.compat.v1.to_float(x, name) + + +def summary_scalar(name, data): + try: + tf.summary.scalar(name, data) + except AttributeError: # tf version 2.5.0+:AttributeError: module 'tensorflow._api.v2.summary' has no attribute 'scalar' + tf.compat.v1.summary.scalar(name, data) -- Gitee From af8b99e0248e334c40f54afce485d5ed2135a696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:37:23 +0000 Subject: [PATCH 18/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DIN_ID0190_for_TensorFlow/deepctr/estimator/models/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From 620d006c10a62dd2fc19fd0b483be0d017300315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:37:41 +0000 Subject: [PATCH 19/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../deepctr/estimator/models/__init__.py | 14 +++ .../deepctr/estimator/models/afm.py | 72 ++++++++++++++ .../deepctr/estimator/models/autoint.py | 97 +++++++++++++++++++ .../deepctr/estimator/models/ccpm.py | 94 ++++++++++++++++++ .../deepctr/estimator/models/dcn.py | 89 +++++++++++++++++ .../deepctr/estimator/models/deepfefm.py | 91 +++++++++++++++++ .../deepctr/estimator/models/deepfm.py | 78 +++++++++++++++ .../deepctr/estimator/models/fibinet.py | 82 ++++++++++++++++ .../deepctr/estimator/models/fnn.py | 67 +++++++++++++ .../deepctr/estimator/models/fwfm.py | 86 ++++++++++++++++ .../deepctr/estimator/models/nfm.py | 76 +++++++++++++++ .../deepctr/estimator/models/pnn.py | 95 ++++++++++++++++++ .../deepctr/estimator/models/wdl.py | 68 +++++++++++++ .../deepctr/estimator/models/xdeepfm.py | 86 ++++++++++++++++ 14 files changed, 1095 insertions(+) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/__init__.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/afm.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/autoint.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/ccpm.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/dcn.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/deepfefm.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/deepfm.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fibinet.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fnn.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fwfm.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/nfm.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/pnn.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/wdl.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/xdeepfm.py diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/__init__.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/__init__.py new file mode 100644 index 000000000..db3bde9ad --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/__init__.py @@ -0,0 +1,14 @@ +from .afm import AFMEstimator +from .autoint import AutoIntEstimator +from .ccpm import CCPMEstimator +from .dcn import DCNEstimator +from .deepfm import DeepFMEstimator +from .fwfm import FwFMEstimator +from .fibinet import FiBiNETEstimator +from .fnn import FNNEstimator +from .nfm import NFMEstimator +from .pnn import PNNEstimator +from .wdl import WDLEstimator +from .xdeepfm import xDeepFMEstimator +from .deepfefm import DeepFEFMEstimator + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/afm.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/afm.py new file mode 100644 index 000000000..c27a1fa0a --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/afm.py @@ -0,0 +1,72 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Xiao J, Ye H, He X, et al. Attentional factorization machines: Learning the weight of feature interactions via attention networks[J]. arXiv preprint arXiv:1708.04617, 2017. + (https://arxiv.org/abs/1708.04617) + +""" +from npu_bridge.npu_init import * +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.interaction import AFMLayer, FM +from ...layers.utils import concat_func + + +def AFMEstimator(linear_feature_columns, dnn_feature_columns, use_attention=True, attention_factor=8, + l2_reg_linear=1e-5, l2_reg_embedding=1e-5, l2_reg_att=1e-5, afm_dropout=0, seed=1024, + task='binary', model_dir=None, config=None, linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the Attentional Factorization Machine architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param use_attention: bool,whether use attention or not,if set to ``False``.it is the same as **standard Factorization Machine** + :param attention_factor: positive integer,units in attention net + :param l2_reg_linear: float. L2 regularizer strength applied to linear part + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_att: float. L2 regularizer strength applied to attention net + :param afm_dropout: float in [0,1), Fraction of the attention net output units to dropout. + :param seed: integer ,to use as random seed. + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, _ = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + if use_attention: + + fm_logit = AFMLayer(attention_factor, l2_reg_att, afm_dropout, + seed)(sparse_embedding_list, training=train_flag) + else: + fm_logit = FM()(concat_func(sparse_embedding_list, axis=1)) + + logits = linear_logits + fm_logit + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/autoint.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/autoint.py new file mode 100644 index 000000000..e2c8599cf --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/autoint.py @@ -0,0 +1,97 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Song W, Shi C, Xiao Z, et al. AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks[J]. arXiv preprint arXiv:1810.11921, 2018.(https://arxiv.org/abs/1810.11921) + +""" +from npu_bridge.npu_init import * + +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.core import DNN +from ...layers.interaction import InteractingLayer +from ...layers.utils import concat_func, combined_dnn_input + + +def AutoIntEstimator(linear_feature_columns, dnn_feature_columns, att_layer_num=3, att_embedding_size=8, att_head_num=2, + att_res=True, + dnn_hidden_units=(256, 128, 64), dnn_activation='relu', l2_reg_linear=1e-5, + l2_reg_embedding=1e-5, l2_reg_dnn=0, dnn_use_bn=False, dnn_dropout=0, seed=1024, + task='binary', model_dir=None, config=None, linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the AutoInt Network architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param att_layer_num: int.The InteractingLayer number to be used. + :param att_embedding_size: int.The embedding size in multi-head self-attention network. + :param att_head_num: int.The head number in multi-head self-attention network. + :param att_res: bool.Whether or not use standard residual connections before output. + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN + :param dnn_activation: Activation function to use in DNN + :param l2_reg_linear: float. L2 regularizer strength applied to linear part + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_dnn: float. L2 regularizer strength applied to DNN + :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in DNN + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param seed: integer ,to use as random seed. + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + att_input = concat_func(sparse_embedding_list, axis=1) + + for _ in range(att_layer_num): + att_input = InteractingLayer( + att_embedding_size, att_head_num, att_res)(att_input) + att_output = tf.keras.layers.Flatten()(att_input) + + dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list) + + if len(dnn_hidden_units) > 0 and att_layer_num > 0: # Deep & Interacting Layer + deep_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(dnn_input, training=train_flag) + stack_out = tf.keras.layers.Concatenate()([att_output, deep_out]) + final_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(stack_out) + elif len(dnn_hidden_units) > 0: # Only Deep + deep_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(dnn_input, training=train_flag) + final_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(deep_out) + elif att_layer_num > 0: # Only Interacting Layer + final_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(att_output) + else: # Error + raise NotImplementedError + + logits = linear_logits + final_logit + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/ccpm.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/ccpm.py new file mode 100644 index 000000000..24855ff13 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/ccpm.py @@ -0,0 +1,94 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Liu Q, Yu F, Wu S, et al. A convolutional click prediction model[C]//Proceedings of the 24th ACM International on Conference on Information and Knowledge Management. ACM, 2015: 1743-1746. + (http://ir.ia.ac.cn/bitstream/173211/12337/1/A%20Convolutional%20Click%20Prediction%20Model.pdf) + +""" +from npu_bridge.npu_init import * +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.core import DNN +from ...layers.sequence import KMaxPooling +from ...layers.utils import concat_func + + +def CCPMEstimator(linear_feature_columns, dnn_feature_columns, conv_kernel_width=(6, 5), conv_filters=(4, 4), + dnn_hidden_units=(128, 64), l2_reg_linear=1e-5, l2_reg_embedding=1e-5, l2_reg_dnn=0, dnn_dropout=0, + seed=1024, task='binary', model_dir=None, config=None, linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the Convolutional Click Prediction Model architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param conv_kernel_width: list,list of positive integer or empty list,the width of filter in each conv layer. + :param conv_filters: list,list of positive integer or empty list,the number of filters in each conv layer. + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN. + :param l2_reg_linear: float. L2 regularizer strength applied to linear part + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_dnn: float. L2 regularizer strength applied to DNN + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param init_std: float,to use as the initialize std of embedding vector + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + if len(conv_kernel_width) != len(conv_filters): + raise ValueError( + "conv_kernel_width must have same element with conv_filters") + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, _ = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + n = len(sparse_embedding_list) + l = len(conv_filters) + + conv_input = concat_func(sparse_embedding_list, axis=1) + pooling_result = tf.keras.layers.Lambda( + lambda x: tf.expand_dims(x, axis=3))(conv_input) + + for i in range(1, l + 1): + filters = conv_filters[i - 1] + width = conv_kernel_width[i - 1] + k = max(1, int((1 - pow(i / l, l - i)) * n)) if i < l else 3 + + conv_result = tf.keras.layers.Conv2D(filters=filters, kernel_size=(width, 1), strides=(1, 1), + padding='same', + activation='tanh', use_bias=True, )(pooling_result) + pooling_result = KMaxPooling( + k=min(k, int(conv_result.shape[1])), axis=1)(conv_result) + + flatten_result = tf.keras.layers.Flatten()(pooling_result) + dnn_out = DNN(dnn_hidden_units, l2_reg=l2_reg_dnn, dropout_rate=dnn_dropout, seed=seed)(flatten_result, training=train_flag) + dnn_logit = tf.keras.layers.Dense(1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_out) + + logits = linear_logits + dnn_logit + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks + ) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/dcn.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/dcn.py new file mode 100644 index 000000000..a922bf545 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/dcn.py @@ -0,0 +1,89 @@ +# -*- coding:utf-8 -*- +""" +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Wang R, Fu B, Fu G, et al. Deep & cross network for ad click predictions[C]//Proceedings of the ADKDD'17. ACM, 2017: 12. (https://arxiv.org/abs/1708.05123) +""" +from npu_bridge.npu_init import * +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.core import DNN +from ...layers.interaction import CrossNet +from ...layers.utils import combined_dnn_input + + +def DCNEstimator(linear_feature_columns, dnn_feature_columns, cross_num=2, dnn_hidden_units=(256, 128, 64), + l2_reg_linear=1e-5, + l2_reg_embedding=1e-5, + l2_reg_cross=1e-5, l2_reg_dnn=0, seed=1024, dnn_dropout=0, dnn_use_bn=False, + dnn_activation='relu', task='binary', model_dir=None, config=None, linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the Deep&Cross Network architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param cross_num: positive integet,cross layer number + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_cross: float. L2 regularizer strength applied to cross net + :param l2_reg_dnn: float. L2 regularizer strength applied to DNN + :param seed: integer ,to use as random seed. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not DNN + :param dnn_activation: Activation function to use in DNN + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + if len(dnn_hidden_units) == 0 and cross_num == 0: + raise ValueError("Either hidden_layer or cross layer must > 0") + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + + dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list) + + if len(dnn_hidden_units) > 0 and cross_num > 0: # Deep & Cross + deep_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(dnn_input, training=train_flag) + cross_out = CrossNet(cross_num, l2_reg=l2_reg_cross)(dnn_input) + stack_out = tf.keras.layers.Concatenate()([cross_out, deep_out]) + final_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(stack_out) + elif len(dnn_hidden_units) > 0: # Only Deep + deep_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(dnn_input, training=train_flag) + final_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(deep_out) + elif cross_num > 0: # Only Cross + cross_out = CrossNet(cross_num, l2_reg=l2_reg_cross)(dnn_input) + final_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(cross_out) + else: # Error + raise NotImplementedError + + logits = linear_logits + final_logit + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/deepfefm.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/deepfefm.py new file mode 100644 index 000000000..d8fa168d6 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/deepfefm.py @@ -0,0 +1,91 @@ +# -*- coding:utf-8 -*- +""" +Author: + Harshit Pande + +Reference: + [1] Field-Embedded Factorization Machines for Click-through Rate Prediction] + (https://arxiv.org/abs/2009.09931) + +""" +from npu_bridge.npu_init import * + +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import DNN_SCOPE_NAME, deepctr_model_fn, variable_scope +from ...layers.core import DNN +from ...layers.interaction import FEFMLayer +from ...layers.utils import concat_func, add_func, combined_dnn_input, reduce_sum + + +def DeepFEFMEstimator(linear_feature_columns, dnn_feature_columns, + dnn_hidden_units=(256, 128, 64), l2_reg_linear=0.00001, l2_reg_embedding_feat=0.00001, + l2_reg_embedding_field=0.00001, l2_reg_dnn=0, seed=1024, dnn_dropout=0.0, + dnn_activation='relu', dnn_use_bn=False, task='binary', model_dir=None, + config=None, linear_optimizer='Ftrl', dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the DeepFEFM Network architecture or the shallow FEFM architecture (Ablation support not provided + as estimator is meant for production, Ablation support provided in DeepFEFM implementation in models + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN + :param l2_reg_linear: float. L2 regularizer strength applied to linear part + :param l2_reg_embedding_feat: float. L2 regularizer strength applied to embedding vector of features + :param l2_reg_embedding_field: float, L2 regularizer to field embeddings + :param l2_reg_dnn: float. L2 regularizer strength applied to DNN + :param seed: integer ,to use as random seed. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_activation: Activation function to use in DNN + :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in DNN + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + final_logit_components = [linear_logits] + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding_feat) + + fefm_interaction_embedding = FEFMLayer( + regularizer=l2_reg_embedding_field)(concat_func(sparse_embedding_list, axis=1)) + + fefm_logit = tf.keras.layers.Lambda(lambda x: reduce_sum(x, axis=1, keep_dims=True))( + fefm_interaction_embedding) + + final_logit_components.append(fefm_logit) + + if dnn_hidden_units: + dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list) + dnn_input = concat_func([dnn_input, fefm_interaction_embedding], axis=1) + + dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)( + dnn_input, training=train_flag) + + dnn_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_output) + + final_logit_components.append(dnn_logit) + + logits = add_func(final_logit_components) + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/deepfm.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/deepfm.py new file mode 100644 index 000000000..6702e4304 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/deepfm.py @@ -0,0 +1,78 @@ +# -*- coding:utf-8 -*- +""" +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Guo H, Tang R, Ye Y, et al. Deepfm: a factorization-machine based neural network for ctr prediction[J]. arXiv preprint arXiv:1703.04247, 2017.(https://arxiv.org/abs/1703.04247) + +""" +from npu_bridge.npu_init import * + +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.core import DNN +from ...layers.interaction import FM +from ...layers.utils import concat_func, combined_dnn_input + + +def DeepFMEstimator(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 128, 64), + l2_reg_linear=0.00001, l2_reg_embedding=0.00001, l2_reg_dnn=0, seed=1024, dnn_dropout=0, + dnn_activation='relu', dnn_use_bn=False, task='binary', model_dir=None, config=None, + linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the DeepFM Network architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param fm_group: list, group_name of features that will be used to do feature interactions. + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN + :param l2_reg_linear: float. L2 regularizer strength applied to linear part + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_dnn: float. L2 regularizer strength applied to DNN + :param seed: integer ,to use as random seed. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_activation: Activation function to use in DNN + :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in DNN + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + + dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list) + + fm_logit = FM()(concat_func(sparse_embedding_list, axis=1)) + + dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(dnn_input, training=train_flag) + dnn_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed=seed))(dnn_output) + + logits = linear_logits + fm_logit + dnn_logit + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks + =training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fibinet.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fibinet.py new file mode 100644 index 000000000..80e376ca7 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fibinet.py @@ -0,0 +1,82 @@ +# -*- coding:utf-8 -*- +""" +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Huang T, Zhang Z, Zhang J. FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction[J]. arXiv preprint arXiv:1905.09433, 2019. +""" +from npu_bridge.npu_init import * + +import tensorflow as tf +from tensorflow.python.keras.layers import Dense, Flatten + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.core import DNN +from ...layers.interaction import SENETLayer, BilinearInteraction +from ...layers.utils import concat_func, combined_dnn_input + + +def FiBiNETEstimator(linear_feature_columns, dnn_feature_columns, bilinear_type='interaction', reduction_ratio=3, + dnn_hidden_units=(256, 128, 64), l2_reg_linear=1e-5, + l2_reg_embedding=1e-5, l2_reg_dnn=0, seed=1024, dnn_dropout=0, dnn_activation='relu', + task='binary', model_dir=None, config=None, linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the Feature Importance and Bilinear feature Interaction NETwork architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param bilinear_type: str,bilinear function type used in Bilinear Interaction Layer,can be ``'all'`` , ``'each'`` or ``'interaction'`` + :param reduction_ratio: integer in [1,inf), reduction ratio used in SENET Layer + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN + :param l2_reg_linear: float. L2 regularizer strength applied to wide part + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_dnn: float. L2 regularizer strength applied to DNN + :param seed: integer ,to use as random seed. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_activation: Activation function to use in DNN + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + + senet_embedding_list = SENETLayer( + reduction_ratio, seed)(sparse_embedding_list) + + senet_bilinear_out = BilinearInteraction( + bilinear_type=bilinear_type, seed=seed)(senet_embedding_list) + bilinear_out = BilinearInteraction( + bilinear_type=bilinear_type, seed=seed)(sparse_embedding_list) + + dnn_input = combined_dnn_input( + [Flatten()(concat_func([senet_bilinear_out, bilinear_out]))], dense_value_list) + dnn_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, False, seed=seed)(dnn_input, training=train_flag) + dnn_logit = Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_out) + + logits = linear_logits + dnn_logit + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fnn.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fnn.py new file mode 100644 index 000000000..949c2e295 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fnn.py @@ -0,0 +1,67 @@ +# -*- coding:utf-8 -*- +""" +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Zhang W, Du T, Wang J. Deep learning over multi-field categorical data[C]//European conference on information retrieval. Springer, Cham, 2016: 45-57.(https://arxiv.org/pdf/1601.02376.pdf) +""" +from npu_bridge.npu_init import * +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.core import DNN +from ...layers.utils import combined_dnn_input + + +def FNNEstimator(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 128, 64), + l2_reg_embedding=1e-5, l2_reg_linear=1e-5, l2_reg_dnn=0, seed=1024, dnn_dropout=0, + dnn_activation='relu', task='binary', model_dir=None, config=None, linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the Factorization-supported Neural Network architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_linear: float. L2 regularizer strength applied to linear weight + :param l2_reg_dnn: float . L2 regularizer strength applied to DNN + :param seed: integer ,to use as random seed. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_activation: Activation function to use in DNN + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list) + deep_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, False, seed=seed)(dnn_input, training=train_flag) + dnn_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(deep_out) + + logits = linear_logits + dnn_logit + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fwfm.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fwfm.py new file mode 100644 index 000000000..886e3410a --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/fwfm.py @@ -0,0 +1,86 @@ +# -*- coding:utf-8 -*- +""" +Author: + Weichen Shen, weichenswc@163.com + Harshit Pande + +Reference: + [1] Field-weighted Factorization Machines for Click-Through Rate Prediction in Display Advertising + (https://arxiv.org/pdf/1806.03514.pdf) + +""" +from npu_bridge.npu_init import * + +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import DNN_SCOPE_NAME, deepctr_model_fn, variable_scope +from ...layers.core import DNN +from ...layers.interaction import FwFMLayer +from ...layers.utils import concat_func, add_func, combined_dnn_input + + +def FwFMEstimator(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 128, 64), + l2_reg_linear=0.00001, l2_reg_embedding=0.00001, l2_reg_field_strength=0.00001, l2_reg_dnn=0, + seed=1024, dnn_dropout=0, dnn_activation='relu', dnn_use_bn=False, task='binary', model_dir=None, + config=None, linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the DeepFwFM Network architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param fm_group: list, group_name of features that will be used to do feature interactions. + :param dnn_hidden_units: list,list of positive integer or empty list if do not want DNN, the layer number and units + in each layer of DNN + :param l2_reg_linear: float. L2 regularizer strength applied to linear part + :param l2_reg_field_strength: float. L2 regularizer strength applied to the field pair strength parameters + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_dnn: float. L2 regularizer strength applied to DNN + :param seed: integer ,to use as random seed. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_activation: Activation function to use in DNN + :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in DNN + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + final_logit_components = [linear_logits] + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + + fwfm_logit = FwFMLayer(num_fields=len(sparse_embedding_list), regularizer=l2_reg_field_strength)( + concat_func(sparse_embedding_list, axis=1)) + + final_logit_components.append(fwfm_logit) + + if dnn_hidden_units: + dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list) + + dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(dnn_input, training=train_flag) + dnn_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_output) + final_logit_components.append(dnn_logit) + + logits = add_func(final_logit_components) + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/nfm.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/nfm.py new file mode 100644 index 000000000..fb6c8c09d --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/nfm.py @@ -0,0 +1,76 @@ +# -*- coding:utf-8 -*- +""" +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] He X, Chua T S. Neural factorization machines for sparse predictive analytics[C]//Proceedings of the 40th International ACM SIGIR conference on Research and Development in Information Retrieval. ACM, 2017: 355-364. (https://arxiv.org/abs/1708.05027) +""" +from npu_bridge.npu_init import * +from npu_bridge.estimator.npu import npu_convert_dropout +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.core import DNN +from ...layers.interaction import BiInteractionPooling +from ...layers.utils import concat_func, combined_dnn_input + + +def NFMEstimator(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 128, 64), + l2_reg_embedding=1e-5, l2_reg_linear=1e-5, l2_reg_dnn=0, seed=1024, bi_dropout=0, + dnn_dropout=0, dnn_activation='relu', task='binary', model_dir=None, config=None, + linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the Neural Factorization Machine architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_linear: float. L2 regularizer strength applied to linear part. + :param l2_reg_dnn: float . L2 regularizer strength applied to DNN + :param seed: integer ,to use as random seed. + :param biout_dropout: When not ``None``, the probability we will drop out the output of BiInteractionPooling Layer. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_activation: Activation function to use in deep net + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + + fm_input = concat_func(sparse_embedding_list, axis=1) + bi_out = BiInteractionPooling()(fm_input) + if bi_dropout: + bi_out = tf.keras.layers.Dropout(bi_dropout)(bi_out, training=None) + dnn_input = combined_dnn_input([bi_out], dense_value_list) + dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, False, seed=seed)(dnn_input, training=train_flag) + dnn_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_output) + + logits = linear_logits + dnn_logit + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/pnn.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/pnn.py new file mode 100644 index 000000000..5476590dd --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/pnn.py @@ -0,0 +1,95 @@ +# -*- coding:utf-8 -*- +""" +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Qu Y, Cai H, Ren K, et al. Product-based neural networks for user response prediction[C]//Data Mining (ICDM), 2016 IEEE 16th International Conference on. IEEE, 2016: 1149-1154.(https://arxiv.org/pdf/1611.00144.pdf) +""" +from npu_bridge.npu_init import * + +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.core import DNN +from ...layers.interaction import InnerProductLayer, OutterProductLayer +from ...layers.utils import concat_func, combined_dnn_input + + +def PNNEstimator(dnn_feature_columns, dnn_hidden_units=(256, 128, 64), l2_reg_embedding=1e-5, l2_reg_dnn=0, + seed=1024, dnn_dropout=0, dnn_activation='relu', use_inner=True, use_outter=False, kernel_type='mat', + task='binary', model_dir=None, config=None, + linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the Product-based Neural Network architecture. + + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net + :param l2_reg_embedding: float . L2 regularizer strength applied to embedding vector + :param l2_reg_dnn: float. L2 regularizer strength applied to DNN + :param seed: integer ,to use as random seed. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_activation: Activation function to use in DNN + :param use_inner: bool,whether use inner-product or not. + :param use_outter: bool,whether use outter-product or not. + :param kernel_type: str,kernel_type used in outter-product,can be ``'mat'`` , ``'vec'`` or ``'num'`` + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + if kernel_type not in ['mat', 'vec', 'num']: + raise ValueError("kernel_type must be mat,vec or num") + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, [], l2_reg_linear=0) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + + inner_product = tf.keras.layers.Flatten()( + InnerProductLayer()(sparse_embedding_list)) + outter_product = OutterProductLayer(kernel_type)(sparse_embedding_list) + + # ipnn deep input + linear_signal = tf.keras.layers.Reshape( + [sum(map(lambda x: int(x.shape[-1]), sparse_embedding_list))])(concat_func(sparse_embedding_list)) + + if use_inner and use_outter: + deep_input = tf.keras.layers.Concatenate()( + [linear_signal, inner_product, outter_product]) + elif use_inner: + deep_input = tf.keras.layers.Concatenate()( + [linear_signal, inner_product]) + elif use_outter: + deep_input = tf.keras.layers.Concatenate()( + [linear_signal, outter_product]) + else: + deep_input = linear_signal + + dnn_input = combined_dnn_input([deep_input], dense_value_list) + dnn_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, False, seed=seed)(dnn_input, training=train_flag) + dnn_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_out) + + logits = linear_logits + dnn_logit + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/wdl.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/wdl.py new file mode 100644 index 000000000..5aff24173 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/wdl.py @@ -0,0 +1,68 @@ +# -*- coding:utf-8 -*- +""" +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Cheng H T, Koc L, Harmsen J, et al. Wide & deep learning for recommender systems[C]//Proceedings of the 1st Workshop on Deep Learning for Recommender Systems. ACM, 2016: 7-10.(https://arxiv.org/pdf/1606.07792.pdf) +""" +from npu_bridge.npu_init import * + +import tensorflow as tf +from tensorflow.python.keras.layers import Dense + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers import DNN, combined_dnn_input + + +def WDLEstimator(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 128, 64), l2_reg_linear=1e-5, + l2_reg_embedding=1e-5, l2_reg_dnn=0, seed=1024, dnn_dropout=0, dnn_activation='relu', + task='binary', model_dir=None, config=None, linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the Wide&Deep Learning architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of DNN + :param l2_reg_linear: float. L2 regularizer strength applied to wide part + :param l2_reg_embedding: float. L2 regularizer strength applied to embedding vector + :param l2_reg_dnn: float. L2 regularizer strength applied to DNN + :param seed: integer ,to use as random seed. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_activation: Activation function to use in DNN + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list) + dnn_out = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, False, seed=seed)(dnn_input, training=train_flag) + dnn_logits = Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_out) + + logits = linear_logits + dnn_logits + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/xdeepfm.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/xdeepfm.py new file mode 100644 index 000000000..fcbe5b37b --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/estimator/models/xdeepfm.py @@ -0,0 +1,86 @@ +# -*- coding:utf-8 -*- +""" +Author: + Weichen Shen, weichenswc@163.com + +Reference: + [1] Lian J, Zhou X, Zhang F, et al. xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems[J]. arXiv preprint arXiv:1803.05170, 2018.(https://arxiv.org/pdf/1803.05170.pdf) +""" +from npu_bridge.npu_init import * +import tensorflow as tf + +from ..feature_column import get_linear_logit, input_from_feature_columns +from ..utils import deepctr_model_fn, DNN_SCOPE_NAME, variable_scope +from ...layers.core import DNN +from ...layers.interaction import CIN +from ...layers.utils import concat_func, add_func, combined_dnn_input + + +def xDeepFMEstimator(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 128, 64), + cin_layer_size=(128, 128,), cin_split_half=True, cin_activation='relu', l2_reg_linear=0.00001, + l2_reg_embedding=0.00001, l2_reg_dnn=0, l2_reg_cin=0, seed=1024, dnn_dropout=0, + dnn_activation='relu', dnn_use_bn=False, task='binary', model_dir=None, config=None, + linear_optimizer='Ftrl', + dnn_optimizer='Adagrad', training_chief_hooks=None): + """Instantiates the xDeepFM architecture. + + :param linear_feature_columns: An iterable containing all the features used by linear part of the model. + :param dnn_feature_columns: An iterable containing all the features used by deep part of the model. + :param dnn_hidden_units: list,list of positive integer or empty list, the layer number and units in each layer of deep net + :param cin_layer_size: list,list of positive integer or empty list, the feature maps in each hidden layer of Compressed Interaction Network + :param cin_split_half: bool.if set to True, half of the feature maps in each hidden will connect to output unit + :param cin_activation: activation function used on feature maps + :param l2_reg_linear: float. L2 regularizer strength applied to linear part + :param l2_reg_embedding: L2 regularizer strength applied to embedding vector + :param l2_reg_dnn: L2 regularizer strength applied to deep net + :param l2_reg_cin: L2 regularizer strength applied to CIN. + :param seed: integer ,to use as random seed. + :param dnn_dropout: float in [0,1), the probability we will drop out a given DNN coordinate. + :param dnn_activation: Activation function to use in DNN + :param dnn_use_bn: bool. Whether use BatchNormalization before activation or not in DNN + :param task: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + :param model_dir: Directory to save model parameters, graph and etc. This can + also be used to load checkpoints from the directory into a estimator + to continue training a previously saved model. + :param config: tf.RunConfig object to configure the runtime settings. + :param linear_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the linear part of the model. Defaults to FTRL optimizer. + :param dnn_optimizer: An instance of `tf.Optimizer` used to apply gradients to + the deep part of the model. Defaults to Adagrad optimizer. + :param training_chief_hooks: Iterable of `tf.train.SessionRunHook` objects to + run on the chief worker during training. + :return: A Tensorflow Estimator instance. + + """ + + def _model_fn(features, labels, mode, config): + train_flag = (mode == tf.estimator.ModeKeys.TRAIN) + + linear_logits = get_linear_logit(features, linear_feature_columns, l2_reg_linear=l2_reg_linear) + logits_list = [linear_logits] + + with variable_scope(DNN_SCOPE_NAME): + sparse_embedding_list, dense_value_list = input_from_feature_columns(features, dnn_feature_columns, + l2_reg_embedding=l2_reg_embedding) + fm_input = concat_func(sparse_embedding_list, axis=1) + + dnn_input = combined_dnn_input(sparse_embedding_list, dense_value_list) + dnn_output = DNN(dnn_hidden_units, dnn_activation, l2_reg_dnn, dnn_dropout, dnn_use_bn, seed=seed)(dnn_input, training=train_flag) + dnn_logit = tf.keras.layers.Dense( + 1, use_bias=False, kernel_initializer=tf.keras.initializers.glorot_normal(seed))(dnn_output) + + logits_list.append(dnn_logit) + + if len(cin_layer_size) > 0: + exFM_out = CIN(cin_layer_size, cin_activation, + cin_split_half, l2_reg_cin, seed)(fm_input, training=train_flag) + exFM_logit = tf.keras.layers.Dense(1, kernel_initializer=tf.keras.initializers.glorot_normal(seed) )(exFM_out) + logits_list.append(exFM_logit) + + logits = add_func(logits_list) + + return deepctr_model_fn(features, mode, logits, labels, task, linear_optimizer, dnn_optimizer, + training_chief_hooks=training_chief_hooks) + + return tf.estimator.Estimator(_model_fn, model_dir=model_dir, config=npu_run_config_init(run_config=config)) + -- Gitee From 89da473eced24f99b179b1d47ed474287cba8a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:37:58 +0000 Subject: [PATCH 20/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20layers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From cd1b0dd441d8039d028f44e200d69ac51a3071af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:38:14 +0000 Subject: [PATCH 21/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../deepctr/layers/__init__.py | 55 + .../deepctr/layers/activation.py | 87 + .../deepctr/layers/core.py | 324 ++++ .../deepctr/layers/interaction.py | 1561 +++++++++++++++++ .../deepctr/layers/normalization.py | 53 + .../deepctr/layers/sequence.py | 870 +++++++++ .../deepctr/layers/utils.py | 348 ++++ 7 files changed, 3298 insertions(+) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__init__.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/activation.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/core.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/interaction.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/normalization.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/sequence.py create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/utils.py diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__init__.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__init__.py new file mode 100644 index 000000000..3c14ec67e --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__init__.py @@ -0,0 +1,55 @@ +import tensorflow as tf + +from .activation import Dice +from .core import DNN, LocalActivationUnit, PredictionLayer, RegulationModule +from .interaction import (CIN, FM, AFMLayer, BiInteractionPooling, CrossNet, CrossNetMix, + InnerProductLayer, InteractingLayer, + OutterProductLayer, FGCNNLayer, SENETLayer, BilinearInteraction, + FieldWiseBiInteraction, FwFMLayer, FEFMLayer, BridgeModule) +from .normalization import LayerNormalization +from .sequence import (AttentionSequencePoolingLayer, BiasEncoding, BiLSTM, + KMaxPooling, SequencePoolingLayer, WeightedSequenceLayer, + Transformer, DynamicGRU, PositionEncoding) +from .utils import NoMask, Hash, Linear, _Add, combined_dnn_input, softmax, reduce_sum, Concat + +custom_objects = {'tf': tf, + 'InnerProductLayer': InnerProductLayer, + 'OutterProductLayer': OutterProductLayer, + 'DNN': DNN, + 'PredictionLayer': PredictionLayer, + 'FM': FM, + 'AFMLayer': AFMLayer, + 'CrossNet': CrossNet, + 'CrossNetMix': CrossNetMix, + 'BiInteractionPooling': BiInteractionPooling, + 'LocalActivationUnit': LocalActivationUnit, + 'Dice': Dice, + 'SequencePoolingLayer': SequencePoolingLayer, + 'AttentionSequencePoolingLayer': AttentionSequencePoolingLayer, + 'CIN': CIN, + 'InteractingLayer': InteractingLayer, + 'LayerNormalization': LayerNormalization, + 'BiLSTM': BiLSTM, + 'Transformer': Transformer, + 'NoMask': NoMask, + 'BiasEncoding': BiasEncoding, + 'KMaxPooling': KMaxPooling, + 'FGCNNLayer': FGCNNLayer, + 'Hash': Hash, + 'Linear': Linear, + 'Concat': Concat, + 'DynamicGRU': DynamicGRU, + 'SENETLayer': SENETLayer, + 'BilinearInteraction': BilinearInteraction, + 'WeightedSequenceLayer': WeightedSequenceLayer, + '_Add': _Add, + 'FieldWiseBiInteraction': FieldWiseBiInteraction, + 'FwFMLayer': FwFMLayer, + 'softmax': softmax, + 'FEFMLayer': FEFMLayer, + 'reduce_sum': reduce_sum, + 'PositionEncoding': PositionEncoding, + 'RegulationModule': RegulationModule, + 'BridgeModule': BridgeModule + } + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/activation.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/activation.py new file mode 100644 index 000000000..91a0a58f6 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/activation.py @@ -0,0 +1,87 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen,weichenswc@163.com + +""" +from npu_bridge.npu_init import * + +import tensorflow as tf + +try: + from tensorflow.python.ops.init_ops import Zeros +except ImportError: + from tensorflow.python.ops.init_ops_v2 import Zeros +from tensorflow.python.keras.layers import Layer, Activation + +try: + from tensorflow.python.keras.layers import BatchNormalization +except ImportError: + BatchNormalization = tf.keras.layers.BatchNormalization + +try: + unicode +except NameError: + unicode = str + + +class Dice(Layer): + """The Data Adaptive Activation Function in DIN,which can be viewed as a generalization of PReLu and can adaptively adjust the rectified point according to distribution of input data. + + Input shape + - Arbitrary. Use the keyword argument `input_shape` (tuple of integers, does not include the samples axis) when using this layer as the first layer in a model. + + Output shape + - Same shape as the input. + + Arguments + - **axis** : Integer, the axis that should be used to compute data distribution (typically the features axis). + + - **epsilon** : Small float added to variance to avoid dividing by zero. + + References + - [Zhou G, Zhu X, Song C, et al. Deep interest network for click-through rate prediction[C]//Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2018: 1059-1068.](https://arxiv.org/pdf/1706.06978.pdf) + """ + + def __init__(self, axis=-1, epsilon=1e-9, **kwargs): + self.axis = axis + self.epsilon = epsilon + super(Dice, self).__init__(**kwargs) + + def build(self, input_shape): + self.bn = BatchNormalization( + axis=self.axis, epsilon=self.epsilon, center=False, scale=False) + self.alphas = self.add_weight(shape=(input_shape[-1],), initializer=Zeros( + ), dtype=tf.float32, name='dice_alpha') # name='alpha_'+self.name + super(Dice, self).build(input_shape) # Be sure to call this somewhere! + self.uses_learning_phase = True + + def call(self, inputs, training=None, **kwargs): + inputs_normed = self.bn(inputs, training=training) + # tf.layers.batch_normalization( + # inputs, axis=self.axis, epsilon=self.epsilon, center=False, scale=False) + x_p = tf.sigmoid(inputs_normed) + return self.alphas * (1.0 - x_p) * inputs + x_p * inputs + + def compute_output_shape(self, input_shape): + return input_shape + + def get_config(self, ): + config = {'axis': self.axis, 'epsilon': self.epsilon} + base_config = super(Dice, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +def activation_layer(activation): + if activation in ("dice", "Dice"): + act_layer = Dice() + elif isinstance(activation, (str, unicode)): + act_layer = Activation(activation) + elif issubclass(activation, Layer): + act_layer = activation() + else: + raise ValueError( + "Invalid activation,found %s.You should use a str or a Activation Layer Class." % (activation)) + return act_layer + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/core.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/core.py new file mode 100644 index 000000000..139480514 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/core.py @@ -0,0 +1,324 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen,weichenswc@163.com + +""" +from npu_bridge.npu_init import * + +import tensorflow as tf +from tensorflow.python.keras import backend as K + +try: + from tensorflow.python.ops.init_ops_v2 import Zeros, Ones, glorot_normal +except ImportError: + from tensorflow.python.ops.init_ops import Zeros, Ones, glorot_normal_initializer as glorot_normal + +from tensorflow.python.keras.layers import Layer, Dropout + +try: + from tensorflow.python.keras.layers import BatchNormalization +except ImportError: + BatchNormalization = tf.keras.layers.BatchNormalization +from tensorflow.python.keras.regularizers import l2 + +from .activation import activation_layer + + +class LocalActivationUnit(Layer): + """The LocalActivationUnit used in DIN with which the representation of + user interests varies adaptively given different candidate items. + + Input shape + - A list of two 3D tensor with shape: ``(batch_size, 1, embedding_size)`` and ``(batch_size, T, embedding_size)`` + + Output shape + - 3D tensor with shape: ``(batch_size, T, 1)``. + + Arguments + - **hidden_units**:list of positive integer, the attention net layer number and units in each layer. + + - **activation**: Activation function to use in attention net. + + - **l2_reg**: float between 0 and 1. L2 regularizer strength applied to the kernel weights matrix of attention net. + + - **dropout_rate**: float in [0,1). Fraction of the units to dropout in attention net. + + - **use_bn**: bool. Whether use BatchNormalization before activation or not in attention net. + + - **seed**: A Python integer to use as random seed. + + References + - [Zhou G, Zhu X, Song C, et al. Deep interest network for click-through rate prediction[C]//Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2018: 1059-1068.](https://arxiv.org/pdf/1706.06978.pdf) + """ + + def __init__(self, hidden_units=(64, 32), activation='sigmoid', l2_reg=0, dropout_rate=0, use_bn=False, seed=1024, + **kwargs): + self.hidden_units = hidden_units + self.activation = activation + self.l2_reg = l2_reg + self.dropout_rate = dropout_rate + self.use_bn = use_bn + self.seed = seed + super(LocalActivationUnit, self).__init__(**kwargs) + self.supports_masking = True + + def build(self, input_shape): + + if not isinstance(input_shape, list) or len(input_shape) != 2: + raise ValueError('A `LocalActivationUnit` layer should be called ' + 'on a list of 2 inputs') + + if len(input_shape[0]) != 3 or len(input_shape[1]) != 3: + raise ValueError("Unexpected inputs dimensions %d and %d, expect to be 3 dimensions" % ( + len(input_shape[0]), len(input_shape[1]))) + + if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1: + raise ValueError('A `LocalActivationUnit` layer requires ' + 'inputs of a two inputs with shape (None,1,embedding_size) and (None,T,embedding_size)' + 'Got different shapes: %s,%s' % (input_shape[0], input_shape[1])) + size = 4 * \ + int(input_shape[0][-1] + ) if len(self.hidden_units) == 0 else self.hidden_units[-1] + self.kernel = self.add_weight(shape=(size, 1), + initializer=glorot_normal( + seed=self.seed), + name="kernel") + self.bias = self.add_weight( + shape=(1,), initializer=Zeros(), name="bias") + self.dnn = DNN(self.hidden_units, self.activation, self.l2_reg, self.dropout_rate, self.use_bn, seed=self.seed) + + super(LocalActivationUnit, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, inputs, training=None, **kwargs): + + query, keys = inputs + + keys_len = keys.get_shape()[1] + #queries = K.repeat_elements(query, keys_len, 1) + queries = tf.tile(query, [1, keys_len, 1]) + + att_input = tf.concat( + [queries, keys, queries - keys, queries * keys], axis=-1) + + att_out = self.dnn(att_input, training=training) + + attention_score = tf.nn.bias_add(tf.tensordot(att_out, self.kernel, axes=(-1, 0)), self.bias) + + return attention_score + + def compute_output_shape(self, input_shape): + return input_shape[1][:2] + (1,) + + def compute_mask(self, inputs, mask): + return mask + + def get_config(self, ): + config = {'activation': self.activation, 'hidden_units': self.hidden_units, + 'l2_reg': self.l2_reg, 'dropout_rate': self.dropout_rate, 'use_bn': self.use_bn, 'seed': self.seed} + base_config = super(LocalActivationUnit, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class DNN(Layer): + """The Multi Layer Percetron + + Input shape + - nD tensor with shape: ``(batch_size, ..., input_dim)``. The most common situation would be a 2D input with shape ``(batch_size, input_dim)``. + + Output shape + - nD tensor with shape: ``(batch_size, ..., hidden_size[-1])``. For instance, for a 2D input with shape ``(batch_size, input_dim)``, the output would have shape ``(batch_size, hidden_size[-1])``. + + Arguments + - **hidden_units**:list of positive integer, the layer number and units in each layer. + + - **activation**: Activation function to use. + + - **l2_reg**: float between 0 and 1. L2 regularizer strength applied to the kernel weights matrix. + + - **dropout_rate**: float in [0,1). Fraction of the units to dropout. + + - **use_bn**: bool. Whether use BatchNormalization before activation or not. + + - **output_activation**: Activation function to use in the last layer.If ``None``,it will be same as ``activation``. + + - **seed**: A Python integer to use as random seed. + """ + + def __init__(self, hidden_units, activation='relu', l2_reg=0, dropout_rate=0, use_bn=False, output_activation=None, + seed=1024, **kwargs): + self.hidden_units = hidden_units + self.activation = activation + self.l2_reg = l2_reg + self.dropout_rate = dropout_rate + self.use_bn = use_bn + self.output_activation = output_activation + self.seed = seed + + super(DNN, self).__init__(**kwargs) + + def build(self, input_shape): + # if len(self.hidden_units) == 0: + # raise ValueError("hidden_units is empty") + input_size = input_shape[-1] + hidden_units = [int(input_size)] + list(self.hidden_units) + self.kernels = [self.add_weight(name='kernel' + str(i), + shape=( + hidden_units[i], hidden_units[i + 1]), + initializer=glorot_normal( + seed=self.seed), + regularizer=l2(self.l2_reg), + trainable=True) for i in range(len(self.hidden_units))] + self.bias = [self.add_weight(name='bias' + str(i), + shape=(self.hidden_units[i],), + initializer=Zeros(), + trainable=True) for i in range(len(self.hidden_units))] + if self.use_bn: + self.bn_layers = [BatchNormalization() for _ in range(len(self.hidden_units))] + + self.dropout_layers = [Dropout(self.dropout_rate, seed=self.seed + i) for i in + range(len(self.hidden_units))] + + self.activation_layers = [activation_layer(self.activation) for _ in range(len(self.hidden_units))] + + if self.output_activation: + self.activation_layers[-1] = activation_layer(self.output_activation) + + super(DNN, self).build(input_shape) # Be sure to call this somewhere! + + def call(self, inputs, training=None, **kwargs): + + deep_input = inputs + + for i in range(len(self.hidden_units)): + fc = tf.nn.bias_add(tf.tensordot( + deep_input, self.kernels[i], axes=(-1, 0)), self.bias[i]) + + if self.use_bn: + fc = self.bn_layers[i](fc, training=training) + try: + fc = self.activation_layers[i](fc, training=training) + except TypeError as e: # TypeError: call() got an unexpected keyword argument 'training' + print("make sure the activation function use training flag properly", e) + fc = self.activation_layers[i](fc) + + fc = self.dropout_layers[i](fc, training=training) + deep_input = fc + + return deep_input + + def compute_output_shape(self, input_shape): + if len(self.hidden_units) > 0: + shape = input_shape[:-1] + (self.hidden_units[-1],) + else: + shape = input_shape + + return tuple(shape) + + def get_config(self, ): + config = {'activation': self.activation, 'hidden_units': self.hidden_units, + 'l2_reg': self.l2_reg, 'use_bn': self.use_bn, 'dropout_rate': self.dropout_rate, + 'output_activation': self.output_activation, 'seed': self.seed} + base_config = super(DNN, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class PredictionLayer(Layer): + """ + Arguments + - **task**: str, ``"binary"`` for binary logloss or ``"regression"`` for regression loss + + - **use_bias**: bool.Whether add bias term or not. + """ + + def __init__(self, task='binary', use_bias=True, **kwargs): + if task not in ["binary", "multiclass", "regression"]: + raise ValueError("task must be binary,multiclass or regression") + self.task = task + self.use_bias = use_bias + super(PredictionLayer, self).__init__(**kwargs) + + def build(self, input_shape): + + if self.use_bias: + self.global_bias = self.add_weight( + shape=(1,), initializer=Zeros(), name="global_bias") + + # Be sure to call this somewhere! + super(PredictionLayer, self).build(input_shape) + + def call(self, inputs, **kwargs): + x = inputs + if self.use_bias: + x = tf.nn.bias_add(x, self.global_bias, data_format='NHWC') + if self.task == "binary": + x = tf.sigmoid(x) + + output = tf.reshape(x, (-1, 1)) + + return output + + def compute_output_shape(self, input_shape): + return (None, 1) + + def get_config(self, ): + config = {'task': self.task, 'use_bias': self.use_bias} + base_config = super(PredictionLayer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class RegulationModule(Layer): + """Regulation module used in EDCN. + + Input shape + - 3D tensor with shape: ``(batch_size,field_size,embedding_size)``. + + Output shape + - 2D tensor with shape: ``(batch_size,field_size * embedding_size)``. + + Arguments + - **tau** : Positive float, the temperature coefficient to control + distribution of field-wise gating unit. + + References + - [Enhancing Explicit and Implicit Feature Interactions via Information Sharing for Parallel Deep CTR Models.](https://dlp-kdd.github.io/assets/pdf/DLP-KDD_2021_paper_12.pdf) + """ + + def __init__(self, tau=1.0, **kwargs): + if tau == 0: + raise ValueError("RegulationModule tau can not be zero.") + self.tau = 1.0 / tau + super(RegulationModule, self).__init__(**kwargs) + + def build(self, input_shape): + self.field_size = int(input_shape[1]) + self.embedding_size = int(input_shape[2]) + self.g = self.add_weight( + shape=(1, self.field_size, 1), + initializer=Ones(), + name=self.name + '_field_weight') + + # Be sure to call this somewhere! + super(RegulationModule, self).build(input_shape) + + def call(self, inputs, **kwargs): + + if K.ndim(inputs) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + feild_gating_score = tf.nn.softmax(self.g * self.tau, 1) + E = inputs * feild_gating_score + return tf.reshape(E, [-1, self.field_size * self.embedding_size]) + + def compute_output_shape(self, input_shape): + return (None, self.field_size * self.embedding_size) + + def get_config(self): + config = {'tau': self.tau} + base_config = super(RegulationModule, self).get_config() + base_config.update(config) + return base_config + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/interaction.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/interaction.py new file mode 100644 index 000000000..03b022bd7 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/interaction.py @@ -0,0 +1,1561 @@ +# -*- coding:utf-8 -*- +""" + +Authors: + Weichen Shen,weichenswc@163.com, + Harshit Pande, + Yi He, heyi_jack@163.com + +""" +from npu_bridge.npu_init import * + +import itertools + +import tensorflow as tf +from tensorflow.python.keras import backend as K +from tensorflow.python.keras.backend import batch_dot + +try: + from tensorflow.python.ops.init_ops import Zeros, Ones, Constant, TruncatedNormal, \ + glorot_normal_initializer as glorot_normal, \ + glorot_uniform_initializer as glorot_uniform +except ImportError: + from tensorflow.python.ops.init_ops_v2 import Zeros, Ones, Constant, TruncatedNormal, glorot_normal, glorot_uniform + +from tensorflow.python.keras.layers import Layer, MaxPooling2D, Conv2D, Dropout, Lambda, Dense, Flatten +from tensorflow.python.keras.regularizers import l2 +from tensorflow.python.layers import utils + +from .activation import activation_layer +from .utils import concat_func, reduce_sum, softmax, reduce_mean +from .core import DNN + + +class AFMLayer(Layer): + """Attentonal Factorization Machine models pairwise (order-2) feature + interactions without linear term and bias. + + Input shape + - A list of 3D tensor with shape: ``(batch_size,1,embedding_size)``. + + Output shape + - 2D tensor with shape: ``(batch_size, 1)``. + + Arguments + - **attention_factor** : Positive integer, dimensionality of the + attention network output space. + + - **l2_reg_w** : float between 0 and 1. L2 regularizer strength + applied to attention network. + + - **dropout_rate** : float between in [0,1). Fraction of the attention net output units to dropout. + + - **seed** : A Python integer to use as random seed. + + References + - [Attentional Factorization Machines : Learning the Weight of Feature + Interactions via Attention Networks](https://arxiv.org/pdf/1708.04617.pdf) + """ + + def __init__(self, attention_factor=4, l2_reg_w=0, dropout_rate=0, seed=1024, **kwargs): + self.attention_factor = attention_factor + self.l2_reg_w = l2_reg_w + self.dropout_rate = dropout_rate + self.seed = seed + super(AFMLayer, self).__init__(**kwargs) + + def build(self, input_shape): + + if not isinstance(input_shape, list) or len(input_shape) < 2: + # input_shape = input_shape[0] + # if not isinstance(input_shape, list) or len(input_shape) < 2: + raise ValueError('A `AttentionalFM` layer should be called ' + 'on a list of at least 2 inputs') + + shape_set = set() + reduced_input_shape = [shape.as_list() for shape in input_shape] + for i in range(len(input_shape)): + shape_set.add(tuple(reduced_input_shape[i])) + + if len(shape_set) > 1: + raise ValueError('A `AttentionalFM` layer requires ' + 'inputs with same shapes ' + 'Got different shapes: %s' % (shape_set)) + + if len(input_shape[0]) != 3 or input_shape[0][1] != 1: + raise ValueError('A `AttentionalFM` layer requires ' + 'inputs of a list with same shape tensor like\ + (None, 1, embedding_size)' + 'Got different shapes: %s' % (input_shape[0])) + + embedding_size = int(input_shape[0][-1]) + + self.attention_W = self.add_weight(shape=(embedding_size, + self.attention_factor), initializer=glorot_normal(seed=self.seed), + regularizer=l2(self.l2_reg_w), name="attention_W") + self.attention_b = self.add_weight( + shape=(self.attention_factor,), initializer=Zeros(), name="attention_b") + self.projection_h = self.add_weight(shape=(self.attention_factor, 1), + initializer=glorot_normal(seed=self.seed), name="projection_h") + self.projection_p = self.add_weight(shape=( + embedding_size, 1), initializer=glorot_normal(seed=self.seed), name="projection_p") + self.dropout = Dropout( + self.dropout_rate, seed=self.seed) + + self.tensordot = Lambda( + lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0))) + + # Be sure to call this somewhere! + super(AFMLayer, self).build(input_shape) + + def call(self, inputs, training=None, **kwargs): + + if K.ndim(inputs[0]) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + embeds_vec_list = inputs + row = [] + col = [] + + for r, c in itertools.combinations(embeds_vec_list, 2): + row.append(r) + col.append(c) + + p = tf.concat(row, axis=1) + q = tf.concat(col, axis=1) + inner_product = p * q + + bi_interaction = inner_product + attention_temp = tf.nn.relu(tf.nn.bias_add(tf.tensordot( + bi_interaction, self.attention_W, axes=(-1, 0)), self.attention_b)) + # Dense(self.attention_factor,'relu',kernel_regularizer=l2(self.l2_reg_w))(bi_interaction) + self.normalized_att_score = softmax(tf.tensordot( + attention_temp, self.projection_h, axes=(-1, 0)), dim=1) + attention_output = reduce_sum( + self.normalized_att_score * bi_interaction, axis=1) + + attention_output = self.dropout(attention_output, training=training) # training + + afm_out = self.tensordot([attention_output, self.projection_p]) + return afm_out + + def compute_output_shape(self, input_shape): + + if not isinstance(input_shape, list): + raise ValueError('A `AFMLayer` layer should be called ' + 'on a list of inputs.') + return (None, 1) + + def get_config(self, ): + config = {'attention_factor': self.attention_factor, + 'l2_reg_w': self.l2_reg_w, 'dropout_rate': self.dropout_rate, 'seed': self.seed} + base_config = super(AFMLayer, self).get_config() + base_config.update(config) + return base_config + + +class BiInteractionPooling(Layer): + """Bi-Interaction Layer used in Neural FM,compress the + pairwise element-wise product of features into one single vector. + + Input shape + - A 3D tensor with shape:``(batch_size,field_size,embedding_size)``. + + Output shape + - 3D tensor with shape: ``(batch_size,1,embedding_size)``. + + References + - [He X, Chua T S. Neural factorization machines for sparse predictive analytics[C]//Proceedings of the 40th International ACM SIGIR conference on Research and Development in Information Retrieval. ACM, 2017: 355-364.](http://arxiv.org/abs/1708.05027) + """ + + def __init__(self, **kwargs): + + super(BiInteractionPooling, self).__init__(**kwargs) + + def build(self, input_shape): + + if len(input_shape) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape))) + + super(BiInteractionPooling, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + + if K.ndim(inputs) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + concated_embeds_value = inputs + square_of_sum = tf.square(reduce_sum( + concated_embeds_value, axis=1, keep_dims=True)) + sum_of_square = reduce_sum( + concated_embeds_value * concated_embeds_value, axis=1, keep_dims=True) + cross_term = 0.5 * (square_of_sum - sum_of_square) + + return cross_term + + def compute_output_shape(self, input_shape): + return (None, 1, input_shape[-1]) + + +class CIN(Layer): + """Compressed Interaction Network used in xDeepFM.This implemention is + adapted from code that the author of the paper published on https://github.com/Leavingseason/xDeepFM. + + Input shape + - 3D tensor with shape: ``(batch_size,field_size,embedding_size)``. + + Output shape + - 2D tensor with shape: ``(batch_size, featuremap_num)`` ``featuremap_num = sum(self.layer_size[:-1]) // 2 + self.layer_size[-1]`` if ``split_half=True``,else ``sum(layer_size)`` . + + Arguments + - **layer_size** : list of int.Feature maps in each layer. + + - **activation** : activation function used on feature maps. + + - **split_half** : bool.if set to False, half of the feature maps in each hidden will connect to output unit. + + - **seed** : A Python integer to use as random seed. + + References + - [Lian J, Zhou X, Zhang F, et al. xDeepFM: Combining Explicit and Implicit Feature Interactions for Recommender Systems[J]. arXiv preprint arXiv:1803.05170, 2018.] (https://arxiv.org/pdf/1803.05170.pdf) + """ + + def __init__(self, layer_size=(128, 128), activation='relu', split_half=True, l2_reg=1e-5, seed=1024, **kwargs): + if len(layer_size) == 0: + raise ValueError( + "layer_size must be a list(tuple) of length greater than 1") + self.layer_size = layer_size + self.split_half = split_half + self.activation = activation + self.l2_reg = l2_reg + self.seed = seed + super(CIN, self).__init__(**kwargs) + + def build(self, input_shape): + if len(input_shape) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape))) + + self.field_nums = [int(input_shape[1])] + self.filters = [] + self.bias = [] + for i, size in enumerate(self.layer_size): + + self.filters.append(self.add_weight(name='filter' + str(i), + shape=[1, self.field_nums[-1] + * self.field_nums[0], size], + dtype=tf.float32, initializer=glorot_uniform( + seed=self.seed + i), + regularizer=l2(self.l2_reg))) + + self.bias.append(self.add_weight(name='bias' + str(i), shape=[size], dtype=tf.float32, + initializer=Zeros())) + + if self.split_half: + if i != len(self.layer_size) - 1 and size % 2 > 0: + raise ValueError( + "layer_size must be even number except for the last layer when split_half=True") + + self.field_nums.append(size // 2) + else: + self.field_nums.append(size) + + self.activation_layers = [activation_layer( + self.activation) for _ in self.layer_size] + + super(CIN, self).build(input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + + if K.ndim(inputs) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + dim = int(inputs.get_shape()[-1]) + hidden_nn_layers = [inputs] + final_result = [] + + split_tensor0 = tf.split(hidden_nn_layers[0], dim * [1], 2) + for idx, layer_size in enumerate(self.layer_size): + split_tensor = tf.split(hidden_nn_layers[-1], dim * [1], 2) + + dot_result_m = tf.matmul( + split_tensor0, split_tensor, transpose_b=True) + + dot_result_o = tf.reshape( + dot_result_m, shape=[dim, -1, self.field_nums[0] * self.field_nums[idx]]) + + dot_result = tf.transpose(dot_result_o, perm=[1, 0, 2]) + + curr_out = tf.nn.conv1d( + dot_result, filters=self.filters[idx], stride=1, padding='VALID') + + curr_out = tf.nn.bias_add(curr_out, self.bias[idx]) + + curr_out = self.activation_layers[idx](curr_out) + + curr_out = tf.transpose(curr_out, perm=[0, 2, 1]) + + if self.split_half: + if idx != len(self.layer_size) - 1: + next_hidden, direct_connect = tf.split( + curr_out, 2 * [layer_size // 2], 1) + else: + direct_connect = curr_out + next_hidden = 0 + else: + direct_connect = curr_out + next_hidden = curr_out + + final_result.append(direct_connect) + hidden_nn_layers.append(next_hidden) + + result = tf.concat(final_result, axis=1) + result = reduce_sum(result, -1, keep_dims=False) + + return result + + def compute_output_shape(self, input_shape): + if self.split_half: + featuremap_num = sum( + self.layer_size[:-1]) // 2 + self.layer_size[-1] + else: + featuremap_num = sum(self.layer_size) + return (None, featuremap_num) + + def get_config(self, ): + + config = {'layer_size': self.layer_size, 'split_half': self.split_half, 'activation': self.activation, + 'seed': self.seed} + base_config = super(CIN, self).get_config() + base_config.update(config) + return base_config + + +class CrossNet(Layer): + """The Cross Network part of Deep&Cross Network model, + which leans both low and high degree cross feature. + + Input shape + - 2D tensor with shape: ``(batch_size, units)``. + + Output shape + - 2D tensor with shape: ``(batch_size, units)``. + + Arguments + - **layer_num**: Positive integer, the cross layer number + + - **l2_reg**: float between 0 and 1. L2 regularizer strength applied to the kernel weights matrix + + - **parameterization**: string, ``"vector"`` or ``"matrix"`` , way to parameterize the cross network. + + - **seed**: A Python integer to use as random seed. + + References + - [Wang R, Fu B, Fu G, et al. Deep & cross network for ad click predictions[C]//Proceedings of the ADKDD'17. ACM, 2017: 12.](https://arxiv.org/abs/1708.05123) + """ + + def __init__(self, layer_num=2, parameterization='vector', l2_reg=0, seed=1024, **kwargs): + self.layer_num = layer_num + self.parameterization = parameterization + self.l2_reg = l2_reg + self.seed = seed + print('CrossNet parameterization:', self.parameterization) + super(CrossNet, self).__init__(**kwargs) + + def build(self, input_shape): + + if len(input_shape) != 2: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 2 dimensions" % (len(input_shape),)) + + dim = int(input_shape[-1]) + if self.parameterization == 'vector': + self.kernels = [self.add_weight(name='kernel' + str(i), + shape=(dim, 1), + initializer=glorot_normal( + seed=self.seed), + regularizer=l2(self.l2_reg), + trainable=True) for i in range(self.layer_num)] + elif self.parameterization == 'matrix': + self.kernels = [self.add_weight(name='kernel' + str(i), + shape=(dim, dim), + initializer=glorot_normal( + seed=self.seed), + regularizer=l2(self.l2_reg), + trainable=True) for i in range(self.layer_num)] + else: # error + raise ValueError("parameterization should be 'vector' or 'matrix'") + self.bias = [self.add_weight(name='bias' + str(i), + shape=(dim, 1), + initializer=Zeros(), + trainable=True) for i in range(self.layer_num)] + # Be sure to call this somewhere! + super(CrossNet, self).build(input_shape) + + def call(self, inputs, **kwargs): + if K.ndim(inputs) != 2: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 2 dimensions" % (K.ndim(inputs))) + + x_0 = tf.expand_dims(inputs, axis=2) + x_l = x_0 + for i in range(self.layer_num): + if self.parameterization == 'vector': + xl_w = tf.tensordot(x_l, self.kernels[i], axes=(1, 0)) + dot_ = tf.matmul(x_0, xl_w) + x_l = dot_ + self.bias[i] + x_l + elif self.parameterization == 'matrix': + xl_w = tf.einsum('ij,bjk->bik', self.kernels[i], x_l) # W * xi (bs, dim, 1) + dot_ = xl_w + self.bias[i] # W * xi + b + x_l = x_0 * dot_ + x_l # x0 · (W * xi + b) +xl Hadamard-product + else: # error + raise ValueError("parameterization should be 'vector' or 'matrix'") + x_l = tf.squeeze(x_l, axis=2) + return x_l + + def get_config(self, ): + + config = {'layer_num': self.layer_num, 'parameterization': self.parameterization, + 'l2_reg': self.l2_reg, 'seed': self.seed} + base_config = super(CrossNet, self).get_config() + base_config.update(config) + return base_config + + def compute_output_shape(self, input_shape): + return input_shape + + +class CrossNetMix(Layer): + """The Cross Network part of DCN-Mix model, which improves DCN-M by: + 1 add MOE to learn feature interactions in different subspaces + 2 add nonlinear transformations in low-dimensional space + + Input shape + - 2D tensor with shape: ``(batch_size, units)``. + + Output shape + - 2D tensor with shape: ``(batch_size, units)``. + + Arguments + - **low_rank** : Positive integer, dimensionality of low-rank sapce. + + - **num_experts** : Positive integer, number of experts. + + - **layer_num**: Positive integer, the cross layer number + + - **l2_reg**: float between 0 and 1. L2 regularizer strength applied to the kernel weights matrix + + - **seed**: A Python integer to use as random seed. + + References + - [Wang R, Shivanna R, Cheng D Z, et al. DCN-M: Improved Deep & Cross Network for Feature Cross Learning in Web-scale Learning to Rank Systems[J]. 2020.](https://arxiv.org/abs/2008.13535) + """ + + def __init__(self, low_rank=32, num_experts=4, layer_num=2, l2_reg=0, seed=1024, **kwargs): + self.low_rank = low_rank + self.num_experts = num_experts + self.layer_num = layer_num + self.l2_reg = l2_reg + self.seed = seed + super(CrossNetMix, self).__init__(**kwargs) + + def build(self, input_shape): + + if len(input_shape) != 2: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 2 dimensions" % (len(input_shape),)) + + dim = int(input_shape[-1]) + + # U: (dim, low_rank) + self.U_list = [self.add_weight(name='U_list' + str(i), + shape=(self.num_experts, dim, self.low_rank), + initializer=glorot_normal( + seed=self.seed), + regularizer=l2(self.l2_reg), + trainable=True) for i in range(self.layer_num)] + # V: (dim, low_rank) + self.V_list = [self.add_weight(name='V_list' + str(i), + shape=(self.num_experts, dim, self.low_rank), + initializer=glorot_normal( + seed=self.seed), + regularizer=l2(self.l2_reg), + trainable=True) for i in range(self.layer_num)] + # C: (low_rank, low_rank) + self.C_list = [self.add_weight(name='C_list' + str(i), + shape=(self.num_experts, self.low_rank, self.low_rank), + initializer=glorot_normal( + seed=self.seed), + regularizer=l2(self.l2_reg), + trainable=True) for i in range(self.layer_num)] + + self.gating = [Dense(1, use_bias=False) for i in range(self.num_experts)] + + self.bias = [self.add_weight(name='bias' + str(i), + shape=(dim, 1), + initializer=Zeros(), + trainable=True) for i in range(self.layer_num)] + # Be sure to call this somewhere! + super(CrossNetMix, self).build(input_shape) + + def call(self, inputs, **kwargs): + if K.ndim(inputs) != 2: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 2 dimensions" % (K.ndim(inputs))) + + x_0 = tf.expand_dims(inputs, axis=2) + x_l = x_0 + for i in range(self.layer_num): + output_of_experts = [] + gating_score_of_experts = [] + for expert_id in range(self.num_experts): + # (1) G(x_l) + # compute the gating score by x_l + gating_score_of_experts.append(self.gating[expert_id](tf.squeeze(x_l, axis=2))) + + # (2) E(x_l) + # project the input x_l to $\mathbb{R}^{r}$ + v_x = tf.einsum('ij,bjk->bik', tf.transpose(self.V_list[i][expert_id]), x_l) # (bs, low_rank, 1) + + # nonlinear activation in low rank space + v_x = tf.nn.tanh(v_x) + v_x = tf.einsum('ij,bjk->bik', self.C_list[i][expert_id], v_x) # (bs, low_rank, 1) + v_x = tf.nn.tanh(v_x) + + # project back to $\mathbb{R}^{d}$ + uv_x = tf.einsum('ij,bjk->bik', self.U_list[i][expert_id], v_x) # (bs, dim, 1) + + dot_ = uv_x + self.bias[i] + dot_ = x_0 * dot_ # Hadamard-product + + output_of_experts.append(tf.squeeze(dot_, axis=2)) + + # (3) mixture of low-rank experts + output_of_experts = tf.stack(output_of_experts, 2) # (bs, dim, num_experts) + gating_score_of_experts = tf.stack(gating_score_of_experts, 1) # (bs, num_experts, 1) + moe_out = tf.matmul(output_of_experts, tf.nn.softmax(gating_score_of_experts, 1)) + x_l = moe_out + x_l # (bs, dim, 1) + x_l = tf.squeeze(x_l, axis=2) + return x_l + + def get_config(self, ): + + config = {'low_rank': self.low_rank, 'num_experts': self.num_experts, 'layer_num': self.layer_num, + 'l2_reg': self.l2_reg, 'seed': self.seed} + base_config = super(CrossNetMix, self).get_config() + base_config.update(config) + return base_config + + def compute_output_shape(self, input_shape): + return input_shape + + +class FM(Layer): + """Factorization Machine models pairwise (order-2) feature interactions + without linear term and bias. + + Input shape + - 3D tensor with shape: ``(batch_size,field_size,embedding_size)``. + + Output shape + - 2D tensor with shape: ``(batch_size, 1)``. + + References + - [Factorization Machines](https://www.csie.ntu.edu.tw/~b97053/paper/Rendle2010FM.pdf) + """ + + def __init__(self, **kwargs): + + super(FM, self).__init__(**kwargs) + + def build(self, input_shape): + if len(input_shape) != 3: + raise ValueError("Unexpected inputs dimensions % d,\ + expect to be 3 dimensions" % (len(input_shape))) + + super(FM, self).build(input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + + if K.ndim(inputs) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" + % (K.ndim(inputs))) + + concated_embeds_value = inputs + + square_of_sum = tf.square(reduce_sum( + concated_embeds_value, axis=1, keep_dims=True)) + sum_of_square = reduce_sum( + concated_embeds_value * concated_embeds_value, axis=1, keep_dims=True) + cross_term = square_of_sum - sum_of_square + cross_term = 0.5 * reduce_sum(cross_term, axis=2, keep_dims=False) + + return cross_term + + def compute_output_shape(self, input_shape): + return (None, 1) + + +class InnerProductLayer(Layer): + """InnerProduct Layer used in PNN that compute the element-wise + product or inner product between feature vectors. + + Input shape + - a list of 3D tensor with shape: ``(batch_size,1,embedding_size)``. + + Output shape + - 3D tensor with shape: ``(batch_size, N*(N-1)/2 ,1)`` if use reduce_sum. or 3D tensor with shape: ``(batch_size, N*(N-1)/2, embedding_size )`` if not use reduce_sum. + + Arguments + - **reduce_sum**: bool. Whether return inner product or element-wise product + + References + - [Qu Y, Cai H, Ren K, et al. Product-based neural networks for user response prediction[C]//Data Mining (ICDM), 2016 IEEE 16th International Conference on. IEEE, 2016: 1149-1154.](https://arxiv.org/pdf/1611.00144.pdf) + """ + + def __init__(self, reduce_sum=True, **kwargs): + self.reduce_sum = reduce_sum + super(InnerProductLayer, self).__init__(**kwargs) + + def build(self, input_shape): + + if not isinstance(input_shape, list) or len(input_shape) < 2: + raise ValueError('A `InnerProductLayer` layer should be called ' + 'on a list of at least 2 inputs') + + reduced_inputs_shapes = [shape.as_list() for shape in input_shape] + shape_set = set() + + for i in range(len(input_shape)): + shape_set.add(tuple(reduced_inputs_shapes[i])) + + if len(shape_set) > 1: + raise ValueError('A `InnerProductLayer` layer requires ' + 'inputs with same shapes ' + 'Got different shapes: %s' % (shape_set)) + + if len(input_shape[0]) != 3 or input_shape[0][1] != 1: + raise ValueError('A `InnerProductLayer` layer requires ' + 'inputs of a list with same shape tensor like (None,1,embedding_size)' + 'Got different shapes: %s' % (input_shape[0])) + super(InnerProductLayer, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + if K.ndim(inputs[0]) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + embed_list = inputs + row = [] + col = [] + num_inputs = len(embed_list) + + for i in range(num_inputs - 1): + for j in range(i + 1, num_inputs): + row.append(i) + col.append(j) + p = tf.concat([embed_list[idx] + for idx in row], axis=1) # batch num_pairs k + q = tf.concat([embed_list[idx] + for idx in col], axis=1) + + inner_product = p * q + if self.reduce_sum: + inner_product = reduce_sum( + inner_product, axis=2, keep_dims=True) + return inner_product + + def compute_output_shape(self, input_shape): + num_inputs = len(input_shape) + num_pairs = int(num_inputs * (num_inputs - 1) / 2) + input_shape = input_shape[0] + embed_size = input_shape[-1] + if self.reduce_sum: + return (input_shape[0], num_pairs, 1) + else: + return (input_shape[0], num_pairs, embed_size) + + def get_config(self, ): + config = {'reduce_sum': self.reduce_sum, } + base_config = super(InnerProductLayer, self).get_config() + base_config.update(config) + return base_config + + +class InteractingLayer(Layer): + """A Layer used in AutoInt that model the correlations between different feature fields by multi-head self-attention mechanism. + + Input shape + - A 3D tensor with shape: ``(batch_size,field_size,embedding_size)``. + + Output shape + - 3D tensor with shape:``(batch_size,field_size,att_embedding_size * head_num)``. + + + Arguments + - **att_embedding_size**: int.The embedding size in multi-head self-attention network. + - **head_num**: int.The head number in multi-head self-attention network. + - **use_res**: bool.Whether or not use standard residual connections before output. + - **seed**: A Python integer to use as random seed. + + References + - [Song W, Shi C, Xiao Z, et al. AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks[J]. arXiv preprint arXiv:1810.11921, 2018.](https://arxiv.org/abs/1810.11921) + """ + + def __init__(self, att_embedding_size=8, head_num=2, use_res=True, scaling=False, seed=1024, **kwargs): + if head_num <= 0: + raise ValueError('head_num must be a int > 0') + self.att_embedding_size = att_embedding_size + self.head_num = head_num + self.use_res = use_res + self.seed = seed + self.scaling = scaling + super(InteractingLayer, self).__init__(**kwargs) + + def build(self, input_shape): + if len(input_shape) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape))) + embedding_size = int(input_shape[-1]) + self.W_Query = self.add_weight(name='query', shape=[embedding_size, self.att_embedding_size * self.head_num], + dtype=tf.float32, + initializer=TruncatedNormal(seed=self.seed)) + self.W_key = self.add_weight(name='key', shape=[embedding_size, self.att_embedding_size * self.head_num], + dtype=tf.float32, + initializer=TruncatedNormal(seed=self.seed + 1)) + self.W_Value = self.add_weight(name='value', shape=[embedding_size, self.att_embedding_size * self.head_num], + dtype=tf.float32, + initializer=TruncatedNormal(seed=self.seed + 2)) + if self.use_res: + self.W_Res = self.add_weight(name='res', shape=[embedding_size, self.att_embedding_size * self.head_num], + dtype=tf.float32, + initializer=TruncatedNormal(seed=self.seed)) + + # Be sure to call this somewhere! + super(InteractingLayer, self).build(input_shape) + + def call(self, inputs, **kwargs): + if K.ndim(inputs) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + querys = tf.tensordot(inputs, self.W_Query, + axes=(-1, 0)) # None F D*head_num + keys = tf.tensordot(inputs, self.W_key, axes=(-1, 0)) + values = tf.tensordot(inputs, self.W_Value, axes=(-1, 0)) + + # head_num None F D + querys = tf.stack(tf.split(querys, self.head_num, axis=2)) + keys = tf.stack(tf.split(keys, self.head_num, axis=2)) + values = tf.stack(tf.split(values, self.head_num, axis=2)) + + inner_product = tf.matmul( + querys, keys, transpose_b=True) # head_num None F F + if self.scaling: + inner_product /= self.att_embedding_size ** 0.5 + self.normalized_att_scores = softmax(inner_product) + + result = tf.matmul(self.normalized_att_scores, + values) # head_num None F D + result = tf.concat(tf.split(result, self.head_num, ), axis=-1) + result = tf.squeeze(result, axis=0) # None F D*head_num + + if self.use_res: + result += tf.tensordot(inputs, self.W_Res, axes=(-1, 0)) + result = tf.nn.relu(result) + + return result + + def compute_output_shape(self, input_shape): + + return (None, input_shape[1], self.att_embedding_size * self.head_num) + + def get_config(self, ): + config = {'att_embedding_size': self.att_embedding_size, 'head_num': self.head_num, 'use_res': self.use_res, + 'seed': self.seed} + base_config = super(InteractingLayer, self).get_config() + base_config.update(config) + return base_config + + +class OutterProductLayer(Layer): + """OutterProduct Layer used in PNN.This implemention is + adapted from code that the author of the paper published on https://github.com/Atomu2014/product-nets. + + Input shape + - A list of N 3D tensor with shape: ``(batch_size,1,embedding_size)``. + + Output shape + - 2D tensor with shape:``(batch_size,N*(N-1)/2 )``. + + Arguments + - **kernel_type**: str. The kernel weight matrix type to use,can be mat,vec or num + + - **seed**: A Python integer to use as random seed. + + References + - [Qu Y, Cai H, Ren K, et al. Product-based neural networks for user response prediction[C]//Data Mining (ICDM), 2016 IEEE 16th International Conference on. IEEE, 2016: 1149-1154.](https://arxiv.org/pdf/1611.00144.pdf) + """ + + def __init__(self, kernel_type='mat', seed=1024, **kwargs): + if kernel_type not in ['mat', 'vec', 'num']: + raise ValueError("kernel_type must be mat,vec or num") + self.kernel_type = kernel_type + self.seed = seed + super(OutterProductLayer, self).__init__(**kwargs) + + def build(self, input_shape): + + if not isinstance(input_shape, list) or len(input_shape) < 2: + raise ValueError('A `OutterProductLayer` layer should be called ' + 'on a list of at least 2 inputs') + + reduced_inputs_shapes = [shape.as_list() for shape in input_shape] + shape_set = set() + + for i in range(len(input_shape)): + shape_set.add(tuple(reduced_inputs_shapes[i])) + + if len(shape_set) > 1: + raise ValueError('A `OutterProductLayer` layer requires ' + 'inputs with same shapes ' + 'Got different shapes: %s' % (shape_set)) + + if len(input_shape[0]) != 3 or input_shape[0][1] != 1: + raise ValueError('A `OutterProductLayer` layer requires ' + 'inputs of a list with same shape tensor like (None,1,embedding_size)' + 'Got different shapes: %s' % (input_shape[0])) + num_inputs = len(input_shape) + num_pairs = int(num_inputs * (num_inputs - 1) / 2) + input_shape = input_shape[0] + embed_size = int(input_shape[-1]) + if self.kernel_type == 'mat': + + self.kernel = self.add_weight(shape=(embed_size, num_pairs, embed_size), + initializer=glorot_uniform( + seed=self.seed), + name='kernel') + elif self.kernel_type == 'vec': + self.kernel = self.add_weight(shape=(num_pairs, embed_size,), initializer=glorot_uniform(self.seed), + name='kernel' + ) + elif self.kernel_type == 'num': + self.kernel = self.add_weight( + shape=(num_pairs, 1), initializer=glorot_uniform(self.seed), name='kernel') + + super(OutterProductLayer, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + + if K.ndim(inputs[0]) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + embed_list = inputs + row = [] + col = [] + num_inputs = len(embed_list) + for i in range(num_inputs - 1): + for j in range(i + 1, num_inputs): + row.append(i) + col.append(j) + p = tf.concat([embed_list[idx] + for idx in row], axis=1) # batch num_pairs k + # Reshape([num_pairs, self.embedding_size]) + q = tf.concat([embed_list[idx] for idx in col], axis=1) + + # ------------------------- + if self.kernel_type == 'mat': + p = tf.expand_dims(p, 1) + # k k* pair* k + # batch * pair + kp = reduce_sum( + + # batch * pair * k + + tf.multiply( + + # batch * pair * k + + tf.transpose( + + # batch * k * pair + + reduce_sum( + + # batch * k * pair * k + + tf.multiply( + + p, self.kernel), + + -1), + + [0, 2, 1]), + + q), + + -1) + else: + # 1 * pair * (k or 1) + + k = tf.expand_dims(self.kernel, 0) + + # batch * pair + + kp = reduce_sum(p * q * k, -1) + + # p q # b * p * k + + return kp + + def compute_output_shape(self, input_shape): + num_inputs = len(input_shape) + num_pairs = int(num_inputs * (num_inputs - 1) / 2) + return (None, num_pairs) + + def get_config(self, ): + config = {'kernel_type': self.kernel_type, 'seed': self.seed} + base_config = super(OutterProductLayer, self).get_config() + base_config.update(config) + return base_config + + +class FGCNNLayer(Layer): + """Feature Generation Layer used in FGCNN,including Convolution,MaxPooling and Recombination. + + Input shape + - A 3D tensor with shape:``(batch_size,field_size,embedding_size)``. + + Output shape + - 3D tensor with shape: ``(batch_size,new_feture_num,embedding_size)``. + + References + - [Liu B, Tang R, Chen Y, et al. Feature Generation by Convolutional Neural Network for Click-Through Rate Prediction[J]. arXiv preprint arXiv:1904.04447, 2019.](https://arxiv.org/pdf/1904.04447) + + """ + + def __init__(self, filters=(14, 16,), kernel_width=(7, 7,), new_maps=(3, 3,), pooling_width=(2, 2), + **kwargs): + if not (len(filters) == len(kernel_width) == len(new_maps) == len(pooling_width)): + raise ValueError("length of argument must be equal") + self.filters = filters + self.kernel_width = kernel_width + self.new_maps = new_maps + self.pooling_width = pooling_width + + super(FGCNNLayer, self).__init__(**kwargs) + + def build(self, input_shape): + + if len(input_shape) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape))) + self.conv_layers = [] + self.pooling_layers = [] + self.dense_layers = [] + pooling_shape = input_shape.as_list() + [1, ] + embedding_size = int(input_shape[-1]) + for i in range(1, len(self.filters) + 1): + filters = self.filters[i - 1] + width = self.kernel_width[i - 1] + new_filters = self.new_maps[i - 1] + pooling_width = self.pooling_width[i - 1] + conv_output_shape = self._conv_output_shape( + pooling_shape, (width, 1)) + pooling_shape = self._pooling_output_shape( + conv_output_shape, (pooling_width, 1)) + self.conv_layers.append(Conv2D(filters=filters, kernel_size=(width, 1), strides=(1, 1), + padding='same', + activation='tanh', use_bias=True, )) + self.pooling_layers.append( + MaxPooling2D(pool_size=(pooling_width, 1))) + self.dense_layers.append(Dense(pooling_shape[1] * embedding_size * new_filters, + activation='tanh', use_bias=True)) + + self.flatten = Flatten() + + super(FGCNNLayer, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + + if K.ndim(inputs) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + embedding_size = int(inputs.shape[-1]) + pooling_result = tf.expand_dims(inputs, axis=3) + + new_feature_list = [] + + for i in range(1, len(self.filters) + 1): + new_filters = self.new_maps[i - 1] + + conv_result = self.conv_layers[i - 1](pooling_result) + + pooling_result = self.pooling_layers[i - 1](conv_result) + + flatten_result = self.flatten(pooling_result) + + new_result = self.dense_layers[i - 1](flatten_result) + + new_feature_list.append( + tf.reshape(new_result, (-1, int(pooling_result.shape[1]) * new_filters, embedding_size))) + + new_features = concat_func(new_feature_list, axis=1) + return new_features + + def compute_output_shape(self, input_shape): + + new_features_num = 0 + features_num = input_shape[1] + + for i in range(0, len(self.kernel_width)): + pooled_features_num = features_num // self.pooling_width[i] + new_features_num += self.new_maps[i] * pooled_features_num + features_num = pooled_features_num + + return (None, new_features_num, input_shape[-1]) + + def get_config(self, ): + config = {'kernel_width': self.kernel_width, 'filters': self.filters, 'new_maps': self.new_maps, + 'pooling_width': self.pooling_width} + base_config = super(FGCNNLayer, self).get_config() + base_config.update(config) + return base_config + + def _conv_output_shape(self, input_shape, kernel_size): + # channels_last + space = input_shape[1:-1] + new_space = [] + for i in range(len(space)): + new_dim = utils.conv_output_length( + space[i], + kernel_size[i], + padding='same', + stride=1, + dilation=1) + new_space.append(new_dim) + return ([input_shape[0]] + new_space + [self.filters]) + + def _pooling_output_shape(self, input_shape, pool_size): + # channels_last + + rows = input_shape[1] + cols = input_shape[2] + rows = utils.conv_output_length(rows, pool_size[0], 'valid', + pool_size[0]) + cols = utils.conv_output_length(cols, pool_size[1], 'valid', + pool_size[1]) + return [input_shape[0], rows, cols, input_shape[3]] + + +class SENETLayer(Layer): + """SENETLayer used in FiBiNET. + + Input shape + - A list of 3D tensor with shape: ``(batch_size,1,embedding_size)``. + + Output shape + - A list of 3D tensor with shape: ``(batch_size,1,embedding_size)``. + + Arguments + - **reduction_ratio** : Positive integer, dimensionality of the + attention network output space. + + - **seed** : A Python integer to use as random seed. + + References + - [FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction](https://arxiv.org/pdf/1905.09433.pdf) + """ + + def __init__(self, reduction_ratio=3, seed=1024, **kwargs): + self.reduction_ratio = reduction_ratio + + self.seed = seed + super(SENETLayer, self).__init__(**kwargs) + + def build(self, input_shape): + + if not isinstance(input_shape, list) or len(input_shape) < 2: + raise ValueError('A `AttentionalFM` layer should be called ' + 'on a list of at least 2 inputs') + + self.filed_size = len(input_shape) + self.embedding_size = input_shape[0][-1] + reduction_size = max(1, self.filed_size // self.reduction_ratio) + + self.W_1 = self.add_weight(shape=( + self.filed_size, reduction_size), initializer=glorot_normal(seed=self.seed), name="W_1") + self.W_2 = self.add_weight(shape=( + reduction_size, self.filed_size), initializer=glorot_normal(seed=self.seed), name="W_2") + + self.tensordot = Lambda( + lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0))) + + # Be sure to call this somewhere! + super(SENETLayer, self).build(input_shape) + + def call(self, inputs, training=None, **kwargs): + + if K.ndim(inputs[0]) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + inputs = concat_func(inputs, axis=1) + Z = reduce_mean(inputs, axis=-1, ) + + A_1 = tf.nn.relu(self.tensordot([Z, self.W_1])) + A_2 = tf.nn.relu(self.tensordot([A_1, self.W_2])) + V = tf.multiply(inputs, tf.expand_dims(A_2, axis=2)) + + return tf.split(V, self.filed_size, axis=1) + + def compute_output_shape(self, input_shape): + + return input_shape + + def compute_mask(self, inputs, mask=None): + return [None] * self.filed_size + + def get_config(self, ): + config = {'reduction_ratio': self.reduction_ratio, 'seed': self.seed} + base_config = super(SENETLayer, self).get_config() + base_config.update(config) + return base_config + + +class BilinearInteraction(Layer): + """BilinearInteraction Layer used in FiBiNET. + + Input shape + - A list of 3D tensor with shape: ``(batch_size,1,embedding_size)``. Its length is ``filed_size``. + + Output shape + - 3D tensor with shape: ``(batch_size,filed_size*(filed_size-1)/2,embedding_size)``. + + Arguments + - **bilinear_type** : String, types of bilinear functions used in this layer. + + - **seed** : A Python integer to use as random seed. + + References + - [FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction](https://arxiv.org/pdf/1905.09433.pdf) + + """ + + def __init__(self, bilinear_type="interaction", seed=1024, **kwargs): + self.bilinear_type = bilinear_type + self.seed = seed + + super(BilinearInteraction, self).__init__(**kwargs) + + def build(self, input_shape): + + if not isinstance(input_shape, list) or len(input_shape) < 2: + raise ValueError('A `AttentionalFM` layer should be called ' + 'on a list of at least 2 inputs') + embedding_size = int(input_shape[0][-1]) + + if self.bilinear_type == "all": + self.W = self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal( + seed=self.seed), name="bilinear_weight") + elif self.bilinear_type == "each": + self.W_list = [self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal( + seed=self.seed), name="bilinear_weight" + str(i)) for i in range(len(input_shape) - 1)] + elif self.bilinear_type == "interaction": + self.W_list = [self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal( + seed=self.seed), name="bilinear_weight" + str(i) + '_' + str(j)) for i, j in + itertools.combinations(range(len(input_shape)), 2)] + else: + raise NotImplementedError + + super(BilinearInteraction, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + + if K.ndim(inputs[0]) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs))) + + n = len(inputs) + if self.bilinear_type == "all": + vidots = [tf.tensordot(inputs[i], self.W, axes=(-1, 0)) for i in range(n)] + p = [tf.multiply(vidots[i], inputs[j]) for i, j in itertools.combinations(range(n), 2)] + elif self.bilinear_type == "each": + vidots = [tf.tensordot(inputs[i], self.W_list[i], axes=(-1, 0)) for i in range(n - 1)] + p = [tf.multiply(vidots[i], inputs[j]) for i, j in itertools.combinations(range(n), 2)] + elif self.bilinear_type == "interaction": + p = [tf.multiply(tf.tensordot(v[0], w, axes=(-1, 0)), v[1]) + for v, w in zip(itertools.combinations(inputs, 2), self.W_list)] + else: + raise NotImplementedError + output = concat_func(p, axis=1) + return output + + def compute_output_shape(self, input_shape): + filed_size = len(input_shape) + embedding_size = input_shape[0][-1] + + return (None, filed_size * (filed_size - 1) // 2, embedding_size) + + def get_config(self, ): + config = {'bilinear_type': self.bilinear_type, 'seed': self.seed} + base_config = super(BilinearInteraction, self).get_config() + base_config.update(config) + return base_config + + +class FieldWiseBiInteraction(Layer): + """Field-Wise Bi-Interaction Layer used in FLEN,compress the + pairwise element-wise product of features into one single vector. + + Input shape + - A list of 3D tensor with shape:``(batch_size,field_size,embedding_size)``. + + Output shape + - 2D tensor with shape: ``(batch_size,embedding_size)``. + + Arguments + - **use_bias** : Boolean, if use bias. + - **seed** : A Python integer to use as random seed. + + References + - [FLEN: Leveraging Field for Scalable CTR Prediction](https://arxiv.org/pdf/1911.04690) + + """ + + def __init__(self, use_bias=True, seed=1024, **kwargs): + self.use_bias = use_bias + self.seed = seed + + super(FieldWiseBiInteraction, self).__init__(**kwargs) + + def build(self, input_shape): + + if not isinstance(input_shape, list) or len(input_shape) < 2: + raise ValueError( + 'A `Field-Wise Bi-Interaction` layer should be called ' + 'on a list of at least 2 inputs') + + self.num_fields = len(input_shape) + embedding_size = input_shape[0][-1] + + self.kernel_mf = self.add_weight( + name='kernel_mf', + shape=(int(self.num_fields * (self.num_fields - 1) / 2), 1), + initializer=Ones(), + regularizer=None, + trainable=True) + + self.kernel_fm = self.add_weight( + name='kernel_fm', + shape=(self.num_fields, 1), + initializer=Constant(value=0.5), + regularizer=None, + trainable=True) + if self.use_bias: + self.bias_mf = self.add_weight(name='bias_mf', + shape=(embedding_size), + initializer=Zeros()) + self.bias_fm = self.add_weight(name='bias_fm', + shape=(embedding_size), + initializer=Zeros()) + + super(FieldWiseBiInteraction, + self).build(input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + + if K.ndim(inputs[0]) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % + (K.ndim(inputs))) + + field_wise_embeds_list = inputs + + # MF module + field_wise_vectors = tf.concat([ + reduce_sum(field_i_vectors, axis=1, keep_dims=True) + for field_i_vectors in field_wise_embeds_list + ], 1) + + left = [] + right = [] + + for i, j in itertools.combinations(list(range(self.num_fields)), 2): + left.append(i) + right.append(j) + + embeddings_left = tf.gather(params=field_wise_vectors, + indices=left, + axis=1) + embeddings_right = tf.gather(params=field_wise_vectors, + indices=right, + axis=1) + + embeddings_prod = embeddings_left * embeddings_right + field_weighted_embedding = embeddings_prod * self.kernel_mf + h_mf = reduce_sum(field_weighted_embedding, axis=1) + if self.use_bias: + h_mf = tf.nn.bias_add(h_mf, self.bias_mf) + + # FM module + square_of_sum_list = [ + tf.square(reduce_sum(field_i_vectors, axis=1, keep_dims=True)) + for field_i_vectors in field_wise_embeds_list + ] + sum_of_square_list = [ + reduce_sum(field_i_vectors * field_i_vectors, + axis=1, + keep_dims=True) + for field_i_vectors in field_wise_embeds_list + ] + + field_fm = tf.concat([ + square_of_sum - sum_of_square for square_of_sum, sum_of_square in + zip(square_of_sum_list, sum_of_square_list) + ], 1) + + h_fm = reduce_sum(field_fm * self.kernel_fm, axis=1) + if self.use_bias: + h_fm = tf.nn.bias_add(h_fm, self.bias_fm) + + return h_mf + h_fm + + def compute_output_shape(self, input_shape): + return (None, input_shape[0][-1]) + + def get_config(self, ): + config = {'use_bias': self.use_bias, 'seed': self.seed} + base_config = super(FieldWiseBiInteraction, self).get_config() + base_config.update(config) + return base_config + + +class FwFMLayer(Layer): + """Field-weighted Factorization Machines + + Input shape + - 3D tensor with shape: ``(batch_size,field_size,embedding_size)``. + + Output shape + - 2D tensor with shape: ``(batch_size, 1)``. + + Arguments + - **num_fields** : integer for number of fields + - **regularizer** : L2 regularizer weight for the field strength parameters of FwFM + + References + - [Field-weighted Factorization Machines for Click-Through Rate Prediction in Display Advertising] + https://arxiv.org/pdf/1806.03514.pdf + """ + + def __init__(self, num_fields=4, regularizer=0.000001, **kwargs): + self.num_fields = num_fields + self.regularizer = regularizer + super(FwFMLayer, self).__init__(**kwargs) + + def build(self, input_shape): + if len(input_shape) != 3: + raise ValueError("Unexpected inputs dimensions % d,\ + expect to be 3 dimensions" % (len(input_shape))) + + if input_shape[1] != self.num_fields: + raise ValueError("Mismatch in number of fields {} and \ + concatenated embeddings dims {}".format(self.num_fields, input_shape[1])) + + self.field_strengths = self.add_weight(name='field_pair_strengths', + shape=(self.num_fields, self.num_fields), + initializer=TruncatedNormal(), + regularizer=l2(self.regularizer), + trainable=True) + + super(FwFMLayer, self).build(input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + if K.ndim(inputs) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" + % (K.ndim(inputs))) + + if inputs.shape[1] != self.num_fields: + raise ValueError("Mismatch in number of fields {} and \ + concatenated embeddings dims {}".format(self.num_fields, inputs.shape[1])) + + pairwise_inner_prods = [] + for fi, fj in itertools.combinations(range(self.num_fields), 2): + # get field strength for pair fi and fj + r_ij = self.field_strengths[fi, fj] + + # get embeddings for the features of both the fields + feat_embed_i = tf.squeeze(inputs[0:, fi:fi + 1, 0:], axis=1) + feat_embed_j = tf.squeeze(inputs[0:, fj:fj + 1, 0:], axis=1) + + f = tf.scalar_mul(r_ij, batch_dot(feat_embed_i, feat_embed_j, axes=1)) + pairwise_inner_prods.append(f) + + sum_ = tf.add_n(pairwise_inner_prods) + return sum_ + + def compute_output_shape(self, input_shape): + return (None, 1) + + def get_config(self): + config = super(FwFMLayer, self).get_config().copy() + config.update({ + 'num_fields': self.num_fields, + 'regularizer': self.regularizer + }) + return config + + +class FEFMLayer(Layer): + """Field-Embedded Factorization Machines + + Input shape + - 3D tensor with shape: ``(batch_size,field_size,embedding_size)``. + + Output shape + - 2D tensor with shape: + ``(batch_size, (num_fields * (num_fields-1))/2)`` # concatenated FEFM interaction embeddings + + Arguments + - **regularizer** : L2 regularizer weight for the field pair matrix embeddings parameters of FEFM + + References + - [Field-Embedded Factorization Machines for Click-through Rate Prediction] + https://arxiv.org/pdf/2009.09931.pdf + """ + + def __init__(self, regularizer, **kwargs): + self.regularizer = regularizer + super(FEFMLayer, self).__init__(**kwargs) + + def build(self, input_shape): + if len(input_shape) != 3: + raise ValueError("Unexpected inputs dimensions % d,\ + expect to be 3 dimensions" % (len(input_shape))) + + self.num_fields = int(input_shape[1]) + embedding_size = int(input_shape[2]) + + self.field_embeddings = {} + for fi, fj in itertools.combinations(range(self.num_fields), 2): + field_pair_id = str(fi) + "-" + str(fj) + self.field_embeddings[field_pair_id] = self.add_weight(name='field_embeddings' + field_pair_id, + shape=(embedding_size, embedding_size), + initializer=TruncatedNormal(), + regularizer=l2(self.regularizer), + trainable=True) + + super(FEFMLayer, self).build(input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + if K.ndim(inputs) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" + % (K.ndim(inputs))) + + pairwise_inner_prods = [] + for fi, fj in itertools.combinations(range(self.num_fields), 2): + field_pair_id = str(fi) + "-" + str(fj) + feat_embed_i = tf.squeeze(inputs[0:, fi:fi + 1, 0:], axis=1) + feat_embed_j = tf.squeeze(inputs[0:, fj:fj + 1, 0:], axis=1) + field_pair_embed_ij = self.field_embeddings[field_pair_id] + + feat_embed_i_tr = tf.matmul(feat_embed_i, field_pair_embed_ij + tf.transpose(field_pair_embed_ij)) + + f = batch_dot(feat_embed_i_tr, feat_embed_j, axes=1) + pairwise_inner_prods.append(f) + + concat_vec = tf.concat(pairwise_inner_prods, axis=1) + return concat_vec + + def compute_output_shape(self, input_shape): + num_fields = int(input_shape[1]) + return (None, (num_fields * (num_fields - 1)) / 2) + + def get_config(self): + config = super(FEFMLayer, self).get_config().copy() + config.update({ + 'regularizer': self.regularizer, + }) + return config + + +class BridgeModule(Layer): + """Bridge Module used in EDCN + + Input shape + - A list of two 2D tensor with shape: ``(batch_size, units)``. + + Output shape + - 2D tensor with shape: ``(batch_size, units)``. + + Arguments + - **bridge_type**: The type of bridge interaction, one of 'pointwise_addition', 'hadamard_product', 'concatenation', 'attention_pooling' + + - **activation**: Activation function to use. + + References + - [Enhancing Explicit and Implicit Feature Interactions via Information Sharing for Parallel Deep CTR Models.](https://dlp-kdd.github.io/assets/pdf/DLP-KDD_2021_paper_12.pdf) + + """ + + def __init__(self, bridge_type='hadamard_product', activation='relu', **kwargs): + self.bridge_type = bridge_type + self.activation = activation + + super(BridgeModule, self).__init__(**kwargs) + + def build(self, input_shape): + if not isinstance(input_shape, list) or len(input_shape) < 2: + raise ValueError( + 'A `BridgeModule` layer should be called ' + 'on a list of 2 inputs') + + self.dnn_dim = int(input_shape[0][-1]) + if self.bridge_type == "concatenation": + self.dense = Dense(self.dnn_dim, self.activation) + elif self.bridge_type == "attention_pooling": + self.dense_x = DNN([self.dnn_dim, self.dnn_dim], self.activation, output_activation='softmax') + self.dense_h = DNN([self.dnn_dim, self.dnn_dim], self.activation, output_activation='softmax') + + super(BridgeModule, self).build(input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + x, h = inputs + if self.bridge_type == "pointwise_addition": + return x + h + elif self.bridge_type == "hadamard_product": + return x * h + elif self.bridge_type == "concatenation": + return self.dense(tf.concat([x, h], axis=-1)) + elif self.bridge_type == "attention_pooling": + a_x = self.dense_x(x) + a_h = self.dense_h(h) + return a_x * x + a_h * h + + def compute_output_shape(self, input_shape): + return (None, self.dnn_dim) + + def get_config(self): + base_config = super(BridgeModule, self).get_config().copy() + config = { + 'bridge_type': self.bridge_type, + 'activation': self.activation + } + config.update(base_config) + return config + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/normalization.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/normalization.py new file mode 100644 index 000000000..850ac9623 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/normalization.py @@ -0,0 +1,53 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen,weichenswc@163.com + +""" +from npu_bridge.npu_init import * + +from tensorflow.python.keras import backend as K +from tensorflow.python.keras.layers import Layer + +try: + from tensorflow.python.ops.init_ops import Zeros, Ones +except ImportError: + from tensorflow.python.ops.init_ops_v2 import Zeros, Ones + + +class LayerNormalization(Layer): + def __init__(self, axis=-1, eps=1e-9, center=True, + scale=True, **kwargs): + self.axis = axis + self.eps = eps + self.center = center + self.scale = scale + super(LayerNormalization, self).__init__(**kwargs) + + def build(self, input_shape): + self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:], + initializer=Ones(), trainable=True) + self.beta = self.add_weight(name='beta', shape=input_shape[-1:], + initializer=Zeros(), trainable=True) + super(LayerNormalization, self).build(input_shape) + + def call(self, inputs): + mean = K.mean(inputs, axis=self.axis, keepdims=True) + variance = K.mean(K.square(inputs - mean), axis=-1, keepdims=True) + std = K.sqrt(variance + self.eps) + outputs = (inputs - mean) / std + if self.scale: + outputs *= self.gamma + if self.center: + outputs += self.beta + return outputs + + def compute_output_shape(self, input_shape): + return input_shape + + def get_config(self, ): + config = {'axis': self.axis, 'eps': self.eps, 'center': self.center, 'scale': self.scale} + base_config = super(LayerNormalization, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/sequence.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/sequence.py new file mode 100644 index 000000000..522daae5a --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/sequence.py @@ -0,0 +1,870 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen,weichenswc@163.com + +""" +from npu_bridge.npu_init import * + +import numpy as np +import tensorflow as tf +from tensorflow.python.keras import backend as K + +try: + from tensorflow.python.ops.init_ops import TruncatedNormal, Constant, glorot_uniform_initializer as glorot_uniform +except ImportError: + from tensorflow.python.ops.init_ops_v2 import TruncatedNormal, Constant, glorot_uniform + +from tensorflow.python.keras.layers import LSTM, Lambda, Layer, Dropout + +from .core import LocalActivationUnit +from .normalization import LayerNormalization + +if tf.__version__ >= '2.0.0': + from ..contrib.rnn_v2 import dynamic_rnn +else: + from ..contrib.rnn import dynamic_rnn +from ..contrib.utils import QAAttGRUCell, VecAttGRUCell +from .utils import reduce_sum, reduce_max, div, softmax, reduce_mean + + +class SequencePoolingLayer(Layer): + """The SequencePoolingLayer is used to apply pooling operation(sum,mean,max) on variable-length sequence feature/multi-value feature. + + Input shape + - A list of two tensor [seq_value,seq_len] + + - seq_value is a 3D tensor with shape: ``(batch_size, T, embedding_size)`` + + - seq_len is a 2D tensor with shape : ``(batch_size, 1)``,indicate valid length of each sequence. + + Output shape + - 3D tensor with shape: ``(batch_size, 1, embedding_size)``. + + Arguments + - **mode**:str.Pooling operation to be used,can be sum,mean or max. + + - **supports_masking**:If True,the input need to support masking. + """ + + def __init__(self, mode='mean', supports_masking=False, **kwargs): + + if mode not in ['sum', 'mean', 'max']: + raise ValueError("mode must be sum or mean") + self.mode = mode + self.eps = tf.constant(1e-8, tf.float32) + super(SequencePoolingLayer, self).__init__(**kwargs) + + self.supports_masking = supports_masking + + def build(self, input_shape): + if not self.supports_masking: + self.seq_len_max = int(input_shape[0][1]) + super(SequencePoolingLayer, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, seq_value_len_list, mask=None, **kwargs): + if self.supports_masking: + if mask is None: + raise ValueError( + "When supports_masking=True,input must support masking") + uiseq_embed_list = seq_value_len_list + mask = tf.cast(mask, tf.float32) # tf.to_float(mask) + user_behavior_length = reduce_sum(mask, axis=-1, keep_dims=True) + mask = tf.expand_dims(mask, axis=2) + else: + uiseq_embed_list, user_behavior_length = seq_value_len_list + + mask = tf.sequence_mask(user_behavior_length, + self.seq_len_max, dtype=tf.float32) + mask = tf.transpose(mask, (0, 2, 1)) + + embedding_size = uiseq_embed_list.shape[-1] + + mask = tf.tile(mask, [1, 1, embedding_size]) + + if self.mode == "max": + hist = uiseq_embed_list - (1 - mask) * 1e9 + return reduce_max(hist, 1, keep_dims=True) + + hist = reduce_sum(uiseq_embed_list * mask, 1, keep_dims=False) + + if self.mode == "mean": + hist = div(hist, tf.cast(user_behavior_length, tf.float32) + self.eps) + + hist = tf.expand_dims(hist, axis=1) + return hist + + def compute_output_shape(self, input_shape): + if self.supports_masking: + return (None, 1, input_shape[-1]) + else: + return (None, 1, input_shape[0][-1]) + + def compute_mask(self, inputs, mask): + return None + + def get_config(self, ): + config = {'mode': self.mode, 'supports_masking': self.supports_masking} + base_config = super(SequencePoolingLayer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class WeightedSequenceLayer(Layer): + """The WeightedSequenceLayer is used to apply weight score on variable-length sequence feature/multi-value feature. + + Input shape + - A list of two tensor [seq_value,seq_len,seq_weight] + + - seq_value is a 3D tensor with shape: ``(batch_size, T, embedding_size)`` + + - seq_len is a 2D tensor with shape : ``(batch_size, 1)``,indicate valid length of each sequence. + + - seq_weight is a 3D tensor with shape: ``(batch_size, T, 1)`` + + Output shape + - 3D tensor with shape: ``(batch_size, T, embedding_size)``. + + Arguments + - **weight_normalization**: bool.Whether normalize the weight score before applying to sequence. + + - **supports_masking**:If True,the input need to support masking. + """ + + def __init__(self, weight_normalization=True, supports_masking=False, **kwargs): + super(WeightedSequenceLayer, self).__init__(**kwargs) + self.weight_normalization = weight_normalization + self.supports_masking = supports_masking + + def build(self, input_shape): + if not self.supports_masking: + self.seq_len_max = int(input_shape[0][1]) + super(WeightedSequenceLayer, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, input_list, mask=None, **kwargs): + if self.supports_masking: + if mask is None: + raise ValueError( + "When supports_masking=True,input must support masking") + key_input, value_input = input_list + mask = tf.expand_dims(mask[0], axis=2) + else: + key_input, key_length_input, value_input = input_list + mask = tf.sequence_mask(key_length_input, + self.seq_len_max, dtype=tf.bool) + mask = tf.transpose(mask, (0, 2, 1)) + + embedding_size = key_input.shape[-1] + + if self.weight_normalization: + paddings = tf.ones_like(value_input) * (-2 ** 32 + 1) + else: + paddings = tf.zeros_like(value_input) + value_input = tf.where(mask, value_input, paddings) + + if self.weight_normalization: + value_input = softmax(value_input, dim=1) + + if len(value_input.shape) == 2: + value_input = tf.expand_dims(value_input, axis=2) + value_input = tf.tile(value_input, [1, 1, embedding_size]) + + return tf.multiply(key_input, value_input) + + def compute_output_shape(self, input_shape): + return input_shape[0] + + def compute_mask(self, inputs, mask): + if self.supports_masking: + return mask[0] + else: + return None + + def get_config(self, ): + config = {'weight_normalization': self.weight_normalization, 'supports_masking': self.supports_masking} + base_config = super(WeightedSequenceLayer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class AttentionSequencePoolingLayer(Layer): + """The Attentional sequence pooling operation used in DIN. + + Input shape + - A list of three tensor: [query,keys,keys_length] + + - query is a 3D tensor with shape: ``(batch_size, 1, embedding_size)`` + + - keys is a 3D tensor with shape: ``(batch_size, T, embedding_size)`` + + - keys_length is a 2D tensor with shape: ``(batch_size, 1)`` + + Output shape + - 3D tensor with shape: ``(batch_size, 1, embedding_size)``. + + Arguments + - **att_hidden_units**:list of positive integer, the attention net layer number and units in each layer. + + - **att_activation**: Activation function to use in attention net. + + - **weight_normalization**: bool.Whether normalize the attention score of local activation unit. + + - **supports_masking**:If True,the input need to support masking. + + References + - [Zhou G, Zhu X, Song C, et al. Deep interest network for click-through rate prediction[C]//Proceedings of the 24th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining. ACM, 2018: 1059-1068.](https://arxiv.org/pdf/1706.06978.pdf) + """ + + def __init__(self, att_hidden_units=(80, 40), att_activation='sigmoid', weight_normalization=False, + return_score=False, + supports_masking=False, **kwargs): + + self.att_hidden_units = att_hidden_units + self.att_activation = att_activation + self.weight_normalization = weight_normalization + self.return_score = return_score + super(AttentionSequencePoolingLayer, self).__init__(**kwargs) + self.supports_masking = supports_masking + + def build(self, input_shape): + if not self.supports_masking: + if not isinstance(input_shape, list) or len(input_shape) != 3: + raise ValueError('A `AttentionSequencePoolingLayer` layer should be called ' + 'on a list of 3 inputs') + + if len(input_shape[0]) != 3 or len(input_shape[1]) != 3 or len(input_shape[2]) != 2: + raise ValueError( + "Unexpected inputs dimensions,the 3 tensor dimensions are %d,%d and %d , expect to be 3,3 and 2" % ( + len(input_shape[0]), len(input_shape[1]), len(input_shape[2]))) + + if input_shape[0][-1] != input_shape[1][-1] or input_shape[0][1] != 1 or input_shape[2][1] != 1: + raise ValueError('A `AttentionSequencePoolingLayer` layer requires ' + 'inputs of a 3 tensor with shape (None,1,embedding_size),(None,T,embedding_size) and (None,1)' + 'Got different shapes: %s' % (input_shape)) + else: + pass + self.local_att = LocalActivationUnit( + self.att_hidden_units, self.att_activation, l2_reg=0, dropout_rate=0, use_bn=False, seed=1024, ) + super(AttentionSequencePoolingLayer, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, inputs, mask=None, training=None, **kwargs): + + if self.supports_masking: + if mask is None: + raise ValueError( + "When supports_masking=True,input must support masking") + queries, keys = inputs + key_masks = tf.expand_dims(mask[-1], axis=1) + + else: + + queries, keys, keys_length = inputs + hist_len = keys.get_shape()[1] + key_masks = tf.sequence_mask(keys_length, hist_len) + + attention_score = self.local_att([queries, keys], training=training) + + outputs = tf.transpose(attention_score, (0, 2, 1)) + + if self.weight_normalization: + paddings = tf.ones_like(outputs) * (-2 ** 32 + 1) + else: + paddings = tf.zeros_like(outputs) + + outputs = tf.where(key_masks, outputs, paddings) + + if self.weight_normalization: + outputs = softmax(outputs) + + if not self.return_score: + outputs = tf.matmul(outputs, keys) + + if tf.__version__ < '1.13.0': + outputs._uses_learning_phase = attention_score._uses_learning_phase + else: + outputs._uses_learning_phase = training is not None + + return outputs + + def compute_output_shape(self, input_shape): + if self.return_score: + return (None, 1, input_shape[1][1]) + else: + return (None, 1, input_shape[0][-1]) + + def compute_mask(self, inputs, mask): + return None + + def get_config(self, ): + + config = {'att_hidden_units': self.att_hidden_units, 'att_activation': self.att_activation, + 'weight_normalization': self.weight_normalization, 'return_score': self.return_score, + 'supports_masking': self.supports_masking} + base_config = super(AttentionSequencePoolingLayer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class BiLSTM(Layer): + """A multiple layer Bidirectional Residual LSTM Layer. + + Input shape + - 3D tensor with shape ``(batch_size, timesteps, input_dim)``. + + Output shape + - 3D tensor with shape: ``(batch_size, timesteps, units)``. + + Arguments + - **units**: Positive integer, dimensionality of the output space. + + - **layers**:Positive integer, number of LSTM layers to stacked. + + - **res_layers**: Positive integer, number of residual connection to used in last ``res_layers``. + + - **dropout_rate**: Float between 0 and 1. Fraction of the units to drop for the linear transformation of the inputs. + + - **merge_mode**: merge_mode: Mode by which outputs of the forward and backward RNNs will be combined. One of { ``'fw'`` , ``'bw'`` , ``'sum'`` , ``'mul'`` , ``'concat'`` , ``'ave'`` , ``None`` }. If None, the outputs will not be combined, they will be returned as a list. + + + """ + + def __init__(self, units, layers=2, res_layers=0, dropout_rate=0.2, merge_mode='ave', **kwargs): + + if merge_mode not in ['fw', 'bw', 'sum', 'mul', 'ave', 'concat', None]: + raise ValueError('Invalid merge mode. ' + 'Merge mode should be one of ' + '{"fw","bw","sum", "mul", "ave", "concat", None}') + + self.units = units + self.layers = layers + self.res_layers = res_layers + self.dropout_rate = dropout_rate + self.merge_mode = merge_mode + + super(BiLSTM, self).__init__(**kwargs) + self.supports_masking = True + + def build(self, input_shape): + + if len(input_shape) != 3: + raise ValueError( + "Unexpected inputs dimensions %d, expect to be 3 dimensions" % (len(input_shape))) + self.fw_lstm = [] + self.bw_lstm = [] + for _ in range(self.layers): + self.fw_lstm.append( + LSTM(self.units, dropout=self.dropout_rate, bias_initializer='ones', return_sequences=True, + unroll=True)) + self.bw_lstm.append( + LSTM(self.units, dropout=self.dropout_rate, bias_initializer='ones', return_sequences=True, + go_backwards=True, unroll=True)) + + super(BiLSTM, self).build( + input_shape) # Be sure to call this somewhere! + + def call(self, inputs, mask=None, **kwargs): + + input_fw = inputs + input_bw = inputs + for i in range(self.layers): + output_fw = self.fw_lstm[i](input_fw) + output_bw = self.bw_lstm[i](input_bw) + output_bw = Lambda(lambda x: K.reverse( + x, 1), mask=lambda inputs, mask: mask)(output_bw) + + if i >= self.layers - self.res_layers: + output_fw += input_fw + output_bw += input_bw + input_fw = output_fw + input_bw = output_bw + + output_fw = input_fw + output_bw = input_bw + + if self.merge_mode == "fw": + output = output_fw + elif self.merge_mode == "bw": + output = output_bw + elif self.merge_mode == 'concat': + output = tf.concat([output_fw, output_bw], axis=-1) + elif self.merge_mode == 'sum': + output = output_fw + output_bw + elif self.merge_mode == 'ave': + output = (output_fw + output_bw) / 2 + elif self.merge_mode == 'mul': + output = output_fw * output_bw + elif self.merge_mode is None: + output = [output_fw, output_bw] + + return output + + def compute_output_shape(self, input_shape): + print(self.merge_mode) + if self.merge_mode is None: + return [input_shape, input_shape] + elif self.merge_mode == 'concat': + return input_shape[:-1] + (input_shape[-1] * 2,) + else: + return input_shape + + def compute_mask(self, inputs, mask): + return mask + + def get_config(self, ): + + config = {'units': self.units, 'layers': self.layers, + 'res_layers': self.res_layers, 'dropout_rate': self.dropout_rate, 'merge_mode': self.merge_mode} + base_config = super(BiLSTM, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class Transformer(Layer): + """ Simplified version of Transformer proposed in 《Attention is all you need》 + + Input shape + - a list of two 3D tensor with shape ``(batch_size, timesteps, input_dim)`` if ``supports_masking=True`` . + - a list of two 4 tensors, first two tensors with shape ``(batch_size, timesteps, input_dim)``,last two tensors with shape ``(batch_size, 1)`` if ``supports_masking=False`` . + + + Output shape + - 3D tensor with shape: ``(batch_size, 1, input_dim)`` if ``output_type='mean'`` or ``output_type='sum'`` , else ``(batch_size, timesteps, input_dim)`` . + + + Arguments + - **att_embedding_size**: int.The embedding size in multi-head self-attention network. + - **head_num**: int.The head number in multi-head self-attention network. + - **dropout_rate**: float between 0 and 1. Fraction of the units to drop. + - **use_positional_encoding**: bool. Whether or not use positional_encoding + - **use_res**: bool. Whether or not use standard residual connections before output. + - **use_feed_forward**: bool. Whether or not use pointwise feed foward network. + - **use_layer_norm**: bool. Whether or not use Layer Normalization. + - **blinding**: bool. Whether or not use blinding. + - **seed**: A Python integer to use as random seed. + - **supports_masking**:bool. Whether or not support masking. + - **attention_type**: str, Type of attention, the value must be one of { ``'scaled_dot_product'`` , ``'cos'`` , ``'ln'`` , ``'additive'`` }. + - **output_type**: ``'mean'`` , ``'sum'`` or `None`. Whether or not use average/sum pooling for output. + + References + - [Vaswani, Ashish, et al. "Attention is all you need." Advances in Neural Information Processing Systems. 2017.](https://papers.nips.cc/paper/7181-attention-is-all-you-need.pdf) + """ + + def __init__(self, att_embedding_size=1, head_num=8, dropout_rate=0.0, use_positional_encoding=True, use_res=True, + use_feed_forward=True, use_layer_norm=False, blinding=True, seed=1024, supports_masking=False, + attention_type="scaled_dot_product", output_type="mean", **kwargs): + if head_num <= 0: + raise ValueError('head_num must be a int > 0') + self.att_embedding_size = att_embedding_size + self.head_num = head_num + self.num_units = att_embedding_size * head_num + self.use_res = use_res + self.use_feed_forward = use_feed_forward + self.seed = seed + self.use_positional_encoding = use_positional_encoding + self.dropout_rate = dropout_rate + self.use_layer_norm = use_layer_norm + self.blinding = blinding + self.attention_type = attention_type + self.output_type = output_type + super(Transformer, self).__init__(**kwargs) + self.supports_masking = supports_masking + + def build(self, input_shape): + embedding_size = int(input_shape[0][-1]) + if self.num_units != embedding_size: + raise ValueError( + "att_embedding_size * head_num must equal the last dimension size of inputs,got %d * %d != %d" % ( + self.att_embedding_size, self.head_num, embedding_size)) + self.seq_len_max = int(input_shape[0][-2]) + self.W_Query = self.add_weight(name='query', shape=[embedding_size, self.att_embedding_size * self.head_num], + dtype=tf.float32, + initializer=TruncatedNormal(seed=self.seed)) + self.W_key = self.add_weight(name='key', shape=[embedding_size, self.att_embedding_size * self.head_num], + dtype=tf.float32, + initializer=TruncatedNormal(seed=self.seed + 1)) + self.W_Value = self.add_weight(name='value', shape=[embedding_size, self.att_embedding_size * self.head_num], + dtype=tf.float32, + initializer=TruncatedNormal(seed=self.seed + 2)) + if self.attention_type == "additive": + self.b = self.add_weight('b', shape=[self.att_embedding_size], dtype=tf.float32, + initializer=glorot_uniform(seed=self.seed)) + self.v = self.add_weight('v', shape=[self.att_embedding_size], dtype=tf.float32, + initializer=glorot_uniform(seed=self.seed)) + elif self.attention_type == "ln": + self.att_ln_q = LayerNormalization() + self.att_ln_k = LayerNormalization() + # if self.use_res: + # self.W_Res = self.add_weight(name='res', shape=[embedding_size, self.att_embedding_size * self.head_num], dtype=tf.float32, + # initializer=TruncatedNormal(seed=self.seed)) + if self.use_feed_forward: + self.fw1 = self.add_weight('fw1', shape=[self.num_units, 4 * self.num_units], dtype=tf.float32, + initializer=glorot_uniform(seed=self.seed)) + self.fw2 = self.add_weight('fw2', shape=[4 * self.num_units, self.num_units], dtype=tf.float32, + initializer=glorot_uniform(seed=self.seed)) + + self.dropout = Dropout( + self.dropout_rate, seed=self.seed) + self.ln = LayerNormalization() + if self.use_positional_encoding: + self.query_pe = PositionEncoding() + self.key_pe = PositionEncoding() + # Be sure to call this somewhere! + super(Transformer, self).build(input_shape) + + def call(self, inputs, mask=None, training=None, **kwargs): + + if self.supports_masking: + queries, keys = inputs + query_masks, key_masks = mask + query_masks = tf.cast(query_masks, tf.float32) + key_masks = tf.cast(key_masks, tf.float32) + else: + queries, keys, query_masks, key_masks = inputs + + query_masks = tf.sequence_mask( + query_masks, self.seq_len_max, dtype=tf.float32) + key_masks = tf.sequence_mask( + key_masks, self.seq_len_max, dtype=tf.float32) + query_masks = tf.squeeze(query_masks, axis=1) + key_masks = tf.squeeze(key_masks, axis=1) + + if self.use_positional_encoding: + queries = self.query_pe(queries) + keys = self.key_pe(keys) + + Q = tf.tensordot(queries, self.W_Query, + axes=(-1, 0)) # N T_q D*h + K = tf.tensordot(keys, self.W_key, axes=(-1, 0)) + V = tf.tensordot(keys, self.W_Value, axes=(-1, 0)) + + # h*N T_q D + Q_ = tf.concat(tf.split(Q, self.head_num, axis=2), axis=0) + K_ = tf.concat(tf.split(K, self.head_num, axis=2), axis=0) + V_ = tf.concat(tf.split(V, self.head_num, axis=2), axis=0) + + if self.attention_type == "scaled_dot_product": + # h*N T_q T_k + outputs = tf.matmul(Q_, K_, transpose_b=True) + + outputs = outputs / (K_.get_shape().as_list()[-1] ** 0.5) + elif self.attention_type == "cos": + Q_cos = tf.nn.l2_normalize(Q_, dim=-1) + K_cos = tf.nn.l2_normalize(K_, dim=-1) + + outputs = tf.matmul(Q_cos, K_cos, transpose_b=True) # h*N T_q T_k + + outputs = outputs * 20 # Scale + elif self.attention_type == 'ln': + Q_ = self.att_ln_q(Q_) + K_ = self.att_ln_k(K_) + + outputs = tf.matmul(Q_, K_, transpose_b=True) # h*N T_q T_k + # Scale + outputs = outputs / (K_.get_shape().as_list()[-1] ** 0.5) + elif self.attention_type == "additive": + Q_reshaped = tf.expand_dims(Q_, axis=-2) + K_reshaped = tf.expand_dims(K_, axis=-3) + outputs = tf.tanh(tf.nn.bias_add(Q_reshaped + K_reshaped, self.b)) + outputs = tf.squeeze(tf.tensordot(outputs, tf.expand_dims(self.v, axis=-1), axes=[-1, 0]), axis=-1) + else: + raise ValueError("attention_type must be [scaled_dot_product,cos,ln,additive]") + + key_masks = tf.tile(key_masks, [self.head_num, 1]) + + # (h*N, T_q, T_k) + key_masks = tf.tile(tf.expand_dims(key_masks, 1), + [1, tf.shape(queries)[1], 1]) + + paddings = tf.ones_like(outputs) * (-2 ** 32 + 1) + + # (h*N, T_q, T_k) + + outputs = tf.where(tf.equal(key_masks, 1), outputs, paddings, ) + if self.blinding: + try: + outputs = tf.matrix_set_diag(outputs, tf.ones_like(outputs)[ + :, :, 0] * (-2 ** 32 + 1)) + except AttributeError: + outputs = tf.compat.v1.matrix_set_diag(outputs, tf.ones_like(outputs)[ + :, :, 0] * (-2 ** 32 + 1)) + + outputs -= reduce_max(outputs, axis=-1, keep_dims=True) + outputs = softmax(outputs) + query_masks = tf.tile(query_masks, [self.head_num, 1]) # (h*N, T_q) + # (h*N, T_q, T_k) + query_masks = tf.tile(tf.expand_dims( + query_masks, -1), [1, 1, tf.shape(keys)[1]]) + + outputs *= query_masks + + outputs = self.dropout(outputs, training=training) + # Weighted sum + # ( h*N, T_q, C/h) + result = tf.matmul(outputs, V_) + result = tf.concat(tf.split(result, self.head_num, axis=0), axis=2) + + if self.use_res: + # tf.tensordot(queries, self.W_Res, axes=(-1, 0)) + result += queries + if self.use_layer_norm: + result = self.ln(result) + + if self.use_feed_forward: + fw1 = tf.nn.relu(tf.tensordot(result, self.fw1, axes=[-1, 0])) + fw1 = self.dropout(fw1, training=training) + fw2 = tf.tensordot(fw1, self.fw2, axes=[-1, 0]) + if self.use_res: + result += fw2 + if self.use_layer_norm: + result = self.ln(result) + + if self.output_type == "mean": + return reduce_mean(result, axis=1, keep_dims=True) + elif self.output_type == "sum": + return reduce_sum(result, axis=1, keep_dims=True) + else: + return result + + def compute_output_shape(self, input_shape): + + return (None, 1, self.att_embedding_size * self.head_num) + + def compute_mask(self, inputs, mask=None): + return None + + def get_config(self, ): + config = {'att_embedding_size': self.att_embedding_size, 'head_num': self.head_num, + 'dropout_rate': self.dropout_rate, 'use_res': self.use_res, + 'use_positional_encoding': self.use_positional_encoding, 'use_feed_forward': self.use_feed_forward, + 'use_layer_norm': self.use_layer_norm, 'seed': self.seed, 'supports_masking': self.supports_masking, + 'blinding': self.blinding, 'attention_type': self.attention_type, 'output_type': self.output_type} + base_config = super(Transformer, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class PositionEncoding(Layer): + def __init__(self, pos_embedding_trainable=True, + zero_pad=False, + scale=True, **kwargs): + self.pos_embedding_trainable = pos_embedding_trainable + self.zero_pad = zero_pad + self.scale = scale + super(PositionEncoding, self).__init__(**kwargs) + + def build(self, input_shape): + # Create a trainable weight variable for this layer. + _, T, num_units = input_shape.as_list() # inputs.get_shape().as_list() + # First part of the PE function: sin and cos argument + position_enc = np.array([ + [pos / np.power(10000, 2. * (i // 2) / num_units) for i in range(num_units)] + for pos in range(T)]) + + # Second part, apply the cosine to even columns and sin to odds. + position_enc[:, 0::2] = np.sin(position_enc[:, 0::2]) # dim 2i + position_enc[:, 1::2] = np.cos(position_enc[:, 1::2]) # dim 2i+1 + if self.zero_pad: + position_enc[0, :] = np.zeros(num_units) + self.lookup_table = self.add_weight("lookup_table", (T, num_units), + initializer=Constant(position_enc), + trainable=self.pos_embedding_trainable) + + # Be sure to call this somewhere! + super(PositionEncoding, self).build(input_shape) + + def call(self, inputs, mask=None): + _, T, num_units = inputs.get_shape().as_list() + position_ind = tf.expand_dims(tf.range(T), 0) + outputs = tf.nn.embedding_lookup(self.lookup_table, position_ind) + if self.scale: + outputs = outputs * num_units ** 0.5 + return outputs + inputs + + def compute_output_shape(self, input_shape): + + return input_shape + + def compute_mask(self, inputs, mask=None): + return mask + + def get_config(self, ): + + config = {'pos_embedding_trainable': self.pos_embedding_trainable, 'zero_pad': self.zero_pad, + 'scale': self.scale} + base_config = super(PositionEncoding, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class BiasEncoding(Layer): + def __init__(self, sess_max_count, seed=1024, **kwargs): + self.sess_max_count = sess_max_count + self.seed = seed + super(BiasEncoding, self).__init__(**kwargs) + + def build(self, input_shape): + # Create a trainable weight variable for this layer. + + if self.sess_max_count == 1: + embed_size = input_shape[2].value + seq_len_max = input_shape[1].value + else: + try: + embed_size = input_shape[0][2].value + seq_len_max = input_shape[0][1].value + except AttributeError: + embed_size = input_shape[0][2] + seq_len_max = input_shape[0][1] + + self.sess_bias_embedding = self.add_weight('sess_bias_embedding', shape=(self.sess_max_count, 1, 1), + initializer=TruncatedNormal( + mean=0.0, stddev=0.0001, seed=self.seed)) + self.seq_bias_embedding = self.add_weight('seq_bias_embedding', shape=(1, seq_len_max, 1), + initializer=TruncatedNormal( + mean=0.0, stddev=0.0001, seed=self.seed)) + self.item_bias_embedding = self.add_weight('item_bias_embedding', shape=(1, 1, embed_size), + initializer=TruncatedNormal( + mean=0.0, stddev=0.0001, seed=self.seed)) + + # Be sure to call this somewhere! + super(BiasEncoding, self).build(input_shape) + + def call(self, inputs, mask=None): + """ + :param concated_embeds_value: None * field_size * embedding_size + :return: None*1 + """ + transformer_out = [] + for i in range(self.sess_max_count): + transformer_out.append( + inputs[i] + self.item_bias_embedding + self.seq_bias_embedding + self.sess_bias_embedding[i]) + return transformer_out + + def compute_output_shape(self, input_shape): + + return input_shape + + def compute_mask(self, inputs, mask=None): + return mask + + def get_config(self, ): + + config = {'sess_max_count': self.sess_max_count, 'seed': self.seed, } + base_config = super(BiasEncoding, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class DynamicGRU(Layer): + def __init__(self, num_units=None, gru_type='GRU', return_sequence=True, **kwargs): + + self.num_units = num_units + self.return_sequence = return_sequence + self.gru_type = gru_type + super(DynamicGRU, self).__init__(**kwargs) + + def build(self, input_shape): + # Create a trainable weight variable for this layer. + input_seq_shape = input_shape[0] + if self.num_units is None: + self.num_units = input_seq_shape.as_list()[-1] + if self.gru_type == "AGRU": + self.gru_cell = QAAttGRUCell(self.num_units) + elif self.gru_type == "AUGRU": + self.gru_cell = VecAttGRUCell(self.num_units) + else: + try: + self.gru_cell = tf.nn.rnn_cell.GRUCell(self.num_units) # GRUCell + except AttributeError: + self.gru_cell = tf.compat.v1.nn.rnn_cell.GRUCell(self.num_units) + + # Be sure to call this somewhere! + super(DynamicGRU, self).build(input_shape) + + def call(self, input_list): + """ + :param concated_embeds_value: None * field_size * embedding_size + :return: None*1 + """ + if self.gru_type == "GRU" or self.gru_type == "AIGRU": + rnn_input, sequence_length = input_list + att_score = None + else: + rnn_input, sequence_length, att_score = input_list + + rnn_output, hidden_state = dynamic_rnn(self.gru_cell, inputs=rnn_input, att_scores=att_score, + sequence_length=tf.squeeze(sequence_length, + ), dtype=tf.float32, scope=self.name) + if self.return_sequence: + return rnn_output + else: + return tf.expand_dims(hidden_state, axis=1) + + def compute_output_shape(self, input_shape): + rnn_input_shape = input_shape[0] + if self.return_sequence: + return rnn_input_shape + else: + return (None, 1, rnn_input_shape[2]) + + def get_config(self, ): + config = {'num_units': self.num_units, 'gru_type': self.gru_type, 'return_sequence': self.return_sequence} + base_config = super(DynamicGRU, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class KMaxPooling(Layer): + """K Max pooling that selects the k biggest value along the specific axis. + + Input shape + - nD tensor with shape: ``(batch_size, ..., input_dim)``. + + Output shape + - nD tensor with shape: ``(batch_size, ..., output_dim)``. + + Arguments + - **k**: positive integer, number of top elements to look for along the ``axis`` dimension. + + - **axis**: positive integer, the dimension to look for elements. + + """ + + def __init__(self, k=1, axis=-1, **kwargs): + + self.k = k + self.axis = axis + super(KMaxPooling, self).__init__(**kwargs) + + def build(self, input_shape): + + if self.axis < 1 or self.axis > len(input_shape): + raise ValueError("axis must be 1~%d,now is %d" % + (len(input_shape), self.axis)) + + if self.k < 1 or self.k > input_shape[self.axis]: + raise ValueError("k must be in 1 ~ %d,now k is %d" % + (input_shape[self.axis], self.k)) + self.dims = len(input_shape) + # Be sure to call this somewhere! + super(KMaxPooling, self).build(input_shape) + + def call(self, inputs): + + # swap the last and the axis dimensions since top_k will be applied along the last dimension + perm = list(range(self.dims)) + perm[-1], perm[self.axis] = perm[self.axis], perm[-1] + shifted_input = tf.transpose(inputs, perm) + + # extract top_k, returns two tensors [values, indices] + top_k = tf.nn.top_k(shifted_input, k=self.k, sorted=True, name=None)[0] + output = tf.transpose(top_k, perm) + + return output + + def compute_output_shape(self, input_shape): + output_shape = list(input_shape) + output_shape[self.axis] = self.k + return tuple(output_shape) + + def get_config(self, ): + config = {'k': self.k, 'axis': self.axis} + base_config = super(KMaxPooling, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/utils.py b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/utils.py new file mode 100644 index 000000000..8efc593f1 --- /dev/null +++ b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/utils.py @@ -0,0 +1,348 @@ +# -*- coding:utf-8 -*- +""" + +Author: + Weichen Shen,weichenswc@163.com + +""" +from npu_bridge.npu_init import * +import tensorflow as tf +from tensorflow.python.keras import backend as K +from tensorflow.python.keras.layers import Flatten, Layer, Add +from tensorflow.python.ops.lookup_ops import TextFileInitializer + +try: + from tensorflow.python.ops.init_ops import Zeros, glorot_normal_initializer as glorot_normal +except ImportError: + from tensorflow.python.ops.init_ops_v2 import Zeros, glorot_normal + +from tensorflow.python.keras.regularizers import l2 + +try: + from tensorflow.python.ops.lookup_ops import StaticHashTable +except ImportError: + from tensorflow.python.ops.lookup_ops import HashTable as StaticHashTable + + +class NoMask(Layer): + def __init__(self, **kwargs): + super(NoMask, self).__init__(**kwargs) + + def build(self, input_shape): + # Be sure to call this somewhere! + super(NoMask, self).build(input_shape) + + def call(self, x, mask=None, **kwargs): + return x + + def compute_mask(self, inputs, mask): + return None + + +class Hash(Layer): + """Looks up keys in a table when setup `vocabulary_path`, which outputs the corresponding values. + If `vocabulary_path` is not set, `Hash` will hash the input to [0,num_buckets). When `mask_zero` = True, + input value `0` or `0.0` will be set to `0`, and other value will be set in range [1,num_buckets). + + The following snippet initializes a `Hash` with `vocabulary_path` file with the first column as keys and + second column as values: + + * `1,emerson` + * `2,lake` + * `3,palmer` + + >>> hash = Hash( + ... num_buckets=3+1, + ... vocabulary_path=filename, + ... default_value=0) + >>> hash(tf.constant('lake')).numpy() + 2 + >>> hash(tf.constant('lakeemerson')).numpy() + 0 + + Args: + num_buckets: An `int` that is >= 1. The number of buckets or the vocabulary size + 1 + when `vocabulary_path` is setup. + mask_zero: default is False. The `Hash` value will hash input `0` or `0.0` to value `0` when + the `mask_zero` is `True`. `mask_zero` is not used when `vocabulary_path` is setup. + vocabulary_path: default `None`. The `CSV` text file path of the vocabulary hash, which contains + two columns seperated by delimiter `comma`, the first column is the value and the second is + the key. The key data type is `string`, the value data type is `int`. The path must + be accessible from wherever `Hash` is initialized. + default_value: default '0'. The default value if a key is missing in the table. + **kwargs: Additional keyword arguments. + """ + + def __init__(self, num_buckets, mask_zero=False, vocabulary_path=None, default_value=0, **kwargs): + self.num_buckets = num_buckets + self.mask_zero = mask_zero + self.vocabulary_path = vocabulary_path + self.default_value = default_value + if self.vocabulary_path: + initializer = TextFileInitializer(vocabulary_path, 'string', 1, 'int64', 0, delimiter=',') + self.hash_table = StaticHashTable(initializer, default_value=self.default_value) + super(Hash, self).__init__(**kwargs) + + def build(self, input_shape): + # Be sure to call this somewhere! + super(Hash, self).build(input_shape) + + def call(self, x, mask=None, **kwargs): + + if x.dtype != tf.string: + zero = tf.as_string(tf.zeros([1], dtype=x.dtype)) + x = tf.as_string(x, ) + else: + zero = tf.as_string(tf.zeros([1], dtype='int32')) + + if self.vocabulary_path: + hash_x = self.hash_table.lookup(x) + return hash_x + + num_buckets = self.num_buckets if not self.mask_zero else self.num_buckets - 1 + try: + hash_x = tf.string_to_hash_bucket_fast(x, num_buckets, + name=None) # weak hash + except AttributeError: + hash_x = tf.strings.to_hash_bucket_fast(x, num_buckets, + name=None) # weak hash + if self.mask_zero: + mask = tf.cast(tf.not_equal(x, zero), dtype='int64') + hash_x = (hash_x + 1) * mask + + return hash_x + + def compute_output_shape(self, input_shape): + return input_shape + + def get_config(self, ): + config = {'num_buckets': self.num_buckets, 'mask_zero': self.mask_zero, 'vocabulary_path': self.vocabulary_path, + 'default_value': self.default_value} + base_config = super(Hash, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class Linear(Layer): + + def __init__(self, l2_reg=0.0, mode=0, use_bias=False, seed=1024, **kwargs): + + self.l2_reg = l2_reg + # self.l2_reg = tf.contrib.layers.l2_regularizer(float(l2_reg_linear)) + if mode not in [0, 1, 2]: + raise ValueError("mode must be 0,1 or 2") + self.mode = mode + self.use_bias = use_bias + self.seed = seed + super(Linear, self).__init__(**kwargs) + + def build(self, input_shape): + if self.use_bias: + self.bias = self.add_weight(name='linear_bias', + shape=(1,), + initializer=Zeros(), + trainable=True) + if self.mode == 1: + self.kernel = self.add_weight( + 'linear_kernel', + shape=[int(input_shape[-1]), 1], + initializer=glorot_normal(self.seed), + regularizer=l2(self.l2_reg), + trainable=True) + elif self.mode == 2: + self.kernel = self.add_weight( + 'linear_kernel', + shape=[int(input_shape[1][-1]), 1], + initializer=glorot_normal(self.seed), + regularizer=l2(self.l2_reg), + trainable=True) + + super(Linear, self).build(input_shape) # Be sure to call this somewhere! + + def call(self, inputs, **kwargs): + if self.mode == 0: + sparse_input = inputs + linear_logit = reduce_sum(sparse_input, axis=-1, keep_dims=True) + elif self.mode == 1: + dense_input = inputs + fc = tf.tensordot(dense_input, self.kernel, axes=(-1, 0)) + linear_logit = fc + else: + sparse_input, dense_input = inputs + fc = tf.tensordot(dense_input, self.kernel, axes=(-1, 0)) + linear_logit = reduce_sum(sparse_input, axis=-1, keep_dims=False) + fc + if self.use_bias: + linear_logit += self.bias + + return linear_logit + + def compute_output_shape(self, input_shape): + return (None, 1) + + def compute_mask(self, inputs, mask): + return None + + def get_config(self, ): + config = {'mode': self.mode, 'l2_reg': self.l2_reg, 'use_bias': self.use_bias, 'seed': self.seed} + base_config = super(Linear, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class Concat(Layer): + def __init__(self, axis, supports_masking=True, **kwargs): + super(Concat, self).__init__(**kwargs) + self.axis = axis + self.supports_masking = supports_masking + + def call(self, inputs): + return tf.concat(inputs, axis=self.axis) + + def compute_mask(self, inputs, mask=None): + if not self.supports_masking: + return None + if mask is None: + mask = [inputs_i._keras_mask if hasattr(inputs_i, "_keras_mask") else None for inputs_i in inputs] + if mask is None: + return None + if not isinstance(mask, list): + raise ValueError('`mask` should be a list.') + if not isinstance(inputs, list): + raise ValueError('`inputs` should be a list.') + if len(mask) != len(inputs): + raise ValueError('The lists `inputs` and `mask` ' + 'should have the same length.') + if all([m is None for m in mask]): + return None + # Make a list of masks while making sure + # the dimensionality of each mask + # is the same as the corresponding input. + masks = [] + for input_i, mask_i in zip(inputs, mask): + if mask_i is None: + # Input is unmasked. Append all 1s to masks, + masks.append(tf.ones_like(input_i, dtype='bool')) + elif K.ndim(mask_i) < K.ndim(input_i): + # Mask is smaller than the input, expand it + masks.append(tf.expand_dims(mask_i, axis=-1)) + else: + masks.append(mask_i) + concatenated = K.concatenate(masks, axis=self.axis) + return K.all(concatenated, axis=-1, keepdims=False) + + def get_config(self, ): + config = {'axis': self.axis, 'supports_masking': self.supports_masking} + base_config = super(Concat, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +def concat_func(inputs, axis=-1, mask=False): + if len(inputs) == 1: + input = inputs[0] + if not mask: + input = NoMask()(input) + return input + return Concat(axis, supports_masking=mask)(inputs) + + +def reduce_mean(input_tensor, + axis=None, + keep_dims=False, + name=None, + reduction_indices=None): + try: + return tf.reduce_mean(input_tensor, + axis=axis, + keep_dims=keep_dims, + name=name, + reduction_indices=reduction_indices) + except TypeError: + return tf.reduce_mean(input_tensor, + axis=axis, + keepdims=keep_dims, + name=name) + + +def reduce_sum(input_tensor, + axis=None, + keep_dims=False, + name=None, + reduction_indices=None): + try: + return tf.reduce_sum(input_tensor, + axis=axis, + keep_dims=keep_dims, + name=name, + reduction_indices=reduction_indices) + except TypeError: + return tf.reduce_sum(input_tensor, + axis=axis, + keepdims=keep_dims, + name=name) + + +def reduce_max(input_tensor, + axis=None, + keep_dims=False, + name=None, + reduction_indices=None): + try: + return tf.reduce_max(input_tensor, + axis=axis, + keep_dims=keep_dims, + name=name, + reduction_indices=reduction_indices) + except TypeError: + return tf.reduce_max(input_tensor, + axis=axis, + keepdims=keep_dims, + name=name) + + +def div(x, y, name=None): + try: + return tf.div(x, y, name=name) + except AttributeError: + return tf.divide(x, y, name=name) + + +def softmax(logits, dim=-1, name=None): + try: + return tf.nn.softmax(logits, dim=dim, name=name) + except TypeError: + return tf.nn.softmax(logits, axis=dim, name=name) + + +class _Add(Layer): + def __init__(self, **kwargs): + super(_Add, self).__init__(**kwargs) + + def build(self, input_shape): + # Be sure to call this somewhere! + super(_Add, self).build(input_shape) + + def call(self, inputs, **kwargs): + if len(inputs) == 0: + return tf.constant([[0.0]]) + + return Add()(inputs) + + +def add_func(inputs): + if not isinstance(inputs, list): + return inputs + if len(inputs) == 1: + return inputs[0] + return _Add()(inputs) + + +def combined_dnn_input(sparse_embedding_list, dense_value_list): + if len(sparse_embedding_list) > 0 and len(dense_value_list) > 0: + sparse_dnn_input = Flatten()(concat_func(sparse_embedding_list)) + dense_dnn_input = Flatten()(concat_func(dense_value_list)) + return concat_func([sparse_dnn_input, dense_dnn_input]) + elif len(sparse_embedding_list) > 0: + return Flatten()(concat_func(sparse_embedding_list)) + elif len(dense_value_list) > 0: + return Flatten()(concat_func(dense_value_list)) + else: + raise NotImplementedError("dnn_feature_columns can not be empty list") + -- Gitee From cf2a0e022afe3d350c60ace525303def71e70a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:38:26 +0000 Subject: [PATCH 22/65] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20=5F=5Fpycache=5F=5F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/.keep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/.keep diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/.keep b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/.keep new file mode 100644 index 000000000..e69de29bb -- Gitee From 76d309c8cb01a3c2b62abc8ae7e93f4f9eeeb93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8B=8F=E7=88=B1=E4=B8=9C?= <384169931@qq.com> Date: Mon, 8 May 2023 12:38:41 +0000 Subject: [PATCH 23/65] ADD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 苏爱东 <384169931@qq.com> --- .../layers/__pycache__/__init__.cpython-37.pyc | Bin 0 -> 1603 bytes .../__pycache__/activation.cpython-37.pyc | Bin 0 -> 3405 bytes .../layers/__pycache__/core.cpython-37.pyc | Bin 0 -> 12287 bytes .../__pycache__/interaction.cpython-37.pyc | Bin 0 -> 51967 bytes .../__pycache__/normalization.cpython-37.pyc | Bin 0 -> 2151 bytes .../layers/__pycache__/sequence.cpython-37.pyc | Bin 0 -> 28266 bytes .../layers/__pycache__/utils.cpython-37.pyc | Bin 0 -> 11191 bytes 7 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/__init__.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/activation.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/core.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/interaction.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/normalization.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/sequence.cpython-37.pyc create mode 100644 TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/utils.cpython-37.pyc diff --git a/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/__init__.cpython-37.pyc b/TensorFlow/built-in/recommendation/DIN_ID0190_for_TensorFlow/deepctr/layers/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7435a4106ed0707d64dd64736b96e5b309758379 GIT binary patch literal 1603 zcmb`H-BR0D5XUWl%CZf{HeVrpLyQPuAZaI^>2!(#Crq)+0GZ638;>F_=CoQJD#?Yk z573wB3-p!FZLjhQz3SN`95*w$tTg`3`e?P<|6i-qu2ig)I!^z%@_tv*-}Gm6Y^mUT z^^10lRLVsvzTK!%(FZeSOHCD>h+voM2lHiVkIoIGFDgx zC)fl|vPrD6Do(K}tg#x_Ssfd!fzxanXV?tRvRRyCb2!iDb)CHb1Q*zXjtl-%Tx5&5 z#FlWGE#nGX!Bw`3YiteI**b2p4cug#I@k1{VUsm+i*2FJY}{ts_?$h*9kzqJY!~;~ zo{<6rs!)R*6rcoUn1D&BKprx%3X5RD6fD3ptV0u)U0^@ z4t8M^OqlnIe>PdmGpho=Wes$fIr60P^!^o1x7jK+?2YhfF zgEzXG)GJ+wP7vLwCe)j-PCf(3xYlm?uM}}MB*k!=RsQPl@DF$M?;kPV-dJcY&T^(z6mGP*o0L= zmXIeD3G}UoWkQ8eBIF1e!XjawU=k(?bA$!LGGUGIgz%IwMOY_n5NyHN`Ao)Lw<@!)2q96Y`_~ewge>v# zJSRHF%C+9Hd~K6fslC!-#*L5L)typnvJ(7vu(UbSj2=fSu}+(#4bu*3PqhcyzZF$Q z>y^1-%>5`abnKuu>KP?zkF-7Hs+@3#Q7o`8-v0z{9Q~CFUr-j|pNrP5Kxjqx4`09f z=H=l*Yj8JgodDb&UtjK`6RFR!uSTbTxPSO!|Ev9%2d@qf_77X0%5dYb<YByONmQv zXNHm`tS^-TpVC`T?ZLhj=>L(wp$A@jXmjhim-hD!S1UO!N`k}T%*XHXoA3M0d@?uZ zTeu#++TjmpE$eUeae6h;`52G-77e!oi`(2u?Pv6^rA|;gT0qZD-JutFw)L5nG1n6H zeP3?b7TspHQO$l~WzOd`;-$@DD`*Y;Hqr31(jaPkvwL%Un63Ul zYKnOy_b%ivz|x;5atpzHC2`;44c>fd@up~<)Pjq=#r>C7aEaG;toF=t*Y`IHJ;>!< zpW*sUB=JCGYzNKKVfCR7L^>X9=W-aO$*~!Y>G#viopjV=bc7Yyz|`S(P~#U~Sv?@hy_c=PHT^no zyle&@UoigLL7mU=S;WyWvvYhNv&~nQYXvRDb+L4}l2}B4tXbBxfnZyaj@Sl|M(|Fs zGbh=-B7?_y#*&O}J=k7493XxcM;Y4{>>v?`g0o1mi1kG#XT#p8lS88_n5 zM0t9|BEDBB%{1&v0qwm6N=A8-X%@wCE_ssmna&wcl$ObEVeCU;l8p+@I9%=cW-%Ch zV20En8i~mWV>j4_+)cEMSuV1t4x)`N`yoMK| zFQi&xJQs>(xyDeO7QE7ned?4dWl4o)ez8&*f8fJ zox1+3LQ^>3_kAZCifZswk8yPR+r}gSbuQk;MRI3xk=+F##{DJZ0ga*)k(IniIjvct z05-ryvckMDaK4T7b$v8S;wZ&RnJ`483wavy)DOaWaZ!v^lIEE)y@M=HSuf3DFXFs1 za}dcS%3^?*v*;jCIBXpxW`%c;*fFYOs`07l2`M0ZKC~|bc+T!Gv0zZJ&zIOvj@;N> zVgfeO4%-r9WR?sLYL*F&Mc!wU@}6eBPn=dviZx+ue#k&WgZKR zWPL>oPqD7v(gU`!`GoB}xc_Kt%b+PUQ|S>~-;|IvUHa<364vs_LBj7dteAi{2zf zNT_Wz+8SGXKw=LZiQDYlJej(N?C=eh$LlO z2p5!SJ93^hzy(xy#cpj|F3`UhskubWH8i*IDAH-!j(x>mu;=j{`vz*Ge|ws4O(AC%otjfGO@pZ2({;&IP$s%p`6$lzowDvx{k5C7$M17-9YABKfwI&Y?DRE zc*g)OjLqXG53&Pr0*)Y4!dvR)1&&AWsm>SJ`&_X{n zoP36ExU<(-u7`f9S5zvjnVdH*}<#4Ozgia6Ks+l>Jl%Oz80Cv zCFt$YlznjRYydj|j@nr#AE}PnjnJtmapu8rluK5PD*$a~% Ly$xiwh37)TcJA)!`D^}4qJAxV%Z}~U2-NULURxF%Te4``ie*}{xR!Uk-eGgPXEZ6iYRWP2`{kFny$j2R@$Y=;{W8d0!4v*363-|Yp6OYE zxyJvAz$zs0-wy0rvXC;3`={`h4AQksAydm1vNfmRn8qEWe$F=hiET%&n+E^TDiuuj zTTQXmkBqwY32V~9bZw?EQ=2W!qQ_a3WKeRXa76X=T(_rv-IJf#I=Xdi3$^|4v%y^L zc;PrkalDD(MD1kZq-nfsc$41L1H+s0XZ8|>Q+S{DX7E0v-cRFw);ogtBkKJO-j906 z@P16aKjS5C8@ajmg5z9mMVk$I*%A2N@vEgxzbua9u=bF8!wTL%$xW35iBotqMMn_#pC87lq=kE31AeT+eM{>V9xvRPe7ZylT0O zCiO^^+`3ow+{hQz$ghQYM|C5_je4^ciEz_x`rQ&CE{LllsD=>+6VXmXTwD>6Uk@8n z)213<7Giz<+yi=1l<; zbw3hHmPEaUNlG;_)d`o)cd=Nix<5L(c>erA5;2w-2!trN>W~g}95uN3?Bu{;BmD%+ zMd?@AM>%M?k=XE~9UtqvsOVVAi<`?3lu9dr9AMvwFv6l&AV+Sq8B`%#7>2^M?aR6! z2yElZCd8)ZMzVU3%lvhNdwS~?C8TFS0x14;aegV67uO}HiMdc*Fo%q z#f>_9-e@#}ytuRJqZ_ICa(J&}?&XH`#elT}f?01wPa7umeNU}|xOE@ut&a9;se4P{ zhC)JI8#Td}{RQ~QS39WW56jV~h0R7wygM%nn=SFlytv({SHv|4b|hTPV8!>Fs(bEE z%Fa$hZVQktNeIeSEjaFq7<6%EydwW&Iz)!DzwybeSc*Idlvh?uJnN$h--~{mqLL_y;+HA-uEY{p`8;r_j*gh!BIW&~V znVev9lF2D1r;)^|?HyNELYSazu~-V+Ff0~-Zv6V@!e*o9FN8byUcL1CE6Ym@m1b*! za&c|#qYD@TD_Df;EiNxEUtIjb;wwv+mX{Wn7d*5mMRGwYlW?H~NzXU$%V#jsB|PCQ z62nTFgTJF@`|KE#lJ8BdWbqUHxDdSs374^25tbt|_bhoLvO4CT>6zOJ`COFntUdFm zCd<6UX#?f|LkR>WcN$-vuDm3^$hFT>n>ZSI+m9#TNItGjORQwy$Nsf zAw?|9T5l?H_UI4kvWaMtU*nm=GmU2k&nzB(OEEkW9gU7r4EN_c<|7Ir#cyWyG1=K( zxhmGjFnzsiBSL70z=NiO*$uE8xiQ_oNnM6&fRzi?j(N;wk(2E7>JRJwy{2D^lrO** zu-~v?A#`gfUhQbS!zcTZhsv#RM z&aF1;{`}JXh+S8N=(@Ghx_6<72VRGA6~fEnrEvbG@G)J!oY~tp$`sK}RE^?^A-^xS z>u$|QQ}QLYH>z%!J03gLuv!lzw_fsNn}$111b#hsK6Zna|BjRmskUmIfMSPI@Vuh7 zbCNEf8dbi`obo)+v**Oh>gu?i9VfhcU7lfMr6rOZt!e<5RzAzGKq(fdsL8hJ{D9K$ zv9VzCJd=w^uHgw^L}FMc%uGU887LjxQ({5=*j8OzfG^Ib&;GJ z)Sm6$t5W@CBIzm%le1Kn5^r7ahfvztir-_J6_sN_xQMaM-CIPit9CQiq;=G?%`?`t^_?SniRI1>($AO^cR&c?IRMO$#|n(PWPzR)%QSqhuW5 zA#zmYvvG-|@<$3N-uxfFhXi8B2j+i^I-X)kot^Z|uTr~a&r#UVJc12~vXRrV_9q^} zIv{82+)2D)7S?Jmr8l}V=40%5Zu0BvhqQr1L5AFxtCiTM>n7(xYn)WTs;49<*laMd z;ljPuFNf6xCwZ5%rC~Op5KfrWX8XBuYd07aQ<*_)d6Wrxr;_XP7_wNe+yuQEDo}Ve zPO(^Pc&&i>$zt*ImK$_y(#4|JC>4v!w@~^@uCT9nn7qq`#Kluxr>5j%D-;Kd-kS@h zQ7K7^YDjIFIc;0h6O(71Bbh`dq3TK~?*IA#{3ePH0PrJE1mH($%^whePkGZ1j6zzq zobhG>yaB=iyt7#4V{!>yx@RH0Yrr8Nw1TKAZZZNOZXpWbN3v1xA4l}w6vfv8MtatCUO|&@O`yieS^)_?&rKH)S{cSs)IWxX-u%7;p#EEg z=6xWj5&dr!iI0pmjA4Eg;62jacZ9P?dhHsZ0R~eTyC3=*7wKS-06?i5QOVyZLs;l( zuCLEmBkYbKpko_Cg9(80_4PsDL*X609N!kyGol&W(hpi%X|8I(s?;xZM-~sQ&$j}s zaHxzTKx+h73a~W`*qTwWmBtq%v%v~w=29RX###sSeawj>%= zzEDp!3l=E-O!)Yxc*elTl+~Vu7d##=(hE?=_}`Gxz6%U39D_Zx0_gaqxrN+G#20ztbPUPn@aGxFJ+yX*x% zTm*>A*?$SnoK~rax6=kdsD&1Q(GN|Yj-2?Vfz@3kC-?0P7o|mqFq4lZ}8}u zBo2^D(Y?(&dP<5^2VSGGXJ}%SGlL!-)OH7n%^??OFu>ok6N*F1I4}TR8N~~Mj2hq+ z2LLz75%Az7xbo2I&fV;;He5uCl@26v`JX<+zeB5W1du<90D82Ffax9qd*<*gEjfi2 z%Ej%C_9+U+^Yb52(;Y20D;M}7JzR~hlL(R`Kz;!QaazZdL$#iW^`aH@0q)m$djPaCg@JG@G2lmkfhRnR#7Jb!Idd{K zXF7>Fgv!nUU(W%trOa8Yo$ZTyANf$Exq}p}{sl4}bJxE2j}f;xmT?c;e8u?euXmGC zV$0rVnqt@z5SdXdiZ_u-hHn7Lh#qZa<;O@JmLRC9fp+Tt36xD7EW^3sW!s1*ac=mR zrm>qgjrQ4S%CkG^J9*f9cT;yDOdi_&LHd*KNh0c}5BAqd?b@Hu-7z{!(X4_| zFJeAjC@vX7#F7&=x8JC_+c2^%gwkn`2WvC1*gTl%fqEP~xD~{z8wi~S_j5Dy$LyC< zpqJt@bHB^vO(t(K`HTtoyAGHBgt>Jjadz!~Q%7v#WK$wsBZ;ze8NIo$Ag^dt)~w(# zCaR&k(gCqmF2&|3SXbcXpYWxY01HtTL|4vY&5t6|asp8n#}w94Gh?+glz;;`or;3b zHGcmJ64*HGVHFa!ic)DP48ggqR%Alwn2i#4x<&Tt8g!w-tr~;LWe*8ZGoAHjA z&{t`wh_17;pHy54j$LxD6%e0a5)P^MZ+D5x@E+o{&15czF_d8HjnBOgey&KSI4;4zlU`s=SSV z+8g_TRjW)sWO9oMJ)}_(tVsNQR)}8Gqev0MdFiR!w zYW_mb^Z@FeR#?{dlKo@nl$Tb=&KWNYtefH)e;@Pmgnt$d;Cd*FE( zSE)0}2q72x=|YD;9v9LQ(+B6@;Hz>)$1MVA+m=d;MU^+q}DyT%PWH8lpcDmub0U)3N>|e$Os`_ z(CxIGjj$c{83R0TQynK+bR3ocorM3L==2$m0QN;kToXCSgYAy#4C)9@-nH|4$qM-{u__hYS71scqiJ& zM`x&<&wk*PQ{$s7^4rKj(bC3(kqAv&hy?NtL2lY!RRi#VbE0Jgzf+yWBeX^?-GSQ; zi)?qTVQjCycjp>H(i6A{;TCZo%L8x$>cx93I~a=<`wxv9sl@oTPwM}vc? zrqzud3zQA7!|2eoEpRThTW6T<7Y_~hGz-(0HaZbp3N342S+FM9d`iM>WleB`3u}^A z)&w%Aa|3H)$^NzgtNZyOp;C(XH?j@5V`YRYz^LuQnbBzu*$7LmJdTs)cV!T`BmM(V zNHNN!5C%<8W-}8Ag0kitnu9524j{OFbAS^VWe#{ViROT}l6oOfhDC?X!E`+Pk-7)M z3mG4%$gg`D(laPToRab2{*|M5R<5n~LptN5qTl5DO}Xj^p1uh*a#}bPv-xoR^&SL9 z$J{SE7#bb)FU}vnXVjm-VVg&56uGVQ=f!1lt9Q#s-J;R)OWcNnOY25-gi8&-T&|X? zyfB5+RYdI(G7kFphdcmcS>0&$F1sk2F6>m{CRY&c#T5`=cyku>&R|c((+4-5j-yazD>ELOl_7pzMDeh#7;m$`PgYH%DZE?2b4h^|{giU?7{UWvXaqCqvdMBtpB!>K4tHYBK% zm@{_mD6yZxX*S_D&!ZijTw9MUM$uN~@1Xbg<)_@5_?qhyL&w+?vH3o39pL0eiJ0sg zf81|sl%tw^Pu|05Qps17xw}kukR0*=3P;O6t_u{kPS8iovH8xZNv1cjOeWJ>7}hjk z>MSn`nC;_-SbWGC;QqUYM@4kKzCE5p^bx#4(AgRyaL7L!C4zpz8zh1r2K<6UJF