diff --git a/.gitignore b/.gitignore index 88c448814000a9c037a8f5c27b797241089b0b00..e89f6224eea5d6f7eb99bf068724d003628dbd4c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ flink-python/dev/download flink-python/dev/.conda/ flink-python/dev/log/ flink-python/dev/.stage.txt +flink-python/.eggs/ atlassian-ide-plugin.xml out/ /docs/api diff --git a/NOTICE b/NOTICE index 88121c1d7e28b81e8a6c7be220a0bcb3fe357ca9..ede0dfa17caaedaf2087a13abbbf586b7a0988de 100644 --- a/NOTICE +++ b/NOTICE @@ -33,6 +33,8 @@ This project bundles the following dependencies under the BSD license. See bundled license files for details. - d3:3.5.12 +- cloudpickle:1.2.2 +- net.sf.py4j:py4j:0.10.8.1 This project bundles the following dependencies under SIL OFL 1.1 license (https://opensource.org/licenses/OFL-1.1). See bundled license files for details. diff --git a/NOTICE-binary b/NOTICE-binary deleted file mode 100644 index f794ace44ba39604abf573207e94c1e2ab8de374..0000000000000000000000000000000000000000 --- a/NOTICE-binary +++ /dev/null @@ -1,16731 +0,0 @@ -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- log4j:log4j:1.2.17 - -This project bundles the following dependencies under the MIT/X11 license. -See bundled license files for details. - -- org.slf4j:slf4j-log4j12:1.7.15 - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-examples-streaming-state-machine -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- org.apache.kafka:kafka-clients:0.10.2.1 - - -flink-connector-kafka-0.10 -Copyright 2014-2019 The Apache Software Foundation - -flink-connector-kafka-0.9 -Copyright 2014-2019 The Apache Software Foundation - -flink-connector-kafka-base -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-examples-streaming-twitter -Copyright 2014-2019 The Apache Software Foundation - -flink-connector-twitter -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.google.guava:guava:14.0.1 -- com.twitter:hbc-core:2.2.0 -- com.twitter:joauth:6.0.2 -- org.apache.httpcomponents:httpclient:4.5.3 -- org.apache.httpcomponents:httpcore:4.4.6 - -Apache HttpClient -Copyright 1999-2017 The Apache Software Foundation - -Apache HttpCore -Copyright 2005-2017 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-azure-fs-hadoop -Copyright 2014-2019 The Apache Software Foundation - -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.fasterxml.jackson.core:jackson-annotations:2.7.0 -- com.fasterxml.jackson.core:jackson-core:2.7.8 -- com.fasterxml.jackson.core:jackson-databind:2.7.8 -- com.google.guava:guava:11.0.2 -- com.microsoft.azure:azure-keyvault-core:0.8.0 -- com.microsoft.azure:azure-storage:5.4.0 -- commons-codec:commons-codec:1.10 -- commons-logging:commons-logging:1.1.3 -- org.apache.hadoop:hadoop-azure:3.1.0 -- org.apache.httpcomponents:httpclient:4.5.3 -- org.apache.httpcomponents:httpcore:4.4.6 -- org.eclipse.jetty:jetty-util:9.3.19.v20170502 -- org.eclipse.jetty:jetty-util-ajax:9.3.19.v20170502 - - -flink-hadoop-fs -Copyright 2014-2019 The Apache Software Foundation - -flink-fs-hadoop-shaded -Copyright 2014-2019 The Apache Software Foundation - -- org.apache.hadoop:hadoop-annotations:3.1.0 -- org.apache.hadoop:hadoop-auth:3.1.0 -- org.apache.hadoop:hadoop-common:3.1.0 -- org.apache.htrace:htrace-core4:4.1.0-incubating -- org.apache.commons:commons-configuration2:2.1.1 -- org.apache.commons:commons-lang3:3.3.2 -- commons-lang:commons-lang:2.6 -- commons-collections:commons-collections:3.2.2 -- commons-io:commons-io:2.4 -- commons-logging:commons-logging:1.1.3 -- commons-beanutils:commons-beanutils:1.9.3 -- com.google.guava:guava:11.0.2 -- com.fasterxml.jackson.core:jackson-annotations:2.7.0 -- com.fasterxml.jackson.core:jackson-core:2.7.8 -- com.fasterxml.jackson.core:jackson-databind:2.7.8 -- com.fasterxml.woodstox:woodstox-core:5.0.3 - -This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). -See bundled license files for details. - -- com.google.re2j:re2j:1.1 - -This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). -See bundled license files for details. - -- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) - -This project bundles org.apache.hadoop:*:3.1.0 from which it inherits the following notices: - -The Apache Hadoop project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -For the org.apache.hadoop.util.bloom.* classes: - -/** - * - * Copyright (c) 2005, European Commission project OneLab under contract - * 034819 (http://www.one-lab.org) - * All rights reserved. - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the distribution. - * - Neither the name of the University Catholique de Louvain - UCL - * nor the names of its contributors may be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -For portions of the native implementation of slicing-by-8 CRC calculation -in src/main/native/src/org/apache/hadoop/util: - -Copyright (c) 2008,2009,2010 Massachusetts Institute of Technology. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -* Neither the name of the Massachusetts Institute of Technology nor - the names of its contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other portions are under the same license from Intel: -http://sourceforge.net/projects/slicing-by-8/ -/*++ - * - * Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved - * - * This software program is licensed subject to the BSD License, - * available at http://www.opensource.org/licenses/bsd-license.html - * - * Abstract: The main routine - * - --*/ - -For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, - -/* - LZ4 - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -For hadoop-common-project/hadoop-common/src/main/native/gtest ---------------------------------------------------------------------- -Copyright 2008, Google Inc. -All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -The binary distribution of this product bundles these dependencies under the -following license: -re2j 1.1 ---------------------------------------------------------------------- -(GO license) -This is a work derived from Russ Cox's RE2 in Go, whose license -http://golang.org/LICENSE is as follows: - -Copyright (c) 2009 The Go Authors. All rights reserved. - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - - * Neither the name of Google Inc. nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h ---------------------------------------------------------------------- -Copyright 2002 Niels Provos -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles binaries of leveldbjni -(https://github.com/fusesource/leveldbjni), which is available under the -following license: - -Copyright (c) 2011 FuseSource Corp. All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of FuseSource Corp. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: - -Copyright (c) 2012 The FreeBSD Foundation -All rights reserved. - -This software was developed by Pawel Jakub Dawidek under sponsorship from -the FreeBSD Foundation. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. - -============= - -The binary distribution of this product bundles binaries of leveldb -(http://code.google.com/p/leveldb/), which is available under the following -license: - -Copyright (c) 2011 The LevelDB Authors. All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -The binary distribution of this product bundles binaries of snappy -(http://code.google.com/p/snappy/), which is available under the following -license: - -Copyright 2011, Google Inc. -All rights reserved. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ --------------------------------------------------------------------------------- -Copyright (C) 2008-2016, SpryMedia Ltd. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2010 Aleksander Williams - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -The binary distribution of this product bundles these dependencies under the -following license: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css -bootstrap v3.3.6 -broccoli-asset-rev v2.4.2 -broccoli-funnel v1.0.1 -datatables v1.10.8 -em-helpers v0.5.13 -em-table v0.1.6 -ember v2.2.0 -ember-array-contains-helper v1.0.2 -ember-bootstrap v0.5.1 -ember-cli v1.13.13 -ember-cli-app-version v1.0.0 -ember-cli-babel v5.1.6 -ember-cli-content-security-policy v0.4.0 -ember-cli-dependency-checker v1.2.0 -ember-cli-htmlbars v1.0.2 -ember-cli-htmlbars-inline-precompile v0.3.1 -ember-cli-ic-ajax v0.2.1 -ember-cli-inject-live-reload v1.4.0 -ember-cli-jquery-ui v0.0.20 -ember-cli-qunit v1.2.1 -ember-cli-release v0.2.8 -ember-cli-shims v0.0.6 -ember-cli-sri v1.2.1 -ember-cli-test-loader v0.2.1 -ember-cli-uglify v1.2.0 -ember-d3 v0.1.0 -ember-data v2.1.0 -ember-disable-proxy-controllers v1.0.1 -ember-export-application-global v1.0.5 -ember-load-initializers v0.1.7 -ember-qunit v0.4.16 -ember-qunit-notifications v0.1.0 -ember-resolver v2.0.3 -ember-spin-spinner v0.2.3 -ember-truth-helpers v1.2.0 -jquery v2.1.4 -jquery-ui v1.11.4 -loader.js v3.3.0 -momentjs v2.10.6 -qunit v1.19.0 -select2 v4.0.0 -snippet-ss v1.11.0 -spin.js v2.3.2 -Azure Data Lake Store - Java client SDK 2.0.11 -JCodings 1.0.8 -Joni 2.1.2 -Mockito 1.8.5 -JUL to SLF4J bridge 1.7.25 -SLF4J API Module 1.7.25 -SLF4J LOG4J-12 Binding 1.7.25 --------------------------------------------------------------------------------- - -The MIT License (MIT) - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery -Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 -Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 --------------------------------------------------------------------------------- - -MIT License - -Copyright (c) 2003-2017 Optimatika - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -For: -oj! Algorithms - version 43.0 --------------------------------------------------------------------------------- - -Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/ - -This software consists of voluntary contributions made by many -individuals. For exact contribution history, see the revision history -available at https://github.com/jquery/jquery - -The following license applies to all parts of this software except as -documented below: - -==== - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -All files located in the node_modules and external directories are -externally maintained libraries used by this software which have their -own licenses; we recommend you read them, as their terms may differ from -the terms above. - -For: -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js --------------------------------------------------------------------------------- - -Copyright (c) 2014 Ivan Bozhanov - -For: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js --------------------------------------------------------------------------------- - -D3 is available under a 3-clause BSD license. For details, see: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE - -The binary distribution of this product bundles these dependencies under the -following license: -HSQLDB Database 2.3.4 --------------------------------------------------------------------------------- -(HSQL License) -"COPYRIGHTS AND LICENSES (based on BSD License) - -For work developed by the HSQL Development Group: - -Copyright (c) 2001-2016, The HSQL Development Group -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -Neither the name of the HSQL Development Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -For work originally developed by the Hypersonic SQL Group: - -Copyright (c) 1995-2000 by the Hypersonic SQL Group. -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Neither the name of the Hypersonic SQL Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -This software consists of voluntary contributions made by many individuals on behalf of the -Hypersonic SQL Group." - -The binary distribution of this product bundles these dependencies under the -following license: -Java Servlet API 3.1.0 -servlet-api 2.5 -jsp-api 2.1 -jsr311-api 1.1.1 -Glassfish Jasper 6.1.14 -Servlet Specification 2.5 API 6.1.14 --------------------------------------------------------------------------------- -(CDDL 1.0) -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 -1. Definitions.  - -1.1. Contributor means each individual or entity -that creates or contributes to the creation of -Modifications.  - -1.2. Contributor Version means the combination of the -Original Software, prior Modifications used by a Contributor (if any), and the -Modifications made by that particular Contributor.  - -1.3. Covered -Software means (a) the Original Software, or (b) Modifications, or (c) the -combination of files containing Original Software with files containing -Modifications, in each case including portions -thereof.  - -1.4. Executable means the Covered Software in any form other -than Source Code.  - -1.5. Initial Developer means the individual or entity -that first makes Original Software available under this -License.  - -1.6. Larger Work means a work which combines Covered Software or -portions thereof with code not governed by the terms of this -License.  - -1.7. License means this document.  - -1.8. Licensable means -having the right to grant, to the maximum extent possible, whether at the time -of the initial grant or subsequently acquired, any and all of the rights -conveyed herein.  - -1.9. Modifications means the Source Code and Executable -form of any of the following: -A. Any file that results from an addition to, -deletion from or modification of the contents of a file containing Original -Software or previous Modifications; -B. Any new file that contains any part of the Original Software -or previous Modification; or -C. Any new file that is contributed or otherwise made available -under the terms of this License.  - -1.10. Original Software means the Source Code and Executable form of -computer software code that is originally released under this License.  - -1.11. Patent Claims means any patent claim(s), now owned or -hereafter acquired, including without limitation, method, process, and apparatus -claims, in any patent Licensable by grantor.  - -1.12. Source Code means (a) the common form of computer software code in which -modifications are made and (b) associated documentation included in or -with such code.  - -1.13. You (or Your) means an individual or a legal entity exercising rights -under, and complying with all of the terms of, this License. For legal entities, -You includes any entity which controls, is controlled by, or is under common control -with You. For purposes of this definition, control means (a) the power, direct -or indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (b) ownership of more than fifty percent (50%) of the -outstanding shares or beneficial ownership of such entity.  - -2. License Grants. - -2.1. The Initial Developer Grant. Conditioned upon Your compliance -with Section 3.1 below and subject to third party intellectual property claims, -the Initial Developer hereby grants You a world-wide, royalty-free, -non-exclusive license:  - -(a) under intellectual property rights (other than -patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, -display, perform, sublicense and distribute the Original Software (or portions -thereof), with or without Modifications, and/or as part of a Larger Work; -and  - -(b) under Patent Claims infringed by the making, using or selling of -Original Software, to make, have made, use, practice, sell, and offer for sale, -and/or otherwise dispose of the Original Software (or portions -thereof); - -(c) The licenses granted in Sections 2.1(a) and (b) are -effective on the date Initial Developer first distributes or otherwise makes the -Original Software available to a third party under the terms of this -License; - -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or -(ii) the combination of the Original Software with other software or -devices.  - -2.2. Contributor Grant. Conditioned upon Your compliance with -Section 3.1 below and subject to third party intellectual property claims, each -Contributor hereby grants You a world-wide, royalty-free, non-exclusive -license:  - -(a) under intellectual property rights (other than patent or -trademark) Licensable by Contributor to use, reproduce, modify, display, -perform, sublicense and distribute the Modifications created by such Contributor -(or portions thereof), either on an unmodified basis, with other Modifications, -as Covered Software and/or as part of a Larger Work; and  - -(b) under Patent -Claims infringed by the making, using, or selling of Modifications made by that -Contributor either alone and/or in combination with its Contributor Version (or -portions of such combination), to make, use, sell, offer for sale, have made, -and/or otherwise dispose of: (1) Modifications made by that Contributor (or -portions thereof); and (2) the combination of Modifications made by that -Contributor with its Contributor Version (or portions of such -combination).  - -(c) The licenses granted in Sections 2.2(a) and 2.2(b) are -effective on the date Contributor first distributes or otherwise makes the -Modifications available to a third party. - -(d) Notwithstanding Section 2.2(b) -above, no patent license is granted: (1) for any code that Contributor has -deleted from the Contributor Version; (2) for infringements caused by: -(i) third party modifications of Contributor Version, or (ii) the combination -of Modifications made by that Contributor with other software (except as part of -the Contributor Version) or other devices; or (3) under Patent Claims infringed -by Covered Software in the absence of Modifications made by that -Contributor.  - -3. Distribution Obligations.  - -3.1. Availability of Source -Code. Any Covered Software that You distribute or otherwise make available in -Executable form must also be made available in Source Code form and that Source -Code form must be distributed only under the terms of this License. You must -include a copy of this License with every copy of the Source Code form of the -Covered Software You distribute or otherwise make available. You must inform -recipients of any such Covered Software in Executable form as to how they can -obtain such Covered Software in Source Code form in a reasonable manner on or -through a medium customarily used for software exchange.  - -3.2. -Modifications. The Modifications that You create or to which You contribute are -governed by the terms of this License. You represent that You believe Your -Modifications are Your original creation(s) and/or You have sufficient rights to -grant the rights conveyed by this License.  - -3.3. Required Notices. You must -include a notice in each of Your Modifications that identifies You as the -Contributor of the Modification. You may not remove or alter any copyright, -patent or trademark notices contained within the Covered Software, or any -notices of licensing or any descriptive text giving attribution to any -Contributor or the Initial Developer.  - -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source Code -form that alters or restricts the applicable version of this License or the -recipients rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. - -3.5. Distribution of Executable Versions. You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipients rights in the Source Code form from the rights set forth in this -License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer.  - -3.6. Larger Works. You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software.  - -4. Versions of the License.  - -4.1. -New Versions. Sun Microsystems, Inc. is the initial license steward and may -publish revised and/or new versions of this License from time to time. Each -version will be given a distinguishing version number. Except as provided in -Section 4.3, no one other than the license steward has the right to modify this -License.  - -4.2. Effect of New Versions. You may always continue to use, -distribute or otherwise make the Covered Software available under the terms of -the version of the License under which You originally received the Covered -Software. If the Initial Developer includes a notice in the Original Software -prohibiting it from being distributed or otherwise made available under any -subsequent version of the License, You must distribute and make the Covered -Software available under the terms of the version of the License under which You -originally received the Covered Software. Otherwise, You may also choose to use, -distribute or otherwise make the Covered Software available under the terms of -any subsequent version of the License published by the license -steward.  - -4.3. Modified Versions. When You are an Initial Developer and You -want to create a new license for Your Original Software, You may create and use -a modified version of this License if You: (a) rename the license and remove -any references to the name of the license steward (except to note that the -license differs from this License); and (b) otherwise make it clear that the -license contains terms which differ from this License.  - -5. DISCLAIMER OF WARRANTY. - -COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, -WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT -LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, -MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY -COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER -OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR -CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS -LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER -THIS DISCLAIMER.  - -6. TERMINATION.  - -6.1. This License and the rights -granted hereunder will terminate automatically if You fail to comply with terms -herein and fail to cure such breach within 30 days of becoming aware of the -breach. Provisions which, by their nature, must remain in effect beyond the -termination of this License shall survive.  - -6.2. If You assert a patent -infringement claim (excluding declaratory judgment actions) against Initial -Developer or a Contributor (the Initial Developer or Contributor against whom -You assert such claim is referred to as Participant) alleging that the -Participant Software (meaning the Contributor Version where the Participant is a -Contributor or the Original Software where the Participant is the Initial -Developer) directly or indirectly infringes any patent, then any and all rights -granted directly or indirectly to You by such Participant, the Initial Developer -(if the Initial Developer is not the Participant) and all Contributors under -Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from -Participant terminate prospectively and automatically at the expiration of such -60 day notice period, unless if within such 60 day period You withdraw Your -claim with respect to the Participant Software against such Participant either -unilaterally or pursuant to a written agreement with Participant.  - -6.3. In -the event of termination under Sections 6.1 or 6.2 above, all end user licenses -that have been validly granted by You or any distributor hereunder prior to -termination (excluding licenses granted to You by any distributor) shall survive -termination.  - -7. LIMITATION OF LIABILITY. -UNDER NO CIRCUMSTANCES AND UNDER -NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, -SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF -COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY -PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY -CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF -GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER -COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE -POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO -LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO -THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT -ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO -THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  - -8. U.S. GOVERNMENT END USERS. - -The Covered Software is a commercial item, as that term is defined in -48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as -that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer -software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. -1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through -227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software -with only those rights set forth herein. This U.S. Government Rights clause is -in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision -that addresses Government rights in computer software under this -License.  - -9. MISCELLANEOUS. -This License represents the complete agreement -concerning subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdictions conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software.  - -10. RESPONSIBILITY FOR CLAIMS. -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability.  - -The binary distribution of this product bundles these dependencies under the -following license: -jersey-client 1.19 -jersey-core 1.19 -jersey-grizzly2 1.19 -jersey-grizzly2-servlet 1.19 -jersey-json 1.19 -jersey-server 1.19 -jersey-servlet 1.19 -jersey-guice 1.19 -Jersey Test Framework - Grizzly 2 Module 1.19 -JAXB RI 2.2.3 -Java Architecture for XML Binding 2.2.11 -grizzly-framework 2.2.21 -grizzly-http 2.2.21 -grizzly-http-server 2.2.21 -grizzly-http-servlet 2.2.21 -grizzly-rcm 2.2.21 --------------------------------------------------------------------------------- -(CDDL 1.1) -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 - -1. Definitions. - -1.1. “Contributor” means each individual or entity that creates or -contributes to the creation of Modifications. -1.2. “Contributor Version” means the combination of the Original Software, -prior Modifications used by a Contributor (if any), and the Modifications made -by that particular Contributor. -1.3. “Covered Software” means (a) the Original Software, or (b) -Modifications, or (c) the combination of files containing Original Software with -files containing Modifications, in each case including portions thereof. -1.4. “Executable” means the Covered Software in any form other than Source -Code. -1.5. “Initial Developer” means the individual or entity that first makes -Original Software available under this License. -1.6. “Larger Work” means a work which combines Covered Software or portions -thereof with code not governed by the terms of this License. -1.7. “License” means this document. -1.8. “Licensable” means having the right to grant, to the maximum extent -possible, whether at the time of the initial grant or subsequently acquired, any -and all of the rights conveyed herein. -1.9. “Modifications” means the Source Code and Executable form of any of the -following: -A. Any file that results from an addition to, deletion from or modification of -the contents of a file containing Original Software or previous Modifications; -B. Any new file that contains any part of the Original Software or previous -Modification; or -C. Any new file that is contributed or otherwise made available under the terms -of this License. -1.10. “Original Software” means the Source Code and Executable form of -computer software code that is originally released under this License. -1.11. “Patent Claims” means any patent claim(s), now owned or hereafter -acquired, including without limitation, method, process, and apparatus claims, -in any patent Licensable by grantor. -1.12. “Source Code” means (a) the common form of computer software code in -which modifications are made and (b) associated documentation included in or -with such code. -1.13. “You” (or “Your”) means an individual or a legal entity exercising -rights under, and complying with all of the terms of, this License. For legal -entities, “You” includes any entity which controls, is controlled by, or is -under common control with You. For purposes of this definition, “control” -means (a) the power, direct or indirect, to cause the direction or management of -such entity, whether by contract or otherwise, or (b) ownership of more than -fifty percent (50%) of the outstanding shares or beneficial ownership of such -entity. - -2.1. The Initial Developer Grant. - -Conditioned upon Your compliance with Section 3.1 below and subject to -third party intellectual property claims, the Initial Developer hereby grants -You a world-wide, royalty-free, non-exclusive license: -(a) under intellectual -property rights (other than patent or trademark) Licensable by Initial -Developer, to use, reproduce, modify, display, perform, sublicense and -distribute the Original Software (or portions thereof), with or without -Modifications, and/or as part of a Larger Work; and -(b) under Patent Claims -infringed by the making, using or selling of Original Software, to make, have -made, use, practice, sell, and offer for sale, and/or otherwise dispose of the -Original Software (or portions thereof). -(c) The licenses granted in Sections -2.1(a) and (b) are effective on the date Initial Developer first distributes or -otherwise makes the Original Software available to a third party under the terms -of this License. -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or (ii) -the combination of the Original Software with other software or devices. - -2.2. Contributor Grant. - -Conditioned upon Your compliance with Section 3.1 below and -subject to third party intellectual property claims, each Contributor hereby -grants You a world-wide, royalty-free, non-exclusive license: -(a) under -intellectual property rights (other than patent or trademark) Licensable by -Contributor to use, reproduce, modify, display, perform, sublicense and -distribute the Modifications created by such Contributor (or portions thereof), -either on an unmodified basis, with other Modifications, as Covered Software -and/or as part of a Larger Work; and -(b) under Patent Claims infringed by the -making, using, or selling of Modifications made by that Contributor either alone -and/or in combination with its Contributor Version (or portions of such -combination), to make, use, sell, offer for sale, have made, and/or otherwise -dispose of: (1) Modifications made by that Contributor (or portions thereof); -and (2) the combination of Modifications made by that Contributor with its -Contributor Version (or portions of such combination). -(c) The licenses granted -in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first -distributes or otherwise makes the Modifications available to a third -party. -(d) Notwithstanding Section 2.2(b) above, no patent license is granted: -(1) for any code that Contributor has deleted from the Contributor Version; (2) -for infringements caused by: (i) third party modifications of Contributor -Version, or (ii) the combination of Modifications made by that Contributor with -other software (except as part of the Contributor Version) or other devices; or -(3) under Patent Claims infringed by Covered Software in the absence of -Modifications made by that Contributor. - -3. Distribution Obligations. - -3.1. Availability of Source Code. -Any Covered Software that You distribute or -otherwise make available in Executable form must also be made available in -Source Code form and that Source Code form must be distributed only under the -terms of this License. You must include a copy of this License with every copy -of the Source Code form of the Covered Software You distribute or otherwise make -available. You must inform recipients of any such Covered Software in Executable -form as to how they can obtain such Covered Software in Source Code form in a -reasonable manner on or through a medium customarily used for software -exchange. -3.2. Modifications. -The Modifications that You create or to which -You contribute are governed by the terms of this License. You represent that You -believe Your Modifications are Your original creation(s) and/or You have -sufficient rights to grant the rights conveyed by this License. -3.3. Required Notices. -You must include a notice in each of Your Modifications that -identifies You as the Contributor of the Modification. You may not remove or -alter any copyright, patent or trademark notices contained within the Covered -Software, or any notices of licensing or any descriptive text giving attribution -to any Contributor or the Initial Developer. -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source -Code form that alters or restricts the applicable version of this License or the -recipients' rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. -3.5. Distribution of Executable Versions. -You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipient's rights in the Source Code form from the rights set forth in -this License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer. -3.6. Larger Works. -You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software. - -4. Versions of the License. - -4.1. New Versions. -Oracle is the initial license steward and may publish revised and/or -new versions of this License from time to time. Each version will be given a -distinguishing version number. Except as provided in Section 4.3, no one other -than the license steward has the right to modify this License. -4.2. Effect of New Versions. -You may always continue to use, distribute or otherwise make the -Covered Software available under the terms of the version of the License under -which You originally received the Covered Software. If the Initial Developer -includes a notice in the Original Software prohibiting it from being distributed -or otherwise made available under any subsequent version of the License, You -must distribute and make the Covered Software available under the terms of the -version of the License under which You originally received the Covered Software. -Otherwise, You may also choose to use, distribute or otherwise make the Covered -Software available under the terms of any subsequent version of the License -published by the license steward. -4.3. Modified Versions. -When You are an -Initial Developer and You want to create a new license for Your Original -Software, You may create and use a modified version of this License if You: (a) -rename the license and remove any references to the name of the license steward -(except to note that the license differs from this License); and (b) otherwise -make it clear that the license contains terms which differ from this -License. - -COVERED SOFTWARE IS PROVIDED UNDER THIS -LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE -IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR -NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED -SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY -RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE -COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF -WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED -SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - -6.1. This License and the rights granted hereunder will -terminate automatically if You fail to comply with terms herein and fail to cure -such breach within 30 days of becoming aware of the breach. Provisions which, by -their nature, must remain in effect beyond the termination of this License shall -survive. -6.2. If You assert a patent infringement claim (excluding declaratory -judgment actions) against Initial Developer or a Contributor (the Initial -Developer or Contributor against whom You assert such claim is referred to as -“Participant”) alleging that the Participant Software (meaning the -Contributor Version where the Participant is a Contributor or the Original -Software where the Participant is the Initial Developer) directly or indirectly -infringes any patent, then any and all rights granted directly or indirectly to -You by such Participant, the Initial Developer (if the Initial Developer is not -the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this -License shall, upon 60 days notice from Participant terminate prospectively and -automatically at the expiration of such 60 day notice period, unless if within -such 60 day period You withdraw Your claim with respect to the Participant -Software against such Participant either unilaterally or pursuant to a written -agreement with Participant. -6.3. If You assert a patent infringement claim -against Participant alleging that the Participant Software directly or -indirectly infringes any patent where such claim is resolved (such as by license -or settlement) prior to the initiation of patent infringement litigation, then -the reasonable value of the licenses granted by such Participant under Sections -2.1 or 2.2 shall be taken into account in determining the amount or value of any -payment or license. -6.4. In the event of termination under Sections 6.1 or 6.2 -above, all end user licenses that have been validly granted by You or any -distributor hereunder prior to termination (excluding licenses granted to You by -any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. - -UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT -(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL -DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY -SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, -SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, -WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER -FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN -IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS -LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL -INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW -PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR -LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND -LIMITATION MAY NOT APPLY TO YOU. - -The Covered -Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 -(Oct. 1995), consisting of “commercial computer software” (as that term is -defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software -documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). -Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 -(June 1995), all U.S. Government End Users acquire Covered Software with only -those rights set forth herein. This U.S. Government Rights clause is in lieu of, -and supersedes, any other FAR, DFAR, or other clause or provision that addresses -Government rights in computer software under this License. - -9. MISCELLANEOUS. - -This License represents the complete agreement concerning -subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdiction's conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys' fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. - -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability. - -The binary distribution of this product bundles these dependencies under the -following license: -Protocol Buffer Java API 2.5.0 --------------------------------------------------------------------------------- -This license applies to all parts of Protocol Buffers except the following: - - - Atomicops support for generic gcc, located in - src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. - This file is copyrighted by Red Hat Inc. - - - Atomicops support for AIX/POWER, located in - src/google/protobuf/stubs/atomicops_internals_power.h. - This file is copyrighted by Bloomberg Finance LP. - -Copyright 2014, Google Inc. All rights reserved. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - -For: -XML Commons External Components XML APIs 1.3.04 --------------------------------------------------------------------------------- -By obtaining, using and/or copying this work, you (the licensee) agree that you -have read, understood, and will comply with the following terms and conditions. - -Permission to copy, modify, and distribute this software and its documentation, -with or without modification, for any purpose and without fee or royalty is -hereby granted, provided that you include the following on ALL copies of the -software and documentation or portions thereof, including modifications: -- The full text of this NOTICE in a location viewable to users of the -redistributed or derivative work. -- Any pre-existing intellectual property disclaimers, notices, or terms and -conditions. If none exist, the W3C Software Short Notice should be included -(hypertext is preferred, text is permitted) within the body of any redistributed -or derivative code. -- Notice of any changes or modifications to the files, including the date changes -were made. (We recommend you provide URIs to the location from which the code is -derived.) - -The binary distribution of this product bundles these dependencies under the -following license: -JUnit 4.11 -Eclipse JDT Core 3.1.1 --------------------------------------------------------------------------------- -(EPL v1.0) -Eclipse Public License - v 1.0 - -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC -LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM -CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: - -a) in the case of the initial Contributor, the initial code and documentation -distributed under this Agreement, and -b) in the case of each subsequent Contributor: -i) changes to the Program, and -ii) additions to the Program; -where such changes and/or additions to the Program originate from and are -distributed by that particular Contributor. A Contribution 'originates' from a -Contributor if it was added to the Program by such Contributor itself or anyone -acting on such Contributor's behalf. Contributions do not include additions to -the Program which: (i) are separate modules of software distributed in -conjunction with the Program under their own license agreement, and (ii) are not -derivative works of the Program. -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents" mean patent claims licensable by a Contributor which are -necessarily infringed by the use or sale of its Contribution alone or when -combined with the Program. - -"Program" means the Contributions distributed in accordance with this Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, -including all Contributors. - -2. GRANT OF RIGHTS - -a) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free copyright license to -reproduce, prepare derivative works of, publicly display, publicly perform, -distribute and sublicense the Contribution of such Contributor, if any, and such -derivative works, in source code and object code form. -b) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed -Patents to make, use, sell, offer to sell, import and otherwise transfer the -Contribution of such Contributor, if any, in source code and object code form. -This patent license shall apply to the combination of the Contribution and the -Program if, at the time the Contribution is added by the Contributor, such -addition of the Contribution causes such combination to be covered by the -Licensed Patents. The patent license shall not apply to any other combinations -which include the Contribution. No hardware per se is licensed hereunder. -c) Recipient understands that although each Contributor grants the licenses to -its Contributions set forth herein, no assurances are provided by any -Contributor that the Program does not infringe the patent or other intellectual -property rights of any other entity. Each Contributor disclaims any liability to -Recipient for claims brought by any other entity based on infringement of -intellectual property rights or otherwise. As a condition to exercising the -rights and licenses granted hereunder, each Recipient hereby assumes sole -responsibility to secure any other intellectual property rights needed, if any. -For example, if a third party patent license is required to allow Recipient to -distribute the Program, it is Recipient's responsibility to acquire that license -before distributing the Program. -d) Each Contributor represents that to its knowledge it has sufficient copyright -rights in its Contribution, if any, to grant the copyright license set forth in -this Agreement. -3. REQUIREMENTS - -A Contributor may choose to distribute the Program in object code form under its -own license agreement, provided that: - -a) it complies with the terms and conditions of this Agreement; and -b) its license agreement: -i) effectively disclaims on behalf of all Contributors all warranties and -conditions, express and implied, including warranties or conditions of title and -non-infringement, and implied warranties or conditions of merchantability and -fitness for a particular purpose; -ii) effectively excludes on behalf of all Contributors all liability for -damages, including direct, indirect, special, incidental and consequential -damages, such as lost profits; -iii) states that any provisions which differ from this Agreement are offered by -that Contributor alone and not by any other party; and -iv) states that source code for the Program is available from such Contributor, -and informs licensees how to obtain it in a reasonable manner on or through a -medium customarily used for software exchange. -When the Program is made available in source code form: - -a) it must be made available under this Agreement; and -b) a copy of this Agreement must be included with each copy of the Program. -Contributors may not remove or alter any copyright notices contained within the -Program. - -Each Contributor must identify itself as the originator of its Contribution, if -any, in a manner that reasonably allows subsequent Recipients to identify the -originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION - -Commercial distributors of software may accept certain responsibilities with -respect to end users, business partners and the like. While this license is -intended to facilitate the commercial use of the Program, the Contributor who -includes the Program in a commercial product offering should do so in a manner -which does not create potential liability for other Contributors. Therefore, if -a Contributor includes the Program in a commercial product offering, such -Contributor ("Commercial Contributor") hereby agrees to defend and indemnify -every other Contributor ("Indemnified Contributor") against any losses, damages -and costs (collectively "Losses") arising from claims, lawsuits and other legal -actions brought by a third party against the Indemnified Contributor to the -extent caused by the acts or omissions of such Commercial Contributor in -connection with its distribution of the Program in a commercial product -offering. The obligations in this section do not apply to any claims or Losses -relating to any actual or alleged intellectual property infringement. In order -to qualify, an Indemnified Contributor must: a) promptly notify the Commercial -Contributor in writing of such claim, and b) allow the Commercial Contributor to -control, and cooperate with the Commercial Contributor in, the defense and any -related settlement negotiations. The Indemnified Contributor may participate in -any such claim at its own expense. - -For example, a Contributor might include the Program in a commercial product -offering, Product X. That Contributor is then a Commercial Contributor. If that -Commercial Contributor then makes performance claims, or offers warranties -related to Product X, those performance claims and warranties are such -Commercial Contributor's responsibility alone. Under this section, the -Commercial Contributor would have to defend claims against the other -Contributors related to those performance claims and warranties, and if a court -requires any other Contributor to pay any damages as a result, the Commercial -Contributor must pay those damages. - -5. NO WARRANTY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR -IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, -NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each -Recipient is solely responsible for determining the appropriateness of using and -distributing the Program and assumes all risks associated with its exercise of -rights under this Agreement , including but not limited to the risks and costs -of program errors, compliance with applicable laws, damage to or loss of data, -programs or equipment, and unavailability or interruption of operations. - -6. DISCLAIMER OF LIABILITY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY -CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST -PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS -GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this Agreement, and without further action by the parties hereto, such -provision shall be reformed to the minimum extent necessary to make such -provision valid and enforceable. - -If Recipient institutes patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Program itself -(excluding combinations of the Program with other software or hardware) -infringes such Recipient's patent(s), then such Recipient's rights granted under -Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to -comply with any of the material terms or conditions of this Agreement and does -not cure such failure in a reasonable period of time after becoming aware of -such noncompliance. If all Recipient's rights under this Agreement terminate, -Recipient agrees to cease use and distribution of the Program as soon as -reasonably practicable. However, Recipient's obligations under this Agreement -and any licenses granted by Recipient relating to the Program shall continue and -survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in -order to avoid inconsistency the Agreement is copyrighted and may only be -modified in the following manner. The Agreement Steward reserves the right to -publish new versions (including revisions) of this Agreement from time to time. -No one other than the Agreement Steward has the right to modify this Agreement. -The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation -may assign the responsibility to serve as the Agreement Steward to a suitable -separate entity. Each new version of the Agreement will be given a -distinguishing version number. The Program (including Contributions) may always -be distributed subject to the version of the Agreement under which it was -received. In addition, after a new version of the Agreement is published, -Contributor may elect to distribute the Program (including its Contributions) -under the new version. Except as expressly stated in Sections 2(a) and 2(b) -above, Recipient receives no rights or licenses to the intellectual property of -any Contributor under this Agreement, whether expressly, by implication, -estoppel or otherwise. All rights in the Program not expressly granted under -this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the -intellectual property laws of the United States of America. No party to this -Agreement will bring a legal action under this Agreement more than one year -after the cause of action arose. Each party waives its rights to a jury trial in -any resulting litigation. - -The binary distribution of this product bundles these dependencies under the -following license: -JSch 0.1.51 -ParaNamer Core 2.3 -JLine 0.9.94 -leveldbjni-all 1.8 -Hamcrest Core 1.3 -ASM Core 5.0.4 -ASM Commons 5.0.2 -ASM Tree 5.0.2 --------------------------------------------------------------------------------- -(3-clause BSD) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles these dependencies under the -following license: -FindBugs-jsr305 3.0.0 -dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved. --------------------------------------------------------------------------------- -(2-clause BSD) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those -of the authors and should not be interpreted as representing official policies, -either expressed or implied, of the FreeBSD Project. - -The binary distribution of this product bundles these dependencies under the -following license: -"Java Concurrency in Practice" book annotations 1.0 --------------------------------------------------------------------------------- -(CCAL v2.5) -THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS -PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR -OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS -LICENSE OR COPYRIGHT LAW IS PROHIBITED. - -BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE -BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED -HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. - -1. Definitions - -"Collective Work" means a work, such as a periodical issue, anthology or -encyclopedia, in which the Work in its entirety in unmodified form, along with a -number of other contributions, constituting separate and independent works in -themselves, are assembled into a collective whole. A work that constitutes a -Collective Work will not be considered a Derivative Work (as defined below) for -the purposes of this License. -"Derivative Work" means a work based upon the Work or upon the Work and other -pre-existing works, such as a translation, musical arrangement, dramatization, -fictionalization, motion picture version, sound recording, art reproduction, -abridgment, condensation, or any other form in which the Work may be recast, -transformed, or adapted, except that a work that constitutes a Collective Work -will not be considered a Derivative Work for the purpose of this License. For -the avoidance of doubt, where the Work is a musical composition or sound -recording, the synchronization of the Work in timed-relation with a moving image -("synching") will be considered a Derivative Work for the purpose of this -License. -"Licensor" means the individual or entity that offers the Work under the terms -of this License. -"Original Author" means the individual or entity who created the Work. -"Work" means the copyrightable work of authorship offered under the terms of -this License. -"You" means an individual or entity exercising rights under this License who has -not previously violated the terms of this License with respect to the Work, or -who has received express permission from the Licensor to exercise rights under -this License despite a previous violation. -2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or -restrict any rights arising from fair use, first sale or other limitations on -the exclusive rights of the copyright owner under copyright law or other -applicable laws. - -3. License Grant. Subject to the terms and conditions of this License, Licensor -hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the -duration of the applicable copyright) license to exercise the rights in the Work -as stated below: - -to reproduce the Work, to incorporate the Work into one or more Collective -Works, and to reproduce the Work as incorporated in the Collective Works; -to create and reproduce Derivative Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission the Work including as -incorporated in Collective Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission Derivative Works. -For the avoidance of doubt, where the work is a musical composition: - -Performance Royalties Under Blanket Licenses. Licensor waives the exclusive -right to collect, whether individually or via a performance rights society (e.g. -ASCAP, BMI, SESAC), royalties for the public performance or public digital -performance (e.g. webcast) of the Work. -Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right -to collect, whether individually or via a music rights agency or designated -agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the -Work ("cover version") and distribute, subject to the compulsory license created -by 17 USC Section 115 of the US Copyright Act (or the equivalent in other -jurisdictions). -Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the -Work is a sound recording, Licensor waives the exclusive right to collect, -whether individually or via a performance-rights society (e.g. SoundExchange), -royalties for the public digital performance (e.g. webcast) of the Work, subject -to the compulsory license created by 17 USC Section 114 of the US Copyright Act -(or the equivalent in other jurisdictions). -The above rights may be exercised in all media and formats whether now known or -hereafter devised. The above rights include the right to make such modifications -as are technically necessary to exercise the rights in other media and formats. -All rights not expressly granted by Licensor are hereby reserved. - -4. Restrictions.The license granted in Section 3 above is expressly made subject -to and limited by the following restrictions: - -You may distribute, publicly display, publicly perform, or publicly digitally -perform the Work only under the terms of this License, and You must include a -copy of, or the Uniform Resource Identifier for, this License with every copy or -phonorecord of the Work You distribute, publicly display, publicly perform, or -publicly digitally perform. You may not offer or impose any terms on the Work -that alter or restrict the terms of this License or the recipients' exercise of -the rights granted hereunder. You may not sublicense the Work. You must keep -intact all notices that refer to this License and to the disclaimer of -warranties. You may not distribute, publicly display, publicly perform, or -publicly digitally perform the Work with any technological measures that control -access or use of the Work in a manner inconsistent with the terms of this -License Agreement. The above applies to the Work as incorporated in a Collective -Work, but this does not require the Collective Work apart from the Work itself -to be made subject to the terms of this License. If You create a Collective -Work, upon notice from any Licensor You must, to the extent practicable, remove -from the Collective Work any credit as required by clause 4(b), as requested. If -You create a Derivative Work, upon notice from any Licensor You must, to the -extent practicable, remove from the Derivative Work any credit as required by -clause 4(b), as requested. -If you distribute, publicly display, publicly perform, or publicly digitally -perform the Work or any Derivative Works or Collective Works, You must keep -intact all copyright notices for the Work and provide, reasonable to the medium -or means You are utilizing: (i) the name of the Original Author (or pseudonym, -if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor -designate another party or parties (e.g. a sponsor institute, publishing entity, -journal) for attribution in Licensor's copyright notice, terms of service or by -other reasonable means, the name of such party or parties; the title of the Work -if supplied; to the extent reasonably practicable, the Uniform Resource -Identifier, if any, that Licensor specifies to be associated with the Work, -unless such URI does not refer to the copyright notice or licensing information -for the Work; and in the case of a Derivative Work, a credit identifying the use -of the Work in the Derivative Work (e.g., "French translation of the Work by -Original Author," or "Screenplay based on original Work by Original Author"). -Such credit may be implemented in any reasonable manner; provided, however, that -in the case of a Derivative Work or Collective Work, at a minimum such credit -will appear where any other comparable authorship credit appears and in a manner -at least as prominent as such other comparable authorship credit. -5. Representations, Warranties and Disclaimer - -UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS -THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING -THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT -LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR -PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, -OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME -JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH -EXCLUSION MAY NOT APPLY TO YOU. - -6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN -NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, -INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS -LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - -7. Termination - -This License and the rights granted hereunder will terminate automatically upon -any breach by You of the terms of this License. Individuals or entities who have -received Derivative Works or Collective Works from You under this License, -however, will not have their licenses terminated provided such individuals or -entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, -and 8 will survive any termination of this License. -Subject to the above terms and conditions, the license granted here is perpetual -(for the duration of the applicable copyright in the Work). Notwithstanding the -above, Licensor reserves the right to release the Work under different license -terms or to stop distributing the Work at any time; provided, however that any -such election will not serve to withdraw this License (or any other license that -has been, or is required to be, granted under the terms of this License), and -this License will continue in full force and effect unless terminated as stated -above. -8. Miscellaneous - -Each time You distribute or publicly digitally perform the Work or a Collective -Work, the Licensor offers to the recipient a license to the Work on the same -terms and conditions as the license granted to You under this License. -Each time You distribute or publicly digitally perform a Derivative Work, -Licensor offers to the recipient a license to the original Work on the same -terms and conditions as the license granted to You under this License. -If any provision of this License is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this License, and without further action by the parties to this -agreement, such provision shall be reformed to the minimum extent necessary to -make such provision valid and enforceable. -No term or provision of this License shall be deemed waived and no breach -consented to unless such waiver or consent shall be in writing and signed by the -party to be charged with such waiver or consent. -This License constitutes the entire agreement between the parties with respect -to the Work licensed here. There are no understandings, agreements or -representations with respect to the Work not specified here. Licensor shall not -be bound by any additional provisions that may appear in any communication from -You. This License may not be modified without the mutual written agreement of -the Licensor and You. - -The binary distribution of this product bundles these dependencies under the -following license: -jamon-runtime 2.4.1 --------------------------------------------------------------------------------- -(MPL 2.0) - Mozilla Public License - Version 2.0 - -1.1. “Contributor” -means each individual or legal entity that creates, contributes to the creation -of, or owns Covered Software. - -1.2. “Contributor Version” -means the combination of the Contributions of others (if any) used by a -Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” -means Covered Software of a particular Contributor. - -1.4. “Covered Software” -means Source Code Form to which the initial Contributor has attached the notice -in Exhibit A, the Executable Form of such Source Code Form, and Modifications of -such Source Code Form, in each case including portions thereof. - -1.5. “Incompatible With Secondary Licenses” -means - -that the initial Contributor has attached the notice described in Exhibit B to -the Covered Software; or - -that the Covered Software was made available under the terms of version 1.1 or -earlier of the License, but not also under the terms of a Secondary License. - -1.6. “Executable Form” -means any form of the work other than Source Code Form. - -1.7. “Larger Work” -means a work that combines Covered Software with other material, in a separate -file or files, that is not Covered Software. - -1.8. “License” -means this document. - -1.9. “Licensable” -means having the right to grant, to the maximum extent possible, whether at the -time of the initial grant or subsequently, any and all of the rights conveyed by -this License. - -1.10. “Modifications” -means any of the following: - -any file in Source Code Form that results from an addition to, deletion from, or -modification of the contents of Covered Software; or - -any new file in Source Code Form that contains any Covered Software. - -1.11. “Patent Claims” of a Contributor -means any patent claim(s), including without limitation, method, process, and -apparatus claims, in any patent Licensable by such Contributor that would be -infringed, but for the grant of the License, by the making, using, selling, -offering for sale, having made, import, or transfer of either its Contributions -or its Contributor Version. - -1.12. “Secondary License” -means either the GNU General Public License, Version 2.0, the GNU Lesser General -Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, -or any later versions of those licenses. - -1.13. “Source Code Form” -means the form of the work preferred for making modifications. - -1.14. “You” (or “Your”) -means an individual or a legal entity exercising rights under this License. For -legal entities, “You” includes any entity that controls, is controlled by, -or is under common control with You. For purposes of this definition, -“control” means (a) the power, direct or indirect, to cause the direction or -management of such entity, whether by contract or otherwise, or (b) ownership of -more than fifty percent (50%) of the outstanding shares or beneficial ownership -of such entity. - -2. License Grants and Conditions - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive -license: - -under intellectual property rights (other than patent or trademark) Licensable -by such Contributor to use, reproduce, make available, modify, display, perform, -distribute, and otherwise exploit its Contributions, either on an unmodified -basis, with Modifications, or as part of a Larger Work; and - -under Patent Claims of such Contributor to make, use, sell, offer for sale, have -made, import, and otherwise transfer either its Contributions or its Contributor -Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution become -effective for each Contribution on the date the Contributor first distributes -such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under this -License. No additional rights or licenses will be implied from the distribution -or licensing of Covered Software under this License. Notwithstanding Section -2.1(b) above, no patent license is granted by a Contributor: - -for any code that a Contributor has removed from Covered Software; or - -for infringements caused by: (i) Your and any other third party’s -modifications of Covered Software, or (ii) the combination of its Contributions -with other software (except as part of its Contributor Version); or - -under Patent Claims infringed by Covered Software in the absence of its -Contributions. - -This License does not grant any rights in the trademarks, service marks, or -logos of any Contributor (except as may be necessary to comply with the notice -requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to distribute -the Covered Software under a subsequent version of this License (see Section -10.2) or under the terms of a Secondary License (if permitted under the terms of -Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its Contributions are -its original creation(s) or it has sufficient rights to grant the rights to its -Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under applicable -copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in -Section 2.1. - -3. Responsibilities - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under the -terms of this License. You must inform recipients that the Source Code Form of -the Covered Software is governed by the terms of this License, and how they can -obtain a copy of this License. You may not attempt to alter or restrict the -recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -such Covered Software must also be made available in Source Code Form, as -described in Section 3.1, and You must inform recipients of the Executable Form -how they can obtain a copy of such Source Code Form by reasonable means in a -timely manner, at a charge no more than the cost of distribution to the -recipient; and - -You may distribute such Executable Form under the terms of this License, or -sublicense it under different terms, provided that the license for the -Executable Form does not attempt to limit or alter the recipients’ rights in -the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, provided -that You also comply with the requirements of this License for the Covered -Software. If the Larger Work is a combination of Covered Software with a work -governed by one or more Secondary Licenses, and the Covered Software is not -Incompatible With Secondary Licenses, this License permits You to additionally -distribute such Covered Software under the terms of such Secondary License(s), -so that the recipient of the Larger Work may, at their option, further -distribute the Covered Software under the terms of either this License or such -Secondary License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices (including -copyright notices, patent notices, disclaimers of warranty, or limitations of -liability) contained within the Source Code Form of the Covered Software, except -that You may alter any license notices to the extent required to remedy known -factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, indemnity -or liability obligations to one or more recipients of Covered Software. However, -You may do so only on Your own behalf, and not on behalf of any Contributor. You -must make it absolutely clear that any such warranty, support, indemnity, or -liability obligation is offered by You alone, and You hereby agree to indemnify -every Contributor for any liability incurred by such Contributor as a result of -warranty, support, indemnity or liability terms You offer. You may include -additional disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - -If it is impossible for You to comply with any of the terms of this License with -respect to some or all of the Covered Software due to statute, judicial order, -or regulation then You must: (a) comply with the terms of this License to the -maximum extent possible; and (b) describe the limitations and the code they -affect. Such description must be placed in a text file included with all -distributions of the Covered Software under this License. Except to the extent -prohibited by statute or regulation, such description must be sufficiently -detailed for a recipient of ordinary skill to be able to understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You -fail to comply with any of its terms. However, if You become compliant, then the -rights granted under this License from a particular Contributor are reinstated -(a) provisionally, unless and until such Contributor explicitly and finally -terminates Your grants, and (b) on an ongoing basis, if such Contributor fails -to notify You of the non-compliance by some reasonable means prior to 60 days -after You have come back into compliance. Moreover, Your grants from a -particular Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the first -time You have received notice of non-compliance with this License from such -Contributor, and You become compliant prior to 30 days after Your receipt of the -notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, counter-claims, and -cross-claims) alleging that a Contributor Version directly or indirectly -infringes any patent, then the rights granted to You by any and all Contributors -for the Covered Software under Section 2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user -license agreements (excluding distributors and resellers) which have been -validly granted by You or Your distributors under this License prior to -termination shall survive termination. - -6. Disclaimer of Warranty - -Covered Software is provided under this License on an “as is” basis, without -warranty of any kind, either expressed, implied, or statutory, including, -without limitation, warranties that the Covered Software is free of defects, -merchantable, fit for a particular purpose or non-infringing. The entire risk as -to the quality and performance of the Covered Software is with You. Should any -Covered Software prove defective in any respect, You (not any Contributor) -assume the cost of any necessary servicing, repair, or correction. This -disclaimer of warranty constitutes an essential part of this License. No use of -any Covered Software is authorized under this License except under this -disclaimer. - -7. Limitation of Liability - -Under no circumstances and under no legal theory, whether tort (including -negligence), contract, or otherwise, shall any Contributor, or anyone who -distributes Covered Software as permitted above, be liable to You for any -direct, indirect, special, incidental, or consequential damages of any character -including, without limitation, damages for lost profits, loss of goodwill, work -stoppage, computer failure or malfunction, or any and all other commercial -damages or losses, even if such party shall have been informed of the -possibility of such damages. This limitation of liability shall not apply to -liability for death or personal injury resulting from such party’s negligence -to the extent applicable law prohibits such limitation. Some jurisdictions do -not allow the exclusion or limitation of incidental or consequential damages, so -this exclusion and limitation may not apply to You. - -8. Litigation - -Any litigation relating to this License may be brought only in the courts of a -jurisdiction where the defendant maintains its principal place of business and -such litigation shall be governed by laws of that jurisdiction, without -reference to its conflict-of-law provisions. Nothing in this Section shall -prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - -This License represents the complete agreement concerning the subject matter -hereof. If any provision of this License is held to be unenforceable, such -provision shall be reformed only to the extent necessary to make it enforceable. -Any law or regulation which provides that the language of a contract shall be -construed against the drafter shall not be used to construe this License against -a Contributor. - -10. Versions of the License - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section 10.3, -no one other than the license steward has the right to modify or publish new -versions of this License. Each version will be given a distinguishing version -number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version of the -License under which You originally received the Covered Software, or under the -terms of any subsequent version published by the license steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to create a -new license for such software, you may create and use a modified version of this -License if you rename the license and remove any references to the name of the -license steward (except to note that such modified license differs from this -License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - -If You choose to distribute Source Code Form that is Incompatible With Secondary -Licenses under the terms of this version of the License, the notice described in -Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - -This Source Code Form is subject to the terms of the Mozilla Public License, v. -2.0. If a copy of the MPL was not distributed with this file, You can obtain one -at https://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - -This Source Code Form is “Incompatible With Secondary Licenses”, as defined -by the Mozilla Public License, v. 2.0. - -The binary distribution of this product bundles these dependencies under the -following license: -JDOM 1.1 --------------------------------------------------------------------------------- -/*-- - - Copyright (C) 2000-2004 Jason Hunter & Brett McLaughlin. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions, and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions, and the disclaimer that follows - these conditions in the documentation and/or other materials - provided with the distribution. - - 3. The name "JDOM" must not be used to endorse or promote products - derived from this software without prior written permission. For - written permission, please contact . - - 4. Products derived from this software may not be called "JDOM", nor - may "JDOM" appear in their name, without prior written permission - from the JDOM Project Management . - - In addition, we request (but do not require) that you include in the - end-user documentation provided with the redistribution and/or in the - software itself an acknowledgement equivalent to the following: - "This product includes software developed by the - JDOM Project (http://www.jdom.org/)." - Alternatively, the acknowledgment may be graphical using the logos - available at http://www.jdom.org/images/logos. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED - WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - - This software consists of voluntary contributions made by many - individuals on behalf of the JDOM Project and was originally - created by Jason Hunter and - Brett McLaughlin . For more information - on the JDOM Project, please see . - - */ - -The binary distribution of this product bundles these dependencies under the -following license: -Hbase Server 1.2.4 --------------------------------------------------------------------------------- -This project bundles a derivative image for our Orca Logo. This image is -available under the Creative Commons By Attribution 3.0 License. - - Creative Commons Legal Code - - Attribution 3.0 Unported - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR - DAMAGES RESULTING FROM ITS USE. - - License - - THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE - COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY - COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS - AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. - - BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE - TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY - BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS - CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND - CONDITIONS. - - 1. Definitions - - a. "Adaptation" means a work based upon the Work, or upon the Work and - other pre-existing works, such as a translation, adaptation, - derivative work, arrangement of music or other alterations of a - literary or artistic work, or phonogram or performance and includes - cinematographic adaptations or any other form in which the Work may be - recast, transformed, or adapted including in any form recognizably - derived from the original, except that a work that constitutes a - Collection will not be considered an Adaptation for the purpose of - this License. For the avoidance of doubt, where the Work is a musical - work, performance or phonogram, the synchronization of the Work in - timed-relation with a moving image ("synching") will be considered an - Adaptation for the purpose of this License. - b. "Collection" means a collection of literary or artistic works, such as - encyclopedias and anthologies, or performances, phonograms or - broadcasts, or other works or subject matter other than works listed - in Section 1(f) below, which, by reason of the selection and - arrangement of their contents, constitute intellectual creations, in - which the Work is included in its entirety in unmodified form along - with one or more other contributions, each constituting separate and - independent works in themselves, which together are assembled into a - collective whole. A work that constitutes a Collection will not be - considered an Adaptation (as defined above) for the purposes of this - License. - c. "Distribute" means to make available to the public the original and - copies of the Work or Adaptation, as appropriate, through sale or - other transfer of ownership. - d. "Licensor" means the individual, individuals, entity or entities that - offer(s) the Work under the terms of this License. - e. "Original Author" means, in the case of a literary or artistic work, - the individual, individuals, entity or entities who created the Work - or if no individual or entity can be identified, the publisher; and in - addition (i) in the case of a performance the actors, singers, - musicians, dancers, and other persons who act, sing, deliver, declaim, - play in, interpret or otherwise perform literary or artistic works or - expressions of folklore; (ii) in the case of a phonogram the producer - being the person or legal entity who first fixes the sounds of a - performance or other sounds; and, (iii) in the case of broadcasts, the - organization that transmits the broadcast. - f. "Work" means the literary and/or artistic work offered under the terms - of this License including without limitation any production in the - literary, scientific and artistic domain, whatever may be the mode or - form of its expression including digital form, such as a book, - pamphlet and other writing; a lecture, address, sermon or other work - of the same nature; a dramatic or dramatico-musical work; a - choreographic work or entertainment in dumb show; a musical - composition with or without words; a cinematographic work to which are - assimilated works expressed by a process analogous to cinematography; - a work of drawing, painting, architecture, sculpture, engraving or - lithography; a photographic work to which are assimilated works - expressed by a process analogous to photography; a work of applied - art; an illustration, map, plan, sketch or three-dimensional work - relative to geography, topography, architecture or science; a - performance; a broadcast; a phonogram; a compilation of data to the - extent it is protected as a copyrightable work; or a work performed by - a variety or circus performer to the extent it is not otherwise - considered a literary or artistic work. - g. "You" means an individual or entity exercising rights under this - License who has not previously violated the terms of this License with - respect to the Work, or who has received express permission from the - Licensor to exercise rights under this License despite a previous - violation. - h. "Publicly Perform" means to perform public recitations of the Work and - to communicate to the public those public recitations, by any means or - process, including by wire or wireless means or public digital - performances; to make available to the public Works in such a way that - members of the public may access these Works from a place and at a - place individually chosen by them; to perform the Work to the public - by any means or process and the communication to the public of the - performances of the Work, including by public digital performance; to - broadcast and rebroadcast the Work by any means including signs, - sounds or images. - i. "Reproduce" means to make copies of the Work by any means including - without limitation by sound or visual recordings and the right of - fixation and reproducing fixations of the Work, including storage of a - protected performance or phonogram in digital form or other electronic - medium. - - 2. Fair Dealing Rights. Nothing in this License is intended to reduce, - limit, or restrict any uses free from copyright or rights arising from - limitations or exceptions that are provided for in connection with the - copyright protection under copyright law or other applicable laws. - - 3. License Grant. Subject to the terms and conditions of this License, - Licensor hereby grants You a worldwide, royalty-free, non-exclusive, - perpetual (for the duration of the applicable copyright) license to - exercise the rights in the Work as stated below: - - a. to Reproduce the Work, to incorporate the Work into one or more - Collections, and to Reproduce the Work as incorporated in the - Collections; - b. to create and Reproduce Adaptations provided that any such Adaptation, - including any translation in any medium, takes reasonable steps to - clearly label, demarcate or otherwise identify that changes were made - to the original Work. For example, a translation could be marked "The - original work was translated from English to Spanish," or a - modification could indicate "The original work has been modified."; - c. to Distribute and Publicly Perform the Work including as incorporated - in Collections; and, - d. to Distribute and Publicly Perform Adaptations. - e. For the avoidance of doubt: - - i. Non-waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme cannot be waived, the Licensor - reserves the exclusive right to collect such royalties for any - exercise by You of the rights granted under this License; - ii. Waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme can be waived, the Licensor waives the - exclusive right to collect such royalties for any exercise by You - of the rights granted under this License; and, - iii. Voluntary License Schemes. The Licensor waives the right to - collect royalties, whether individually or, in the event that the - Licensor is a member of a collecting society that administers - voluntary licensing schemes, via that society, from any exercise - by You of the rights granted under this License. - - The above rights may be exercised in all media and formats whether now - known or hereafter devised. The above rights include the right to make - such modifications as are technically necessary to exercise the rights in - other media and formats. Subject to Section 8(f), all rights not expressly - granted by Licensor are hereby reserved. - - 4. Restrictions. The license granted in Section 3 above is expressly made - subject to and limited by the following restrictions: - - a. You may Distribute or Publicly Perform the Work only under the terms - of this License. You must include a copy of, or the Uniform Resource - Identifier (URI) for, this License with every copy of the Work You - Distribute or Publicly Perform. You may not offer or impose any terms - on the Work that restrict the terms of this License or the ability of - the recipient of the Work to exercise the rights granted to that - recipient under the terms of the License. You may not sublicense the - Work. You must keep intact all notices that refer to this License and - to the disclaimer of warranties with every copy of the Work You - Distribute or Publicly Perform. When You Distribute or Publicly - Perform the Work, You may not impose any effective technological - measures on the Work that restrict the ability of a recipient of the - Work from You to exercise the rights granted to that recipient under - the terms of the License. This Section 4(a) applies to the Work as - incorporated in a Collection, but this does not require the Collection - apart from the Work itself to be made subject to the terms of this - License. If You create a Collection, upon notice from any Licensor You - must, to the extent practicable, remove from the Collection any credit - as required by Section 4(b), as requested. If You create an - Adaptation, upon notice from any Licensor You must, to the extent - practicable, remove from the Adaptation any credit as required by - Section 4(b), as requested. - b. If You Distribute, or Publicly Perform the Work or any Adaptations or - Collections, You must, unless a request has been made pursuant to - Section 4(a), keep intact all copyright notices for the Work and - provide, reasonable to the medium or means You are utilizing: (i) the - name of the Original Author (or pseudonym, if applicable) if supplied, - and/or if the Original Author and/or Licensor designate another party - or parties (e.g., a sponsor institute, publishing entity, journal) for - attribution ("Attribution Parties") in Licensor's copyright notice, - terms of service or by other reasonable means, the name of such party - or parties; (ii) the title of the Work if supplied; (iii) to the - extent reasonably practicable, the URI, if any, that Licensor - specifies to be associated with the Work, unless such URI does not - refer to the copyright notice or licensing information for the Work; - and (iv) , consistent with Section 3(b), in the case of an Adaptation, - a credit identifying the use of the Work in the Adaptation (e.g., - "French translation of the Work by Original Author," or "Screenplay - based on original Work by Original Author"). The credit required by - this Section 4 (b) may be implemented in any reasonable manner; - provided, however, that in the case of a Adaptation or Collection, at - a minimum such credit will appear, if a credit for all contributing - authors of the Adaptation or Collection appears, then as part of these - credits and in a manner at least as prominent as the credits for the - other contributing authors. For the avoidance of doubt, You may only - use the credit required by this Section for the purpose of attribution - in the manner set out above and, by exercising Your rights under this - License, You may not implicitly or explicitly assert or imply any - connection with, sponsorship or endorsement by the Original Author, - Licensor and/or Attribution Parties, as appropriate, of You or Your - use of the Work, without the separate, express prior written - permission of the Original Author, Licensor and/or Attribution - Parties. - c. Except as otherwise agreed in writing by the Licensor or as may be - otherwise permitted by applicable law, if You Reproduce, Distribute or - Publicly Perform the Work either by itself or as part of any - Adaptations or Collections, You must not distort, mutilate, modify or - take other derogatory action in relation to the Work which would be - prejudicial to the Original Author's honor or reputation. Licensor - agrees that in those jurisdictions (e.g. Japan), in which any exercise - of the right granted in Section 3(b) of this License (the right to - make Adaptations) would be deemed to be a distortion, mutilation, - modification or other derogatory action prejudicial to the Original - Author's honor and reputation, the Licensor will waive or not assert, - as appropriate, this Section, to the fullest extent permitted by the - applicable national law, to enable You to reasonably exercise Your - right under Section 3(b) of this License (right to make Adaptations) - but not otherwise. - - 5. Representations, Warranties and Disclaimer - - UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR - OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY - KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, - INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, - FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF - LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, - WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION - OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. - - 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE - LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR - ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES - ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS - BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - 7. Termination - - a. This License and the rights granted hereunder will terminate - automatically upon any breach by You of the terms of this License. - Individuals or entities who have received Adaptations or Collections - from You under this License, however, will not have their licenses - terminated provided such individuals or entities remain in full - compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will - survive any termination of this License. - b. Subject to the above terms and conditions, the license granted here is - perpetual (for the duration of the applicable copyright in the Work). - Notwithstanding the above, Licensor reserves the right to release the - Work under different license terms or to stop distributing the Work at - any time; provided, however that any such election will not serve to - withdraw this License (or any other license that has been, or is - required to be, granted under the terms of this License), and this - License will continue in full force and effect unless terminated as - stated above. - - 8. Miscellaneous - - a. Each time You Distribute or Publicly Perform the Work or a Collection, - the Licensor offers to the recipient a license to the Work on the same - terms and conditions as the license granted to You under this License. - b. Each time You Distribute or Publicly Perform an Adaptation, Licensor - offers to the recipient a license to the original Work on the same - terms and conditions as the license granted to You under this License. - c. If any provision of this License is invalid or unenforceable under - applicable law, it shall not affect the validity or enforceability of - the remainder of the terms of this License, and without further action - by the parties to this agreement, such provision shall be reformed to - the minimum extent necessary to make such provision valid and - enforceable. - d. No term or provision of this License shall be deemed waived and no - breach consented to unless such waiver or consent shall be in writing - and signed by the party to be charged with such waiver or consent. - e. This License constitutes the entire agreement between the parties with - respect to the Work licensed here. There are no understandings, - agreements or representations with respect to the Work not specified - here. Licensor shall not be bound by any additional provisions that - may appear in any communication from You. This License may not be - modified without the mutual written agreement of the Licensor and You. - f. The rights granted under, and the subject matter referenced, in this - License were drafted utilizing the terminology of the Berne Convention - for the Protection of Literary and Artistic Works (as amended on - September 28, 1979), the Rome Convention of 1961, the WIPO Copyright - Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 - and the Universal Copyright Convention (as revised on July 24, 1971). - These rights and subject matter take effect in the relevant - jurisdiction in which the License terms are sought to be enforced - according to the corresponding provisions of the implementation of - those treaty provisions in the applicable national law. If the - standard suite of rights granted under applicable copyright law - includes additional rights not granted under this License, such - additional rights are deemed to be included in the License; this - License is not intended to restrict the license of any rights under - applicable law. - - Creative Commons Notice - - Creative Commons is not a party to this License, and makes no warranty - whatsoever in connection with the Work. Creative Commons will not be - liable to You or any party on any legal theory for any damages - whatsoever, including without limitation any general, special, - incidental or consequential damages arising in connection to this - license. Notwithstanding the foregoing two (2) sentences, if Creative - Commons has expressly identified itself as the Licensor hereunder, it - shall have all rights and obligations of Licensor. - - Except for the limited purpose of indicating to the public that the - Work is licensed under the CCPL, Creative Commons does not authorize - the use by either party of the trademark "Creative Commons" or any - related trademark or logo of Creative Commons without the prior - written consent of Creative Commons. Any permitted use will be in - compliance with Creative Commons' then-current trademark usage - guidelines, as may be published on its website or otherwise made - available upon request from time to time. For the avoidance of doubt, - this trademark restriction does not form part of this License. - - Creative Commons may be contacted at https://creativecommons.org/. --------------------------------------------------------------------------------- - -For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs -/server/datanode/checker/AbstractFuture.java and -hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs -/server/datanode/checker/TimeoutFuture.java - -Copyright (C) 2007 The Guava Authors - -Licensed under the Apache License, Version 2.0 (the "License"); you may not -use this file except in compliance with the License. You may obtain a copy of -the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations under -the License. - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). - -The binary distribution of this product bundles binaries of -org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the -following notices: -* Copyright 2011 Dain Sundstrom -* Copyright 2011 FuseSource Corp. http://fusesource.com - -The binary distribution of this product bundles binaries of -AWS SDK for Java - Bundle 1.11.134, -AWS Java SDK for AWS KMS 1.11.134, -AWS Java SDK for Amazon S3 1.11.134, -AWS Java SDK for AWS STS 1.11.134, -JMES Path Query library 1.0, -which has the following notices: - * This software includes third party software subject to the following - copyrights: - XML parsing and utility functions from JetS3t - Copyright - 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - - Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility - functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. - -The binary distribution of this product bundles binaries of -Gson 2.2.4, -which has the following notices: - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'ArrayDeque', written by Josh -Bloch of Google, Inc: - - * LICENSE: - * license/LICENSE.deque.txt (Public Domain) - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified version of Roland Kuhn's ASL2 -AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. -It can be obtained at: - - * LICENSE: - * license/LICENSE.abstractnodequeue.txt (Public Domain) - * HOMEPAGE: - * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/libdivsufsort/ - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/protobuf/ - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/snappy/ - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * http://code.google.com/p/caliper/ - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - -This product contains a modified portion of 'JDOM 1.1', which can be obtained at: - - * LICENSE: - * https://github.com/hunterhacker/jdom/blob/jdom-1.1/core/LICENSE.txt - * HOMEPAGE: - * http://www.jdom.org/ - -The binary distribution of this product bundles binaries of -Commons Codec 1.4, -which has the following notices: - * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - =============================================================================== - The content of package org.apache.commons.codec.language.bm has been translated - from the original php source code available at http://stevemorse.org/phoneticinfo.htm - with permission from the original authors. - Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -The binary distribution of this product bundles binaries of -Commons Lang 2.6, -which has the following notices: - * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -The binary distribution of this product bundles binaries of -Apache Log4j 1.2.17, -which has the following notices: - * ResolverUtil.java - Copyright 2005-2006 Tim Fennell - Dumbster SMTP test server - Copyright 2004 Jason Paul Kitchen - TypeUtil.java - Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams - -The binary distribution of this product bundles binaries of -"Java Concurrency in Practice" book annotations 1.0, -which has the following notices: - * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative - Commons Attribution License (http://creativecommons.org/licenses/by/2.5) - Official home: http://www.jcip.net Any republication or derived work - distributed in source code form must include this copyright and license - notice. - -The binary distribution of this product bundles binaries of -Jetty :: Http Utility 9.3.19., -Jetty :: IO Utility 9.3.19., -Jetty :: Security 9.3.19., -Jetty :: Server Core 9.3.19., -Jetty :: Servlet Handling 9.3.19., -Jetty :: Utilities 9.3.19., -Jetty :: Utilities :: Ajax, -Jetty :: Webapp Application Support 9.3.19., -Jetty :: XML utilities 9.3.19., -which has the following notices: - * ============================================================== - Jetty Web Container - Copyright 1995-2016 Mort Bay Consulting Pty Ltd. - ============================================================== - - The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd - unless otherwise noted. - - Jetty is dual licensed under both - - * The Apache 2.0 License - http://www.apache.org/licenses/LICENSE-2.0.html - - and - - * The Eclipse Public 1.0 License - http://www.eclipse.org/legal/epl-v10.html - - Jetty may be distributed under either license. - - ------ - Eclipse - - The following artifacts are EPL. - * org.eclipse.jetty.orbit:org.eclipse.jdt.core - - The following artifacts are EPL and ASL2. - * org.eclipse.jetty.orbit:javax.security.auth.message - - The following artifacts are EPL and CDDL 1.0. - * org.eclipse.jetty.orbit:javax.mail.glassfish - - ------ - Oracle - - The following artifacts are CDDL + GPLv2 with classpath exception. - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - * javax.servlet:javax.servlet-api - * javax.annotation:javax.annotation-api - * javax.transaction:javax.transaction-api - * javax.websocket:javax.websocket-api - - ------ - Oracle OpenJDK - - If ALPN is used to negotiate HTTP/2 connections, then the following - artifacts may be included in the distribution or downloaded when ALPN - module is selected. - - * java.sun.security.ssl - - These artifacts replace/modify OpenJDK classes. The modififications - are hosted at github and both modified and original are under GPL v2 with - classpath exceptions. - http://openjdk.java.net/legal/gplv2+ce.html - - ------ - OW2 - - The following artifacts are licensed by the OW2 Foundation according to the - terms of http://asm.ow2.org/license.html - - org.ow2.asm:asm-commons - org.ow2.asm:asm - - ------ - Apache - - The following artifacts are ASL2 licensed. - - org.apache.taglibs:taglibs-standard-spec - org.apache.taglibs:taglibs-standard-impl - - ------ - MortBay - - The following artifacts are ASL2 licensed. Based on selected classes from - following Apache Tomcat jars, all ASL2 licensed. - - org.mortbay.jasper:apache-jsp - org.apache.tomcat:tomcat-jasper - org.apache.tomcat:tomcat-juli - org.apache.tomcat:tomcat-jsp-api - org.apache.tomcat:tomcat-el-api - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-api - org.apache.tomcat:tomcat-util-scan - org.apache.tomcat:tomcat-util - - org.mortbay.jasper:apache-el - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-el-api - - ------ - Mortbay - - The following artifacts are CDDL + GPLv2 with classpath exception. - - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - org.eclipse.jetty.toolchain:jetty-schemas - - ------ - Assorted - - The UnixCrypt.java code implements the one way cryptography used by - Unix systems for simple password protection. Copyright 1996 Aki Yoshida, - modified April 2001 by Iris Van den Broeke, Daniel Deville. - Permission to use, copy, modify and distribute UnixCrypt - for non-commercial or commercial purposes and without fee is - granted provided that the copyright notice appears in all copies./ - -The binary distribution of this product bundles binaries of -Snappy for Java 1.0.4.1, -which has the following notices: - * This product includes software developed by Google - Snappy: http://code.google.com/p/snappy/ (New BSD License) - - This product includes software developed by Apache - PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ - (Apache 2.0 license) - - This library containd statically linked libstdc++. This inclusion is allowed by - "GCC RUntime Library Exception" - http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html - - == Contributors == - * Tatu Saloranta - * Providing benchmark suite - * Alec Wysoker - * Performance and memory usage improvement - -The binary distribution of this product bundles binaries of -Xerces2 Java Parser 2.9.1, -which has the following notices: - * ========================================================================= - == NOTICE file corresponding to section 4(d) of the Apache License, == - == Version 2.0, in this case for the Apache Xerces Java distribution. == - ========================================================================= - - Apache Xerces Java - Copyright 1999-2007 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were originally based on the following: - - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. - - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. - - voluntary contributions made by Paul Eng on behalf of the - Apache Software Foundation that were originally developed at iClick, Inc., - software copyright (c) 1999. - -The binary distribution of this product bundles binaries of -Logback Classic Module 1.1.2, -Logback Core Module 1.1.2, -which has the following notices: - * Logback: the reliable, generic, fast and flexible logging framework. - Copyright (C) 1999-2012, QOS.ch. All rights reserved. - -The binary distribution of this product bundles binaries of -Apache HBase - Annotations 1.2.6, -Apache HBase - Client 1.2.6, -Apache HBase - Common 1.2.6, -Apache HBase - Hadoop Compatibility 1.2.6, -Apache HBase - Hadoop Two Compatibility 1.2.6, -Apache HBase - Prefix Tree 1.2.6, -Apache HBase - Procedure 1.2.6, -Apache HBase - Protocol 1.2.6, -Apache HBase - Server 1.2.6, -which has the following notices: - * Apache HBase - Copyright 2007-2015 The Apache Software Foundation - - -- - This product incorporates portions of the 'Hadoop' project - - Copyright 2007-2009 The Apache Software Foundation - - Licensed under the Apache License v2.0 - -- - Our Orca logo we got here: http://www.vectorfree.com/jumping-orca - It is licensed Creative Commons Attribution 3.0. - See https://creativecommons.org/licenses/by/3.0/us/ - We changed the logo by stripping the colored background, inverting - it and then rotating it some. - - Later we found that vectorfree.com image is not properly licensed. - The original is owned by vectorportal.com. The original was - relicensed so we could use it as Creative Commons Attribution 3.0. - The license is bundled with the download available here: - http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp - -- - This product includes portions of the Bootstrap project v3.0.0 - - Copyright 2013 Twitter, Inc. - - Licensed under the Apache License v2.0 - - This product uses the Glyphicons Halflings icon set. - - http://glyphicons.com/ - - Copyright Jan Kovařík - - Licensed under the Apache License v2.0 as a part of the Bootstrap project. - - -- - This product includes portions of the Guava project v14, specifically - 'hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java' - - Copyright (C) 2007 The Guava Authors - - Licensed under the Apache License, Version 2.0 - -The binary distribution of this product bundles binaries of -Phoenix Core 4.7.0, -which has the following notices: - Apache Phoenix - Copyright 2013-2016 The Apache Software Foundation - - This product includes software developed by The Apache Software - Foundation (http://www.apache.org/). - - This also includes: - - The phoenix-spark module has been adapted from the phoenix-spark library - distributed under the terms of the Apache 2 license. Original source copyright: - Copyright 2014 Simply Measured, Inc. - Copyright 2015 Interset Software Inc. - - The file bin/daemon.py is based on the file of the same name in python-daemon 2.0.5 - (https://pypi.python.org/pypi/python-daemon/). Original source copyright: - # Copyright © 2008–2015 Ben Finney - # Copyright © 2007–2008 Robert Niederreiter, Jens Klein - # Copyright © 2004–2005 Chad J. Schroeder - # Copyright © 2003 Clark Evans - # Copyright © 2002 Noah Spurrier - # Copyright © 2001 Jürgen Hermann - -The binary distribution of this product bundles binaries of -Plexus Cipher: encryption/decryption Component 1.4, -which has the following notices: - * The code in this component contains a class - Base64 taken from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/Base64.java - which is Apache license: http://www.apache.org/licenses/LICENSE-2.0 - - The PBE key processing routine PBECipher.createCipher() is adopted from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/OpenSSL.java - which is also Apache APL-2.0 license: http://www.apache.org/licenses/LICENSE-2.0 - -The binary distribution of this product bundles binaries of -software.amazon.ion:ion-java 1.0.1, -which has the following notices: - * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -The binary distribution of this product bundles binaries of -joda-time:joda-time:2.9.9 -which has the following notices: - * ============================================================================= - = NOTICE file corresponding to section 4d of the Apache License Version 2.0 = - ============================================================================= - This product includes software developed by - Joda.org (http://www.joda.org/). - -The binary distribution of this product bundles binaries of -Ehcache 3.3.1, -which has the following notices: - * Ehcache V3 Copyright 2014-2016 Terracotta, Inc. - -The binary distribution of this product bundles binaries of -snakeyaml (https://bitbucket.org/asomov/snakeyaml), -which has the following notices: - * Copyright (c) 2008, http://www.snakeyaml.org - -The binary distribution of this product bundles binaries of -swagger-annotations (https://github.com/swagger-api/swagger-core), -which has the following notices: - * Copyright 2016 SmartBear Software - -The binary distribution of this product bundles binaries of -metrics-core 3.2.4 -which has the following notices: - * Copyright 2010-2013 Coda Hale and Yammer, Inc. - - This product includes software developed by Coda Hale and Yammer, Inc. - - This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, - LongAdder), which was released with the following comments: - - Written by Doug Lea with assistance from members of JCP JSR-166 - Expert Group and released to the public domain, as explained at - http://creativecommons.org/publicdomain/zero/1.0/ - -Apache Commons IO -Copyright 2002-2012 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -Apache Commons Collections -Copyright 2001-2015 The Apache Software Foundation - -Apache Commons Logging -Copyright 2003-2013 The Apache Software Foundation - -Apache Commons Lang -Copyright 2001-2011 The Apache Software Foundation - -Apache Commons BeanUtils -Copyright 2000-2016 The Apache Software Foundation - -Apache Commons Configuration -Copyright 2001-2017 The Apache Software Foundation - -Apache Commons Lang -Copyright 2001-2014 The Apache Software Foundation - -This product includes software from the Spring Framework, -under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -htrace-core4 -Copyright 2016 The Apache Software Foundation - -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -Apache HttpClient -Copyright 1999-2017 The Apache Software Foundation - -Apache HttpCore -Copyright 2005-2017 The Apache Software Foundation - -Apache Commons Codec -Copyright 2002-2014 The Apache Software Foundation - -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-cep-scala -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-cep -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-dist -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.data-artisans:frocksdbjni:5.17.2-artisans-1.0 -- com.google.code.findbugs:jsr305:1.3.9 -- com.twitter:chill-java:0.7.6 -- com.twitter:chill_2.11:0.7.6 -- com.typesafe:config:1.3.0 -- com.typesafe:ssl-config-core_2.11:0.3.7 -- com.typesafe.akka:akka-actor_2.11:2.5.1 -- com.typesafe.akka:akka-camel_2.11:2.5.21 -- com.typesafe.akka:akka-protobuf_2.11:2.5.21 -- com.typesafe.akka:akka-slf4j_2.11:2.5.21 -- com.typesafe.akka:akka-stream_2.11:2.5.21 -- commons-cli:commons-cli:1.3.1 -- commons-collections:commons-collections:3.2.2 -- commons-io:commons-io:2.4 -- org.apache.camel:camel-core:2.17.7 -- org.apache.commons:commons-compress:1.18 -- org.apache.commons:commons-lang3:3.3.2 -- org.apache.commons:commons-math3:3.5 -- org.javassist:javassist:3.19.0-GA -- org.objenesis:objenesis:2.1 -- org.xerial.snappy:snappy-java:1.1.4 - -This project bundles the following dependencies under the BSD license. -See bundled license files for details. - -- com.esotericsoftware.kryo:kryo:2.24.0 -- com.esotericsoftware.minlog:minlog:1.2 -- org.clapper:grizzled-slf4j_2.11:1.3.2 - -The following dependencies all share the same BSD license which you find under licenses/LICENSE.scala. - -- org.scala-lang:scala-compiler:2.11.12 -- org.scala-lang:scala-library:2.11.12 -- org.scala-lang:scala-reflect:2.11.12 -- org.scala-lang.modules:scala-java8-compat_2.11:0.7.0 -- org.scala-lang.modules:scala-parser-combinators_2.11:1.1.1 -- org.scala-lang.modules:scala-xml_2.11:1.0.5 - -This project bundles the following dependencies under the MIT/X11 license. -See bundled license files for details. - -- com.github.scopt:scopt_2.11:3.5.0 -- org.slf4j:slf4j-api:1.7.15 - -This project bundles "org.tukaani:xz:1.5". -This Java implementation of XZ has been put into the public domain, thus you can do -whatever you want with it. All the files in the package have been written by Lasse Collin, -but some files are heavily based on public domain code written by Igor Pavlov. - -This project bundles the following dependencies under the Creative Commons CC0 "No Rights Reserved". - -- org.reactivestreams:reactive-streams:1.0.2 - - -flink-core -Copyright 2014-2019 The Apache Software Foundation - -flink-annotations -Copyright 2014-2019 The Apache Software Foundation - -Apache Flink-shaded -Copyright 2006-2019 The Apache Software Foundation - -flink-shaded-asm7 -Copyright 2014-2018 The Apache Software Foundation - -- org.ow2.asm:asm:7.1 -- org.ow2.asm:asm-analysis:7.1 -- org.ow2.asm:asm-commons:7.1 -- org.ow2.asm:asm-tree:7.1 - -Apache Commons Lang -Copyright 2001-2014 The Apache Software Foundation - -This product includes software from the Spring Framework, -under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -Apache Commons Collections -Copyright 2001-2015 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -Apache Commons Compress -Copyright 2002-2018 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). - -The files in the package org.apache.commons.compress.archivers.sevenz -were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), -which has been placed in the public domain: - -"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) - -flink-shaded-guava-18 -Copyright 2014-2018 The Apache Software Foundation - -- com.google.guava:guava:18.0 - -flink-java -Copyright 2014-2019 The Apache Software Foundation - -Apache Commons Math -Copyright 2001-2015 The Apache Software Foundation - -This product includes software developed for Orekit by -CS Systèmes d'Information (http://www.c-s.fr/) -Copyright 2010-2012 CS Systèmes d'Information - -flink-scala -Copyright 2014-2019 The Apache Software Foundation - -flink-runtime -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.typesafe.akka:akka-remote_2.11:2.5.21 -- io.netty:netty:3.10.6.Final -- org.apache.zookeeper:zookeeper:3.4.10 -- org.uncommons.maths:uncommons-maths:1.2.2a - -This project bundles io.netty:netty:3.10.6.Final from which it inherits the following notices: - -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * licenses/LICENSE.jsr166y (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * licenses/LICENSE.base64 (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified version of 'JZlib', a re-implementation of -zlib in pure Java, which can be obtained at: - - * LICENSE: - * licenses/LICENSE.jzlib (BSD Style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product contains a modified version of 'Webbit', a Java event based -WebSocket and HTTP server: - - * LICENSE: - * licenses/LICENSE.webbit (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -flink-shaded-curator -Copyright 2014-2019 The Apache Software Foundation - -- com.google.guava:guava:16.0.1 -- org.apache.curator:curator-client:2.12.0 -- org.apache.curator:curator-framework:2.12.0 -- org.apache.curator:curator-recipes:2.12.0 - -Curator Recipes -Copyright 2011-2017 The Apache Software Foundation - -Curator Framework -Copyright 2011-2017 The Apache Software Foundation - -Curator Client -Copyright 2011-2017 The Apache Software Foundation - -flink-queryable-state-client-java -Copyright 2014-2019 The Apache Software Foundation - -Apache Commons IO -Copyright 2002-2012 The Apache Software Foundation - -flink-shaded-netty -Copyright 2014-2018 The Apache Software Foundation - -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -- io.netty:netty-all:4.1.39.Final - -flink-shaded-jackson -Copyright 2014-2018 The Apache Software Foundation - -- com.fasterxml.jackson.core:jackson-annotations:2.9.8 -- com.fasterxml.jackson.core:jackson-core:2.9.8 -- com.fasterxml.jackson.core:jackson-databind:2.9.8 -- com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.9.8 -- com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.9.8 -- org.yaml:snakeyaml:1.18 - -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -Apache Commons CLI -Copyright 2001-2015 The Apache Software Foundation - -flink-runtime-web -Copyright 2014-2019 The Apache Software Foundation - --------------------------------------------------------------------------------------------------------------------- - -The old frontend bundles the following dependencies: - -- nvd3#1.8.4 - -This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) -See bundled license files for details. - -- angular:1.4.8 -- angular-drag-and-drop-list:1.4.0 -- angular-moment:0.10.3 -- angular-ui-router:0.2.15 -- bootstrap:3.3.6 -- dagre:0.7.5 -- dagre-d3:0.4.17 -- ev-emitter:1.1.1 -- font-awesome:4.5.0 (CSS) -- graphlib:1.0.7 -- imagesloaded:4.1.4 -- jquery:2.2.0 -- lodash:3.10.1 -- moment:2.10.6 -- moment-duration-format:1.3.0 -- qtip2:2.2.1 -- Split.js:1.0.6 - -- d3:3.5.12 - -This project bundles the following dependencies under SIL OFL 1.1 license. (https://opensource.org/licenses/OFL-1.1) -See bundled license files for details. - -- font-awesome:4.5.0 (Font) - -The new frontend bundles the following dependencies: - -@angular/animations 7.2.10 : MIT License -@angular/cdk 7.3.6 : MIT License -@angular/common 7.2.10 : MIT License -@angular/compiler 7.2.10 : MIT License -@angular/core 7.2.10 : MIT License -@angular/forms 7.2.10 : MIT License -@angular/platform-browser 7.2.10 : MIT License -@angular/platform-browser-dynamic 7.2.10 : MIT License -@angular/router 7.2.10 : MIT License -@ant-design/icons-angular 2.0.2 : MIT License -@antv/adjust 0.1.1 : MIT License -@antv/attr 0.1.2 : MIT License -@antv/component 0.3.1 : MIT License -@antv/coord 0.1.0 : MIT License -@antv/g 3.3.6 : MIT License -@antv/g2 3.4.10 : MIT License -@antv/gl-matrix 2.7.1 : MIT License -@antv/scale 0.1.2 : MIT License -@antv/util 1.3.1 : ISC License -align-text 0.1.4 : MIT License -amdefine 1.0.1 : MIT License -ansi-regex 2.1.1 : MIT License -ant-design-palettes 1.1.3 : MIT License -balanced-match 1.0.0 : MIT License -brace-expansion 1.1.11 : MIT License -camelcase 1.2.1 : MIT License -center-align 0.1.3 : MIT License -Chalk 1.1.3 : MIT License -cliui 2.1.0 : ISC License -Commander.js 2.19.0 : MIT License -contour_plot 0.0.1 : MIT License -core-js v2.6.5 : MIT License -cpettitt/graphlib 2.1.7 : MIT License -d3-array 1.2.4 : BSD 3-clause "New" or "Revised" License -d3-axis 1.0.12 : BSD 3-clause "New" or "Revised" License -d3-brush 1.0.6 : BSD 3-clause "New" or "Revised" License -d3-chord 1.0.6 : BSD 3-clause "New" or "Revised" License -d3-collection 1.0.7 : BSD 3-clause "New" or "Revised" License -d3-color 1.2.3 : BSD 3-clause "New" or "Revised" License -d3-contour 1.3.2 : BSD 3-clause "New" or "Revised" License -d3-dispatch 1.0.5 : BSD 3-clause "New" or "Revised" License -d3-drag 1.2.3 : BSD 3-clause "New" or "Revised" License -d3-dsv 1.1.1 : BSD 3-clause "New" or "Revised" License -d3-ease 1.0.5 : BSD 3-clause "New" or "Revised" License -d3-fetch 1.1.2 : BSD 3-clause "New" or "Revised" License -d3-force 1.2.1 : BSD 3-clause "New" or "Revised" License -d3-format 1.3.2 : BSD 3-clause "New" or "Revised" License -d3-geo 1.11.3 : BSD 3-clause "New" or "Revised" License -d3-hierarchy 1.1.8 : BSD 3-clause "New" or "Revised" License -d3-interpolate v1.1.6 : BSD 3-clause "New" or "Revised" License -d3-path 1.0.7 : BSD 3-clause "New" or "Revised" License -d3-polygon v1.0.5 : BSD 3-clause "New" or "Revised" License -d3-quadtree 1.0.6 : BSD 3-clause "New" or "Revised" License -d3-random 1.1.2 : BSD 3-clause "New" or "Revised" License -d3-scale 2.2.2 : BSD 3-clause "New" or "Revised" License -d3-scale-chromatic 1.3.3 : BSD 3-clause "New" or "Revised" License -d3-selection v1.4.0 : BSD 3-clause "New" or "Revised" License -d3-shape v1.3.5 : BSD 3-clause "New" or "Revised" License -d3-time 1.0.11 : BSD 3-clause "New" or "Revised" License -d3-time-format 2.1.3 : BSD 3-clause "New" or "Revised" License -d3-timer v1.0.9 : BSD 3-clause "New" or "Revised" License -d3-transition v1.2.0 : BSD 3-clause "New" or "Revised" License -d3-voronoi v1.1.4 : (BSD 3-clause "New" or "Revised" License AND MIT License) -d3-zoom 1.7.3 : BSD 3-clause "New" or "Revised" License -D3.js 5.9.2 : BSD 3-clause "New" or "Revised" License -dagre 0.8.4 : MIT License -Decamelize 1.2.0 : MIT License -define-properties v1.1.3 : MIT License -defined 1.0.0 : MIT License -es-abstract 1.13.0 : MIT License -es-to-primitive 1.2.0 : MIT License -escape-string-regexp 1.0.5 : MIT License -fecha 2.3.3 : MIT License -fmin 0.0.2 : BSD 3-clause "New" or "Revised" License -fs.realpath 1.0.0 : ISC License -function-bind 1.1.1 : MIT License -has 1.0.3 : MIT License -has-ansi 2.0.0 : MIT License -has-symbols 1.0.0 : MIT License -iconv-lite v0.4.24 : MIT License -inflight 1.0.6 : ISC License -inherits 2.0.3 : ISC License -is-buffer 1.1.6 : MIT License -is-callable 1.1.4 : MIT License -is-date-object 1.0.1 : MIT License -is-regex 1.0.4 : MIT License -is-symbol 1.0.2 : MIT License -isaacs/once 1.4.0 : ISC License -json2module 0.0.3 : BSD 3-clause "New" or "Revised" License -kind-of 3.2.2 : MIT License -kossnocorp/date-fns 1.30.1 : MIT License -lazy-cache 1.0.4 : MIT License -Lo-Dash 4.17.11 : MIT License -longest 1.0.1 : MIT License -minimatch 3.0.4 : ISC License -minimist 1.2.0 : MIT License -monaco-editor 0.16.2 : MIT License -ng-zorro-antd 7.2.0 : MIT License -node-concat-map 0.0.1 : MIT License -node-deep-equal 1.0.1 : MIT License -node-glob v7.1.3 : ISC License -node-source-map-support 0.3.3 : MIT License -node-tape v4.10.1 : MIT License -object-inspect 1.6.0 : MIT License -object-keys 1.1.0 : MIT License -parse5 5.1.0 : MIT License -path-is-absolute 1.0.1 : MIT License -path-parse 1.0.6 : MIT License -Raynos/for-each 0.3.3 : MIT License -repeat-string 1.6.1 : MIT License -RESOLVE v1.10.0 : MIT License -resumer 0.0.0 : MIT License -right-align 0.1.3 : MIT License -rollup 0.25.8 : MIT License -rw 1.3.3 : BSD 3-clause "New" or "Revised" License -RxJS 6.3.3 : Apache License 2.0 - Copyright (c) 2015-2018 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors - - No NOTICE file was provided by RxJS -safer-buffer 2.1.2 : MIT License -sindresorhus/ansi-styles 2.2.1 : MIT License -sindresorhus/supports-color 2.0.0 : MIT License -source-map 0.1.32 : BSD 3-clause "New" or "Revised" License -source-map 0.5.7 : BSD 3-clause "New" or "Revised" License -string.prototype.trim 1.1.2 : MIT License -Strip ANSI 3.0.1 : MIT License -through 2.3.8 : MIT License -TinyColor 1.4.1 : MIT License -tslib 1.9.3 : Apache License 2.0 - Copyright (c) Microsoft Corporation. All rights reserved. - - No NOTICE file was provided. -uglify-to-browserify 1.0.2 : MIT License -UglifyJS 2.8.29 : BSD 2-clause "Simplified" License -venn.js 0.2.20 : MIT License -window-size 0.1.0 : MIT License -wolfy87-eventemitter 5.1.0 : The Unlicense -wordwrap 0.0.2 : MIT License -wrappy 1.0.2 : ISC License -yargs 3.10.0 : MIT License -Zone.js v0.8.29 : MIT License - -Licenses: - -Apache License 2.0 -(RxJS 6.3.3, tslib 1.9.3) - -Apache License -Version 2.0, January 2004 -========================= - -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions -granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is -based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work by -the copyright owner or by an individual or Legal Entity authorized to submit on -behalf of the copyright owner. For the purposes of this definition, "submitted" -means any form of electronic, verbal, or written communication sent to the -Licensor or its representatives, including but not limited to communication on -electronic mailing lists, source code control systems, and issue tracking systems -that are managed by, or on behalf of, the Licensor for the purpose of discussing -and improving the Work, but excluding communication that is conspicuously marked -or otherwise designated in writing by the copyright owner as "Not a -Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of -whom a Contribution has been received by Licensor and subsequently incorporated -within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of this -License, each Contributor hereby grants to You a perpetual, worldwide, -non-exclusive, no-charge, royalty-free, irrevocable copyright license to -reproduce, prepare Derivative Works of, publicly display, publicly perform, -sublicense, and distribute the Work and such Derivative Works in Source or Object -form. - -3. Grant of Patent License. Subject to the terms and conditions of this License, -each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, -no-charge, royalty-free, irrevocable (except as stated in this section) patent -license to make, have made, use, offer to sell, sell, import, and otherwise -transfer the Work, where such license applies only to those patent claims -licensable by such Contributor that are necessarily infringed by their -Contribution(s) alone or by combination of their Contribution(s) with the Work to -which such Contribution(s) was submitted. If You institute patent litigation -against any entity (including a cross-claim or counterclaim in a lawsuit) -alleging that the Work or a Contribution incorporated within the Work constitutes -direct or contributory patent infringement, then any patent licenses granted to -You under this License for that Work shall terminate as of the date such -litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the Work or -Derivative Works thereof in any medium, with or without modifications, and in -Source or Object form, provided that You meet the following conditions: - - a. You must give any other recipients of the Work or Derivative Works a copy of - this License; and - - b. You must cause any modified files to carry prominent notices stating that - You changed the files; and - - c. You must retain, in the Source form of any Derivative Works that You - distribute, all copyright, patent, trademark, and attribution notices from - the Source form of the Work, excluding those notices that do not pertain to - any part of the Derivative Works; and - - d. If the Work includes a "NOTICE" text file as part of its distribution, then - any Derivative Works that You distribute must include a readable copy of the - attribution notices contained within such NOTICE file, excluding those - notices that do not pertain to any part of the Derivative Works, in at least - one of the following places: within a NOTICE text file distributed as part of - the Derivative Works; within the Source form or documentation, if provided - along with the Derivative Works; or, within a display generated by the - Derivative Works, if and wherever such third-party notices normally appear. - The contents of the NOTICE file are for informational purposes only and do - not modify the License. You may add Your own attribution notices within - Derivative Works that You distribute, alongside or as an addendum to the - NOTICE text from the Work, provided that such additional attribution notices - cannot be construed as modifying the License. - -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, any -Contribution intentionally submitted for inclusion in the Work by You to the -Licensor shall be under the terms and conditions of this License, without any -additional terms or conditions. Notwithstanding the above, nothing herein shall -supersede or modify the terms of any separate license agreement you may have -executed with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade names, -trademarks, service marks, or product names of the Licensor, except as required -for reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or agreed to in -writing, Licensor provides the Work (and each Contributor provides its -Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, -either express or implied, including, without limitation, any warranties or -conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A -PARTICULAR PURPOSE. You are solely responsible for determining the -appropriateness of using or redistributing the Work and assume any risks -associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, whether in -tort (including negligence), contract, or otherwise, unless required by -applicable law (such as deliberate and grossly negligent acts) or agreed to in -writing, shall any Contributor be liable to You for damages, including any -direct, indirect, special, incidental, or consequential damages of any character -arising as a result of this License or out of the use or inability to use the -Work (including but not limited to damages for loss of goodwill, work stoppage, -computer failure or malfunction, or any and all other commercial damages or -losses), even if such Contributor has been advised of the possibility of such -damages. - -9. Accepting Warranty or Additional Liability. While redistributing the Work or -Derivative Works thereof, You may choose to offer, and charge a fee for, -acceptance of support, warranty, indemnity, or other liability obligations and/or -rights consistent with this License. However, in accepting such obligations, You -may act only on Your own behalf and on Your sole responsibility, not on behalf of -any other Contributor, and only if You agree to indemnify, defend, and hold each -Contributor harmless for any liability incurred by, or claims asserted against, -such Contributor by reason of your accepting any such warranty or additional -liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also recommend -that a file or class name and description of purpose be included on the same -"printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, - Version 2.0 (the "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law - or agreed to in writing, software distributed under the License is - distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the specific language - governing permissions and limitations under the License. - ---- - -BSD 2-clause "Simplified" License -(UglifyJS 2.8.29) - -UglifyJS is released under the BSD license: - -Copyright 2012-2013 (c) Mihai Bazon - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - * Redistributions of source code must retain the above - copyright notice, this list of conditions and the following - disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials - provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER “AS IS” AND ANY -EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, -OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR -TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF -THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE - -BSD 3-clause "New" or "Revised" License -(d3-array 1.2.4, d3-axis 1.0.12, d3-brush 1.0.6, d3-chord 1.0.6, d3-collection 1.0.7, d3-color 1.2.3, d3-contour 1.3.2, d3-dispatch 1.0.5, d3-drag 1.2.3, d3-dsv 1.1.1, d3-ease 1.0.5, d3-fetch 1.1.2, d3-force 1.2.1, d3-format 1.3.2, d3-geo 1.11.3, d3-hierarchy 1.1.8, d3-interpolate v1.1.6, d3-path 1.0.7, d3-polygon v1.0.5, d3-quadtree 1.0.6, d3-random 1.1.2, d3-scale 2.2.2, d3-scale-chromatic 1.3.3, d3-selection v1.4.0, d3-shape v1.3.5, d3-time 1.0.11, d3-time-format 2.1.3, d3-timer v1.0.9, d3-transition v1.2.0, d3-voronoi v1.1.4, d3-zoom 1.7.3, D3.js 5.9.2, fmin 0.0.2, json2module 0.0.3, source-map 0.1.32) - -Copyright (c) , -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name of the nor the names of its contributors may - be used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS -OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -BSD 3-clause "New" or "Revised" License -(source-map 0.5.7) - -License: BSD-3-clause - -Files: debian/* -Copyright: 2014 Leo Iannacone -License: BSD-3-clause - -Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the University nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - . - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE HOLDERS OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE - -BSD 3-clause "New" or "Revised" License -(rw 1.3.3) - -Upstream-Contact: https://github.com/mbostock/rw/issues -Source: https://github.com/mbostock/rw - -Files: * -Copyright: 2014-2016 Mike Bostock (http://bost.ocks.org/mike) -License: BSD-3-Clause - -Files: debian/* -Copyright: 2017 Pirate Praveen -License: BSD-3-Clause - -Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the University nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - . - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE HOLDERS OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE - -ISC License -(cliui 2.1.0) - -Copyright (c) 2015, Contributors - -Permission to use, copy, modify, and/or distribute this software -for any purpose with or without fee is hereby granted, provided -that the above copyright notice and this permission notice -appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE -LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES -OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, -WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, -ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE - -ISC License -(@antv/util 1.3.1) - -ISC License (ISCL) -================== - -Copyright (c) 4-digit year, Company or Person's Name - -Permission to use, copy, modify, and/or distribute this software for any purpose -with or without fee is hereby granted, provided that the above copyright notice -and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS -OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER -TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF -THIS SOFTWARE. - -ISC License -(inflight 1.0.6) - -The ISC License - -Copyright (c) Isaac Z. Schlueter - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR -IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE - -ISC License -(inherits 2.0.3) - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE - -ISC License -(fs.realpath 1.0.0, isaacs/once 1.4.0, minimatch 3.0.4, node-glob v7.1.3) - -Copyright (c) Isaac Z. Schlueter and Contributors - -ISC License -(wrappy 1.0.2) - -Upstream-Contact: https://github.com/npm/wrappy/issues -Source: https://github.com/npm/wrappy - -Files: * -Copyright: 2015 Isaac Z. Schlueter (http://blog.izs.me/) -License: ISC - -Files: debian/* -Copyright: 2015 Thorsten Alteholz -License: ISC - -License: ISC - -Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - . - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE - -MIT License -(Commander.js 2.19.0) - -(The MIT License) - -Copyright (c) 2011 TJ Holowaychuk - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -'Software'), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE - -MIT License -(Chalk 1.1.3) - -2016, Mathias Behrle -License: Expat - -License: Expat - -Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation files - (the "Software"), to deal in the Software without restriction, - including without limitation the rights to use, copy, modify, merge, - publish, distribute, sublicense, and/or sell copies of the Software, - and to permit persons to whom the Software is furnished to do so, - subject to the following conditions: - . - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - . - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE - -MIT License -(Lo-Dash 4.17.11) - -==== - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE - -MIT License -(d3-voronoi v1.1.4) - -Copyright (C) 2010-2013 Raymond Hill -https://github.com/gorhill/Javascript-Voronoi - -Licensed under The MIT License -http://en.wikipedia.org/wiki/MIT_License - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE - -MIT License -(iconv-lite v0.4.24) - -Copyright (c) 2011 Alexander Shtuchkin - -MIT License -(venn.js 0.2.20) - -Copyright (C) 2013 Ben Frederickson - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE - -MIT License -(uglify-to-browserify 1.0.2) - -Copyright (c) 2013 Forbes Lindesay - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE - -MIT License -(balanced-match 1.0.0) - -Copyright (c) 2013 Julian Gruber <julian@juliangruber.com> - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE - -MIT License -(has 1.0.3) - -Copyright (c) 2013 Thiago de Arruda - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE - -MIT License -(parse5 5.1.0) - -Copyright (c) 2013-2018 Ivan Nikulin (ifaaan@gmail.com, https://github.com/inikulin) - -MIT License -(window-size 0.1.0) - -Copyright (c) 2014 Jon Schlinkert - -MIT License -(TinyColor 1.4.1) - -Copyright (c), Brian Grinstead, http://briangrinstead.com - -MIT License -(wordwrap 0.0.2) - -Files: * -Copyright: © 2010, James Halliday -License: MIT - -Files: debian/* -Copyright: © 2011, David Paleino -License: MIT - -License: MIT - -Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - . - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - . - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - THE SOFTWARE - -MIT License -(function-bind 1.1.1) - -MIT License -(brace-expansion 1.1.11) - -MIT License - -Copyright (c) 2013 Julian Gruber - -MIT License -(has-symbols 1.0.0) - -Copyright (c) 2016 Jordan Harband - -MIT License -(safer-buffer 2.1.2) - -Copyright (c) 2018 Nikita Skovoroda - -MIT License -(amdefine 1.0.1) - -MIT License ------------ - -Copyright (c) 2011-2016, The Dojo Foundation - -MIT License -(yargs 3.10.0) - -MIT License -(through 2.3.8) - -The MIT License - -Copyright (c) 2011 Dominic Tarr - -Permission is hereby granted, free of charge, -to any person obtaining a copy of this software and -associated documentation files (the "Software"), to -deal in the Software without restriction, including -without limitation the rights to use, copy, modify, -merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom -the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR -ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE - -MIT License -(@angular/animations 7.2.10, @angular/cdk 7.3.6, @angular/common 7.2.10, @angular/compiler 7.2.10, @angular/core 7.2.10, @angular/forms 7.2.10, @angular/platform-browser 7.2.10, @angular/platform-browser-dynamic 7.2.10, @angular/router 7.2.10, @ant-design/icons-angular 2.0.2, @antv/adjust 0.1.1, @antv/attr 0.1.2, @antv/component 0.3.1, @antv/coord 0.1.0, @antv/g 3.3.6, @antv/g2 3.4.10, @antv/gl-matrix 2.7.1, @antv/scale 0.1.2, ant-design-palettes 1.1.3, contour_plot 0.0.1, core-js v2.6.5, cpettitt/graphlib 2.1.7, dagre 0.8.4, es-abstract 1.13.0, is-callable 1.1.4, kossnocorp/date-fns 1.30.1, monaco-editor 0.16.2, ng-zorro-antd 7.2.0, node-tape v4.10.1, object-keys 1.1.0, RESOLVE v1.10.0, Zone.js v0.8.29) - -The MIT License -=============== - -Copyright (c) - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in the -Software without restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the -Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN -AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -MIT License -(Raynos/for-each 0.3.3) - -The MIT License (MIT) - -Copyright (c) 2012 Raynos. - -MIT License -(node-source-map-support 0.3.3) - -Copyright (c) 2014 Evan Wallace - -MIT License -(is-regex 1.0.4) - -Copyright (c) 2014 Jordan Harband - -MIT License -(longest 1.0.1) - -Copyright (c) 2014-2015, Jon Schlinkert. - -MIT License -(repeat-string 1.6.1) - -Copyright (c) 2014-2016, Jon Schlinkert. - -MIT License -(kind-of 3.2.2) - -Copyright (c) 2014-2017, Jon Schlinkert - -MIT License -(rollup 0.25.8) - -Copyright (c) 2015 [these people](https://github.com/rollup/rollup/graphs/contributors) - -MIT License -(path-parse 1.0.6) - -Copyright (c) 2015 Javier Blanco - -MIT License -(define-properties v1.1.3) - -Copyright (C) 2015 Jordan Harband - -MIT License -(es-to-primitive 1.2.0, is-date-object 1.0.1, is-symbol 1.0.2, string.prototype.trim 1.1.2) - -Copyright (c) 2015 Jordan Harband - -MIT License -(fecha 2.3.3) - -Copyright (c) 2015 Taylor Hakes - -MIT License -(align-text 0.1.4, center-align 0.1.3, right-align 0.1.3) - -Copyright (c) 2015, Jon Schlinkert. - -MIT License -(lazy-cache 1.0.4) - -Copyright (c) 2015-2016, Jon Schlinkert. - -MIT License -(is-buffer 1.1.6) - -Copyright (c) Feross Aboukhadijeh - -MIT License -(ansi-regex 2.1.1, camelcase 1.2.1, Decamelize 1.2.0, escape-string-regexp 1.0.5, has-ansi 2.0.0, path-is-absolute 1.0.1, sindresorhus/ansi-styles 2.2.1, sindresorhus/supports-color 2.0.0, Strip ANSI 3.0.1) - -Copyright (c) Sindre Sorhus (sindresorhus.com) - -MIT License -(defined 1.0.0, minimist 1.2.0, node-concat-map 0.0.1, node-deep-equal 1.0.1, object-inspect 1.6.0, resumer 0.0.0) - -This software is released under the MIT license: - -The Unlicense -(wolfy87-eventemitter 5.1.0) - -The Unlicense -============= - -This is free and unencumbered software released into the public domain. - -Anyone is free to copy, modify, publish, use, compile, sell, or distribute this -software, either in source code form or as a compiled binary, for any purpose, -commercial or non-commercial, and by any means. - -In jurisdictions that recognize copyright laws, the author or authors of this -software dedicate any and all copyright interest in the software to the public -domain. We make this dedication for the benefit of the public at large and to the -detriment of our heirs and successors. We intend this dedication to be an overt -act of relinquishment in perpetuity of all present and future rights to this -software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to http://unlicense.org/ - -flink-optimizer -Copyright 2014-2019 The Apache Software Foundation - -flink-clients -Copyright 2014-2019 The Apache Software Foundation - -flink-streaming-java -Copyright 2014-2019 The Apache Software Foundation - -flink-streaming-scala -Copyright 2014-2019 The Apache Software Foundation - -flink-metrics-core -Copyright 2014-2019 The Apache Software Foundation - -flink-metrics-jmx -Copyright 2014-2019 The Apache Software Foundation - -flink-mesos -Copyright 2014-2019 The Apache Software Foundation - -- com.netflix.fenzo:fenzo-core:0.10.1 -- org.apache.mesos:mesos:1.0.1 -- com.fasterxml.jackson.core:jackson-annotations:2.4.0 -- com.fasterxml.jackson.core:jackson-core:2.4.5 -- com.fasterxml.jackson.core:jackson-databind:2.4.5 - -- com.google.protobuf:protobuf-java:2.6.2 - -mesos -Copyright 2016 The Apache Software Foundation - -flink-container -Copyright 2014-2019 The Apache Software Foundation - -flink-statebackend-rocksdb -Copyright 2014-2019 The Apache Software Foundation - -flink-yarn -Copyright 2014-2019 The Apache Software Foundation - -flink-hadoop-fs -Copyright 2014-2019 The Apache Software Foundation - -flink-mapr-fs -Copyright 2014-2019 The Apache Software Foundation - -flink-scala-shell -Copyright 2014-2019 The Apache Software Foundation - -force-shading -Copyright 2019 The Apache Software Foundation - -Objenesis -Copyright 2006-2013 Joe Walnes, Henri Tremblay, Leonardo Mesquita - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-gelly-examples -Copyright 2014-2019 The Apache Software Foundation - - -force-shading -Copyright 2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-gelly -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-metrics-datadog -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.squareup.okhttp3:okhttp:3.7.0 -- com.squareup.okio:okio:1.12.0 - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-metrics-graphite -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- io.dropwizard.metrics:metrics-core:3.1.5 -- io.dropwizard.metrics:metrics-graphite:3.1.5 - - -flink-metrics-dropwizard -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-metrics-influxdb -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.squareup.moshi:moshi:1.5.0 -- com.squareup.okhttp3:logging-interceptor:3.11.0 -- com.squareup.okhttp3:okhttp:3.11.0 -- com.squareup.okio:okio:1.14.0 -- com.squareup.retrofit2:converter-moshi:2.4.0 -- com.squareup.retrofit2:retrofit:2.4.0 - -This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) - -- org.influxdb:influxdb-java:2.14 - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-metrics-prometheus -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- io.prometheus:simpleclient:0.3.0 -- io.prometheus:simpleclient_common:0.3.0 -- io.prometheus:simpleclient_httpserver:0.3.0 -- io.prometheus:simpleclient_pushgateway:0.3.0 - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-metrics-slf4j -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-metrics-statsd -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-oss-fs-hadoop -Copyright 2014-2019 The Apache Software Foundation - -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.aliyun.oss:aliyun-sdk-oss:3.4.1 -- com.aliyun:aliyun-java-sdk-core:3.4.0 -- com.aliyun:aliyun-java-sdk-ecs:4.2.0 -- com.aliyun:aliyun-java-sdk-ram:3.0.0 -- com.aliyun:aliyun-java-sdk-sts:3.0.0 -- com.fasterxml.jackson.core:jackson-annotations:2.7.0 -- com.fasterxml.jackson.core:jackson-core:2.7.8 -- com.fasterxml.jackson.core:jackson-databind:2.7.8 -- com.fasterxml.woodstox:woodstox-core:5.0.3 -- com.github.stephenc.jcip:jcip-annotations:1.0-1 -- com.google.code.gson:gson:2.2.4 -- com.google.guava:guava:11.0.2 -- com.nimbusds:nimbus-jose-jwt:4.41.1 -- commons-beanutils:commons-beanutils:1.9.3 -- commons-cli:commons-cli:1.3.1 -- commons-codec:commons-codec:1.10 -- commons-collections:commons-collections:3.2.2 -- commons-io:commons-io:2.4 -- commons-lang:commons-lang:3.3.2 -- commons-logging:commons-logging:1.1.3 -- commons-net:commons-net:3.6 -- net.minidev:accessors-smart:1.2 -- net.minidev:json-smart:2.3 -- org.apache.avro:avro:1.8.2 -- org.apache.commons:commons-compress:1.18 -- org.apache.commons:commons-configuration2:2.1.1 -- org.apache.commons:commons-lang3:3.3.2 -- org.apache.commons:commons-math3:3.5 -- org.apache.curator:curator-client:2.12.0 -- org.apache.curator:curator-framework:2.12.0 -- org.apache.curator:curator-recipes:2.12.0 -- org.apache.hadoop:hadoop-aliyun:3.1.0 -- org.apache.hadoop:hadoop-annotations:3.1.0 -- org.apache.hadoop:hadoop-auth:3.1.0 -- org.apache.hadoop:hadoop-common:3.1.0 -- org.apache.htrace:htrace-core4:4.1.0-incubating -- org.apache.httpcomponents:httpclient:4.5.3 -- org.apache.httpcomponents:httpcore:4.4.6 -- org.apache.kerby:kerb-admin:1.0.1 -- org.apache.kerby:kerb-client:1.0.1 -- org.apache.kerby:kerb-common:1.0.1 -- org.apache.kerby:kerb-core:1.0.1 -- org.apache.kerby:kerb-crypto:1.0.1 -- org.apache.kerby:kerb-identity:1.0.1 -- org.apache.kerby:kerb-server:1.0.1 -- org.apache.kerby:kerb-simplekdc:1.0.1 -- org.apache.kerby:kerb-util:1.0.1 -- org.apache.kerby:kerby-asn1:1.0.1 -- org.apache.kerby:kerby-config:1.0.1 -- org.apache.kerby:kerby-pkix:1.0.1 -- org.apache.kerby:kerby-util:1.0.1 -- org.apache.kerby:kerby-xdr:1.0.1 -- org.apache.kerby:token-provider:1.0.1 -- org.apache.zookeeper:zookeeper:3.4.10 -- org.codehaus.jackson:jackson-core-asl:1.9.2 -- org.codehaus.jackson:jackson-jaxrs:1.9.2 -- org.codehaus.jackson:jackson-mapper-asl:1.9.2 -- org.codehaus.jackson:jackson-xc:1.9.2 -- org.codehaus.jettison:jettison:1.1 -- org.eclipse.jetty:jetty-http:9.3.19.v20170502 -- org.eclipse.jetty:jetty-io:9.3.19.v20170502 -- org.eclipse.jetty:jetty-security:9.3.19.v20170502 -- org.eclipse.jetty:jetty-server:9.3.19.v20170502 -- org.eclipse.jetty:jetty-servlet:9.3.19.v20170502 -- org.eclipse.jetty:jetty-util:9.3.19.v20170502 -- org.eclipse.jetty:jetty-webapp:9.3.19.v20170502 -- org.eclipse.jetty:jetty-xml:9.3.19.v20170502 -- org.xerial.snappy:snappy-java:1.1.4 - -This project bundles the following dependencies under the BSD license. -See bundled license files for details. - -- com.google.protobuf:protobuf-java:2.5.0 -- com.jcraft:jsch:0.1.54 -- com.thoughtworks.paranamer:paranamer:2.7 -- org.codehaus.woodstox:stax2-api:3.1.4 -- org.ow2.asm:asm:5.0.4 - -This project bundles the following dependencies under the Common Development and Distribution License (CDDL) 1.0. -See bundled license files for details. - -- javax.activation:activation:1.1 -- javax.ws.rs:jsr311-api:1.1.1 -- javax.xml.stream:stax-api:1.0-2 -- stax:stax-api:1.0.1 - -This project bundles the following dependencies under the Common Development and Distribution License (CDDL) 1.1. -See bundled license files for details. - -- com.sun.jersey:jersey-core:1.19 -- com.sun.jersey:jersey-json:1.9 -- com.sun.jersey:jersey-server:1.19 -- com.sun.jersey:jersey-servlet:1.19 -- com.sun.xml.bind:jaxb-impl:2.2.3-1 -- javax.servlet.jsp:jsp-api:2.1 -- javax.servlet:javax.servlet-api:3.1.0 -- javax.xml.bind:jaxb-api:2.2.2 - -This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). -See bundled license files for details. - -- com.google.re2j:re2j:1.1 - -This project bundles the following dependencies under the JDOM license. -See bundled license files for details. - -- org.jdom:jdom:1.1 - -This project bundles org.tukaani:xz:1.5. -This Java implementation of XZ has been put into the public domain, thus you can do -whatever you want with it. All the files in the package have been written by Lasse Collin, -but some files are heavily based on public domain code written by Igor Pavlov. - - -Apache Commons Lang -Copyright 2001-2014 The Apache Software Foundation - -This product includes software from the Spring Framework, -under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -Apache Commons Collections -Copyright 2001-2015 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -Apache Commons Compress -Copyright 2002-2018 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). - -The files in the package org.apache.commons.compress.archivers.sevenz -were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), -which has been placed in the public domain: - -"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) - -flink-hadoop-fs -Copyright 2014-2019 The Apache Software Foundation - -flink-fs-hadoop-shaded -Copyright 2014-2019 The Apache Software Foundation - -- org.apache.hadoop:hadoop-annotations:3.1.0 -- org.apache.hadoop:hadoop-auth:3.1.0 -- org.apache.hadoop:hadoop-common:3.1.0 -- org.apache.htrace:htrace-core4:4.1.0-incubating -- org.apache.commons:commons-configuration2:2.1.1 -- org.apache.commons:commons-lang3:3.3.2 -- commons-lang:commons-lang:2.6 -- commons-collections:commons-collections:3.2.2 -- commons-io:commons-io:2.4 -- commons-logging:commons-logging:1.1.3 -- commons-beanutils:commons-beanutils:1.9.3 -- com.google.guava:guava:11.0.2 -- com.fasterxml.jackson.core:jackson-annotations:2.7.0 -- com.fasterxml.jackson.core:jackson-core:2.7.8 -- com.fasterxml.jackson.core:jackson-databind:2.7.8 -- com.fasterxml.woodstox:woodstox-core:5.0.3 - -This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). -See bundled license files for details. - -- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) - -This project bundles org.apache.hadoop:*:3.1.0 from which it inherits the following notices: - -The Apache Hadoop project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -For the org.apache.hadoop.util.bloom.* classes: - -/** - * - * Copyright (c) 2005, European Commission project OneLab under contract - * 034819 (http://www.one-lab.org) - * All rights reserved. - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the distribution. - * - Neither the name of the University Catholique de Louvain - UCL - * nor the names of its contributors may be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -For portions of the native implementation of slicing-by-8 CRC calculation -in src/main/native/src/org/apache/hadoop/util: - -Copyright (c) 2008,2009,2010 Massachusetts Institute of Technology. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -* Neither the name of the Massachusetts Institute of Technology nor - the names of its contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other portions are under the same license from Intel: -http://sourceforge.net/projects/slicing-by-8/ -/*++ - * - * Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved - * - * This software program is licensed subject to the BSD License, - * available at http://www.opensource.org/licenses/bsd-license.html - * - * Abstract: The main routine - * - --*/ - -For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, - -/* - LZ4 - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -For hadoop-common-project/hadoop-common/src/main/native/gtest ---------------------------------------------------------------------- -Copyright 2008, Google Inc. -All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -The binary distribution of this product bundles these dependencies under the -following license: -re2j 1.1 ---------------------------------------------------------------------- -(GO license) -This is a work derived from Russ Cox's RE2 in Go, whose license -http://golang.org/LICENSE is as follows: - -Copyright (c) 2009 The Go Authors. All rights reserved. - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - - * Neither the name of Google Inc. nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h ---------------------------------------------------------------------- -Copyright 2002 Niels Provos -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles binaries of leveldbjni -(https://github.com/fusesource/leveldbjni), which is available under the -following license: - -Copyright (c) 2011 FuseSource Corp. All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of FuseSource Corp. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: - -Copyright (c) 2012 The FreeBSD Foundation -All rights reserved. - -This software was developed by Pawel Jakub Dawidek under sponsorship from -the FreeBSD Foundation. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. - -============= - -The binary distribution of this product bundles binaries of leveldb -(http://code.google.com/p/leveldb/), which is available under the following -license: - -Copyright (c) 2011 The LevelDB Authors. All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -The binary distribution of this product bundles binaries of snappy -(http://code.google.com/p/snappy/), which is available under the following -license: - -Copyright 2011, Google Inc. -All rights reserved. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ --------------------------------------------------------------------------------- -Copyright (C) 2008-2016, SpryMedia Ltd. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2010 Aleksander Williams - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -The binary distribution of this product bundles these dependencies under the -following license: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css -bootstrap v3.3.6 -broccoli-asset-rev v2.4.2 -broccoli-funnel v1.0.1 -datatables v1.10.8 -em-helpers v0.5.13 -em-table v0.1.6 -ember v2.2.0 -ember-array-contains-helper v1.0.2 -ember-bootstrap v0.5.1 -ember-cli v1.13.13 -ember-cli-app-version v1.0.0 -ember-cli-babel v5.1.6 -ember-cli-content-security-policy v0.4.0 -ember-cli-dependency-checker v1.2.0 -ember-cli-htmlbars v1.0.2 -ember-cli-htmlbars-inline-precompile v0.3.1 -ember-cli-ic-ajax v0.2.1 -ember-cli-inject-live-reload v1.4.0 -ember-cli-jquery-ui v0.0.20 -ember-cli-qunit v1.2.1 -ember-cli-release v0.2.8 -ember-cli-shims v0.0.6 -ember-cli-sri v1.2.1 -ember-cli-test-loader v0.2.1 -ember-cli-uglify v1.2.0 -ember-d3 v0.1.0 -ember-data v2.1.0 -ember-disable-proxy-controllers v1.0.1 -ember-export-application-global v1.0.5 -ember-load-initializers v0.1.7 -ember-qunit v0.4.16 -ember-qunit-notifications v0.1.0 -ember-resolver v2.0.3 -ember-spin-spinner v0.2.3 -ember-truth-helpers v1.2.0 -jquery v2.1.4 -jquery-ui v1.11.4 -loader.js v3.3.0 -momentjs v2.10.6 -qunit v1.19.0 -select2 v4.0.0 -snippet-ss v1.11.0 -spin.js v2.3.2 -Azure Data Lake Store - Java client SDK 2.0.11 -JCodings 1.0.8 -Joni 2.1.2 -Mockito 1.8.5 -JUL to SLF4J bridge 1.7.25 -SLF4J API Module 1.7.25 -SLF4J LOG4J-12 Binding 1.7.25 --------------------------------------------------------------------------------- - -The MIT License (MIT) - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery -Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 -Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 --------------------------------------------------------------------------------- - -MIT License - -Copyright (c) 2003-2017 Optimatika - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -For: -oj! Algorithms - version 43.0 --------------------------------------------------------------------------------- - -Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/ - -This software consists of voluntary contributions made by many -individuals. For exact contribution history, see the revision history -available at https://github.com/jquery/jquery - -The following license applies to all parts of this software except as -documented below: - -==== - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -All files located in the node_modules and external directories are -externally maintained libraries used by this software which have their -own licenses; we recommend you read them, as their terms may differ from -the terms above. - -For: -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js --------------------------------------------------------------------------------- - -Copyright (c) 2014 Ivan Bozhanov - -For: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js --------------------------------------------------------------------------------- - -D3 is available under a 3-clause BSD license. For details, see: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE - -The binary distribution of this product bundles these dependencies under the -following license: -HSQLDB Database 2.3.4 --------------------------------------------------------------------------------- -(HSQL License) -"COPYRIGHTS AND LICENSES (based on BSD License) - -For work developed by the HSQL Development Group: - -Copyright (c) 2001-2016, The HSQL Development Group -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -Neither the name of the HSQL Development Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -For work originally developed by the Hypersonic SQL Group: - -Copyright (c) 1995-2000 by the Hypersonic SQL Group. -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Neither the name of the Hypersonic SQL Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -This software consists of voluntary contributions made by many individuals on behalf of the -Hypersonic SQL Group." - -The binary distribution of this product bundles these dependencies under the -following license: -Java Servlet API 3.1.0 -servlet-api 2.5 -jsp-api 2.1 -jsr311-api 1.1.1 -Glassfish Jasper 6.1.14 -Servlet Specification 2.5 API 6.1.14 --------------------------------------------------------------------------------- -(CDDL 1.0) -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 -1. Definitions.  - -1.1. Contributor means each individual or entity -that creates or contributes to the creation of -Modifications.  - -1.2. Contributor Version means the combination of the -Original Software, prior Modifications used by a Contributor (if any), and the -Modifications made by that particular Contributor.  - -1.3. Covered -Software means (a) the Original Software, or (b) Modifications, or (c) the -combination of files containing Original Software with files containing -Modifications, in each case including portions -thereof.  - -1.4. Executable means the Covered Software in any form other -than Source Code.  - -1.5. Initial Developer means the individual or entity -that first makes Original Software available under this -License.  - -1.6. Larger Work means a work which combines Covered Software or -portions thereof with code not governed by the terms of this -License.  - -1.7. License means this document.  - -1.8. Licensable means -having the right to grant, to the maximum extent possible, whether at the time -of the initial grant or subsequently acquired, any and all of the rights -conveyed herein.  - -1.9. Modifications means the Source Code and Executable -form of any of the following: -A. Any file that results from an addition to, -deletion from or modification of the contents of a file containing Original -Software or previous Modifications; -B. Any new file that contains any part of the Original Software -or previous Modification; or -C. Any new file that is contributed or otherwise made available -under the terms of this License.  - -1.10. Original Software means the Source Code and Executable form of -computer software code that is originally released under this License.  - -1.11. Patent Claims means any patent claim(s), now owned or -hereafter acquired, including without limitation, method, process, and apparatus -claims, in any patent Licensable by grantor.  - -1.12. Source Code means (a) the common form of computer software code in which -modifications are made and (b) associated documentation included in or -with such code.  - -1.13. You (or Your) means an individual or a legal entity exercising rights -under, and complying with all of the terms of, this License. For legal entities, -You includes any entity which controls, is controlled by, or is under common control -with You. For purposes of this definition, control means (a) the power, direct -or indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (b) ownership of more than fifty percent (50%) of the -outstanding shares or beneficial ownership of such entity.  - -2. License Grants. - -2.1. The Initial Developer Grant. Conditioned upon Your compliance -with Section 3.1 below and subject to third party intellectual property claims, -the Initial Developer hereby grants You a world-wide, royalty-free, -non-exclusive license:  - -(a) under intellectual property rights (other than -patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, -display, perform, sublicense and distribute the Original Software (or portions -thereof), with or without Modifications, and/or as part of a Larger Work; -and  - -(b) under Patent Claims infringed by the making, using or selling of -Original Software, to make, have made, use, practice, sell, and offer for sale, -and/or otherwise dispose of the Original Software (or portions -thereof); - -(c) The licenses granted in Sections 2.1(a) and (b) are -effective on the date Initial Developer first distributes or otherwise makes the -Original Software available to a third party under the terms of this -License; - -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or -(ii) the combination of the Original Software with other software or -devices.  - -2.2. Contributor Grant. Conditioned upon Your compliance with -Section 3.1 below and subject to third party intellectual property claims, each -Contributor hereby grants You a world-wide, royalty-free, non-exclusive -license:  - -(a) under intellectual property rights (other than patent or -trademark) Licensable by Contributor to use, reproduce, modify, display, -perform, sublicense and distribute the Modifications created by such Contributor -(or portions thereof), either on an unmodified basis, with other Modifications, -as Covered Software and/or as part of a Larger Work; and  - -(b) under Patent -Claims infringed by the making, using, or selling of Modifications made by that -Contributor either alone and/or in combination with its Contributor Version (or -portions of such combination), to make, use, sell, offer for sale, have made, -and/or otherwise dispose of: (1) Modifications made by that Contributor (or -portions thereof); and (2) the combination of Modifications made by that -Contributor with its Contributor Version (or portions of such -combination).  - -(c) The licenses granted in Sections 2.2(a) and 2.2(b) are -effective on the date Contributor first distributes or otherwise makes the -Modifications available to a third party. - -(d) Notwithstanding Section 2.2(b) -above, no patent license is granted: (1) for any code that Contributor has -deleted from the Contributor Version; (2) for infringements caused by: -(i) third party modifications of Contributor Version, or (ii) the combination -of Modifications made by that Contributor with other software (except as part of -the Contributor Version) or other devices; or (3) under Patent Claims infringed -by Covered Software in the absence of Modifications made by that -Contributor.  - -3. Distribution Obligations.  - -3.1. Availability of Source -Code. Any Covered Software that You distribute or otherwise make available in -Executable form must also be made available in Source Code form and that Source -Code form must be distributed only under the terms of this License. You must -include a copy of this License with every copy of the Source Code form of the -Covered Software You distribute or otherwise make available. You must inform -recipients of any such Covered Software in Executable form as to how they can -obtain such Covered Software in Source Code form in a reasonable manner on or -through a medium customarily used for software exchange.  - -3.2. -Modifications. The Modifications that You create or to which You contribute are -governed by the terms of this License. You represent that You believe Your -Modifications are Your original creation(s) and/or You have sufficient rights to -grant the rights conveyed by this License.  - -3.3. Required Notices. You must -include a notice in each of Your Modifications that identifies You as the -Contributor of the Modification. You may not remove or alter any copyright, -patent or trademark notices contained within the Covered Software, or any -notices of licensing or any descriptive text giving attribution to any -Contributor or the Initial Developer.  - -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source Code -form that alters or restricts the applicable version of this License or the -recipients rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. - -3.5. Distribution of Executable Versions. You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipients rights in the Source Code form from the rights set forth in this -License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer.  - -3.6. Larger Works. You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software.  - -4. Versions of the License.  - -4.1. -New Versions. Sun Microsystems, Inc. is the initial license steward and may -publish revised and/or new versions of this License from time to time. Each -version will be given a distinguishing version number. Except as provided in -Section 4.3, no one other than the license steward has the right to modify this -License.  - -4.2. Effect of New Versions. You may always continue to use, -distribute or otherwise make the Covered Software available under the terms of -the version of the License under which You originally received the Covered -Software. If the Initial Developer includes a notice in the Original Software -prohibiting it from being distributed or otherwise made available under any -subsequent version of the License, You must distribute and make the Covered -Software available under the terms of the version of the License under which You -originally received the Covered Software. Otherwise, You may also choose to use, -distribute or otherwise make the Covered Software available under the terms of -any subsequent version of the License published by the license -steward.  - -4.3. Modified Versions. When You are an Initial Developer and You -want to create a new license for Your Original Software, You may create and use -a modified version of this License if You: (a) rename the license and remove -any references to the name of the license steward (except to note that the -license differs from this License); and (b) otherwise make it clear that the -license contains terms which differ from this License.  - -5. DISCLAIMER OF WARRANTY. - -COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, -WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT -LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, -MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY -COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER -OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR -CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS -LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER -THIS DISCLAIMER.  - -6. TERMINATION.  - -6.1. This License and the rights -granted hereunder will terminate automatically if You fail to comply with terms -herein and fail to cure such breach within 30 days of becoming aware of the -breach. Provisions which, by their nature, must remain in effect beyond the -termination of this License shall survive.  - -6.2. If You assert a patent -infringement claim (excluding declaratory judgment actions) against Initial -Developer or a Contributor (the Initial Developer or Contributor against whom -You assert such claim is referred to as Participant) alleging that the -Participant Software (meaning the Contributor Version where the Participant is a -Contributor or the Original Software where the Participant is the Initial -Developer) directly or indirectly infringes any patent, then any and all rights -granted directly or indirectly to You by such Participant, the Initial Developer -(if the Initial Developer is not the Participant) and all Contributors under -Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from -Participant terminate prospectively and automatically at the expiration of such -60 day notice period, unless if within such 60 day period You withdraw Your -claim with respect to the Participant Software against such Participant either -unilaterally or pursuant to a written agreement with Participant.  - -6.3. In -the event of termination under Sections 6.1 or 6.2 above, all end user licenses -that have been validly granted by You or any distributor hereunder prior to -termination (excluding licenses granted to You by any distributor) shall survive -termination.  - -7. LIMITATION OF LIABILITY. -UNDER NO CIRCUMSTANCES AND UNDER -NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, -SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF -COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY -PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY -CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF -GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER -COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE -POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO -LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO -THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT -ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO -THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  - -8. U.S. GOVERNMENT END USERS. - -The Covered Software is a commercial item, as that term is defined in -48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as -that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer -software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. -1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through -227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software -with only those rights set forth herein. This U.S. Government Rights clause is -in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision -that addresses Government rights in computer software under this -License.  - -9. MISCELLANEOUS. -This License represents the complete agreement -concerning subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdictions conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software.  - -10. RESPONSIBILITY FOR CLAIMS. -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability.  - -The binary distribution of this product bundles these dependencies under the -following license: -jersey-client 1.19 -jersey-core 1.19 -jersey-grizzly2 1.19 -jersey-grizzly2-servlet 1.19 -jersey-json 1.19 -jersey-server 1.19 -jersey-servlet 1.19 -jersey-guice 1.19 -Jersey Test Framework - Grizzly 2 Module 1.19 -JAXB RI 2.2.3 -Java Architecture for XML Binding 2.2.11 -grizzly-framework 2.2.21 -grizzly-http 2.2.21 -grizzly-http-server 2.2.21 -grizzly-http-servlet 2.2.21 -grizzly-rcm 2.2.21 --------------------------------------------------------------------------------- -(CDDL 1.1) -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 - -1. Definitions. - -1.1. “Contributor” means each individual or entity that creates or -contributes to the creation of Modifications. -1.2. “Contributor Version” means the combination of the Original Software, -prior Modifications used by a Contributor (if any), and the Modifications made -by that particular Contributor. -1.3. “Covered Software” means (a) the Original Software, or (b) -Modifications, or (c) the combination of files containing Original Software with -files containing Modifications, in each case including portions thereof. -1.4. “Executable” means the Covered Software in any form other than Source -Code. -1.5. “Initial Developer” means the individual or entity that first makes -Original Software available under this License. -1.6. “Larger Work” means a work which combines Covered Software or portions -thereof with code not governed by the terms of this License. -1.7. “License” means this document. -1.8. “Licensable” means having the right to grant, to the maximum extent -possible, whether at the time of the initial grant or subsequently acquired, any -and all of the rights conveyed herein. -1.9. “Modifications” means the Source Code and Executable form of any of the -following: -A. Any file that results from an addition to, deletion from or modification of -the contents of a file containing Original Software or previous Modifications; -B. Any new file that contains any part of the Original Software or previous -Modification; or -C. Any new file that is contributed or otherwise made available under the terms -of this License. -1.10. “Original Software” means the Source Code and Executable form of -computer software code that is originally released under this License. -1.11. “Patent Claims” means any patent claim(s), now owned or hereafter -acquired, including without limitation, method, process, and apparatus claims, -in any patent Licensable by grantor. -1.12. “Source Code” means (a) the common form of computer software code in -which modifications are made and (b) associated documentation included in or -with such code. -1.13. “You” (or “Your”) means an individual or a legal entity exercising -rights under, and complying with all of the terms of, this License. For legal -entities, “You” includes any entity which controls, is controlled by, or is -under common control with You. For purposes of this definition, “control” -means (a) the power, direct or indirect, to cause the direction or management of -such entity, whether by contract or otherwise, or (b) ownership of more than -fifty percent (50%) of the outstanding shares or beneficial ownership of such -entity. - -2.1. The Initial Developer Grant. - -Conditioned upon Your compliance with Section 3.1 below and subject to -third party intellectual property claims, the Initial Developer hereby grants -You a world-wide, royalty-free, non-exclusive license: -(a) under intellectual -property rights (other than patent or trademark) Licensable by Initial -Developer, to use, reproduce, modify, display, perform, sublicense and -distribute the Original Software (or portions thereof), with or without -Modifications, and/or as part of a Larger Work; and -(b) under Patent Claims -infringed by the making, using or selling of Original Software, to make, have -made, use, practice, sell, and offer for sale, and/or otherwise dispose of the -Original Software (or portions thereof). -(c) The licenses granted in Sections -2.1(a) and (b) are effective on the date Initial Developer first distributes or -otherwise makes the Original Software available to a third party under the terms -of this License. -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or (ii) -the combination of the Original Software with other software or devices. - -2.2. Contributor Grant. - -Conditioned upon Your compliance with Section 3.1 below and -subject to third party intellectual property claims, each Contributor hereby -grants You a world-wide, royalty-free, non-exclusive license: -(a) under -intellectual property rights (other than patent or trademark) Licensable by -Contributor to use, reproduce, modify, display, perform, sublicense and -distribute the Modifications created by such Contributor (or portions thereof), -either on an unmodified basis, with other Modifications, as Covered Software -and/or as part of a Larger Work; and -(b) under Patent Claims infringed by the -making, using, or selling of Modifications made by that Contributor either alone -and/or in combination with its Contributor Version (or portions of such -combination), to make, use, sell, offer for sale, have made, and/or otherwise -dispose of: (1) Modifications made by that Contributor (or portions thereof); -and (2) the combination of Modifications made by that Contributor with its -Contributor Version (or portions of such combination). -(c) The licenses granted -in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first -distributes or otherwise makes the Modifications available to a third -party. -(d) Notwithstanding Section 2.2(b) above, no patent license is granted: -(1) for any code that Contributor has deleted from the Contributor Version; (2) -for infringements caused by: (i) third party modifications of Contributor -Version, or (ii) the combination of Modifications made by that Contributor with -other software (except as part of the Contributor Version) or other devices; or -(3) under Patent Claims infringed by Covered Software in the absence of -Modifications made by that Contributor. - -3. Distribution Obligations. - -3.1. Availability of Source Code. -Any Covered Software that You distribute or -otherwise make available in Executable form must also be made available in -Source Code form and that Source Code form must be distributed only under the -terms of this License. You must include a copy of this License with every copy -of the Source Code form of the Covered Software You distribute or otherwise make -available. You must inform recipients of any such Covered Software in Executable -form as to how they can obtain such Covered Software in Source Code form in a -reasonable manner on or through a medium customarily used for software -exchange. -3.2. Modifications. -The Modifications that You create or to which -You contribute are governed by the terms of this License. You represent that You -believe Your Modifications are Your original creation(s) and/or You have -sufficient rights to grant the rights conveyed by this License. -3.3. Required Notices. -You must include a notice in each of Your Modifications that -identifies You as the Contributor of the Modification. You may not remove or -alter any copyright, patent or trademark notices contained within the Covered -Software, or any notices of licensing or any descriptive text giving attribution -to any Contributor or the Initial Developer. -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source -Code form that alters or restricts the applicable version of this License or the -recipients' rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. -3.5. Distribution of Executable Versions. -You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipient's rights in the Source Code form from the rights set forth in -this License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer. -3.6. Larger Works. -You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software. - -4. Versions of the License. - -4.1. New Versions. -Oracle is the initial license steward and may publish revised and/or -new versions of this License from time to time. Each version will be given a -distinguishing version number. Except as provided in Section 4.3, no one other -than the license steward has the right to modify this License. -4.2. Effect of New Versions. -You may always continue to use, distribute or otherwise make the -Covered Software available under the terms of the version of the License under -which You originally received the Covered Software. If the Initial Developer -includes a notice in the Original Software prohibiting it from being distributed -or otherwise made available under any subsequent version of the License, You -must distribute and make the Covered Software available under the terms of the -version of the License under which You originally received the Covered Software. -Otherwise, You may also choose to use, distribute or otherwise make the Covered -Software available under the terms of any subsequent version of the License -published by the license steward. -4.3. Modified Versions. -When You are an -Initial Developer and You want to create a new license for Your Original -Software, You may create and use a modified version of this License if You: (a) -rename the license and remove any references to the name of the license steward -(except to note that the license differs from this License); and (b) otherwise -make it clear that the license contains terms which differ from this -License. - -COVERED SOFTWARE IS PROVIDED UNDER THIS -LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE -IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR -NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED -SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY -RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE -COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF -WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED -SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - -6.1. This License and the rights granted hereunder will -terminate automatically if You fail to comply with terms herein and fail to cure -such breach within 30 days of becoming aware of the breach. Provisions which, by -their nature, must remain in effect beyond the termination of this License shall -survive. -6.2. If You assert a patent infringement claim (excluding declaratory -judgment actions) against Initial Developer or a Contributor (the Initial -Developer or Contributor against whom You assert such claim is referred to as -“Participant”) alleging that the Participant Software (meaning the -Contributor Version where the Participant is a Contributor or the Original -Software where the Participant is the Initial Developer) directly or indirectly -infringes any patent, then any and all rights granted directly or indirectly to -You by such Participant, the Initial Developer (if the Initial Developer is not -the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this -License shall, upon 60 days notice from Participant terminate prospectively and -automatically at the expiration of such 60 day notice period, unless if within -such 60 day period You withdraw Your claim with respect to the Participant -Software against such Participant either unilaterally or pursuant to a written -agreement with Participant. -6.3. If You assert a patent infringement claim -against Participant alleging that the Participant Software directly or -indirectly infringes any patent where such claim is resolved (such as by license -or settlement) prior to the initiation of patent infringement litigation, then -the reasonable value of the licenses granted by such Participant under Sections -2.1 or 2.2 shall be taken into account in determining the amount or value of any -payment or license. -6.4. In the event of termination under Sections 6.1 or 6.2 -above, all end user licenses that have been validly granted by You or any -distributor hereunder prior to termination (excluding licenses granted to You by -any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. - -UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT -(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL -DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY -SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, -SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, -WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER -FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN -IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS -LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL -INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW -PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR -LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND -LIMITATION MAY NOT APPLY TO YOU. - -The Covered -Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 -(Oct. 1995), consisting of “commercial computer software” (as that term is -defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software -documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). -Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 -(June 1995), all U.S. Government End Users acquire Covered Software with only -those rights set forth herein. This U.S. Government Rights clause is in lieu of, -and supersedes, any other FAR, DFAR, or other clause or provision that addresses -Government rights in computer software under this License. - -9. MISCELLANEOUS. - -This License represents the complete agreement concerning -subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdiction's conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys' fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. - -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability. - -The binary distribution of this product bundles these dependencies under the -following license: -Protocol Buffer Java API 2.5.0 --------------------------------------------------------------------------------- -This license applies to all parts of Protocol Buffers except the following: - - - Atomicops support for generic gcc, located in - src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. - This file is copyrighted by Red Hat Inc. - - - Atomicops support for AIX/POWER, located in - src/google/protobuf/stubs/atomicops_internals_power.h. - This file is copyrighted by Bloomberg Finance LP. - -Copyright 2014, Google Inc. All rights reserved. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - -For: -XML Commons External Components XML APIs 1.3.04 --------------------------------------------------------------------------------- -By obtaining, using and/or copying this work, you (the licensee) agree that you -have read, understood, and will comply with the following terms and conditions. - -Permission to copy, modify, and distribute this software and its documentation, -with or without modification, for any purpose and without fee or royalty is -hereby granted, provided that you include the following on ALL copies of the -software and documentation or portions thereof, including modifications: -- The full text of this NOTICE in a location viewable to users of the -redistributed or derivative work. -- Any pre-existing intellectual property disclaimers, notices, or terms and -conditions. If none exist, the W3C Software Short Notice should be included -(hypertext is preferred, text is permitted) within the body of any redistributed -or derivative code. -- Notice of any changes or modifications to the files, including the date changes -were made. (We recommend you provide URIs to the location from which the code is -derived.) - -The binary distribution of this product bundles these dependencies under the -following license: -JUnit 4.11 -Eclipse JDT Core 3.1.1 --------------------------------------------------------------------------------- -(EPL v1.0) -Eclipse Public License - v 1.0 - -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC -LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM -CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: - -a) in the case of the initial Contributor, the initial code and documentation -distributed under this Agreement, and -b) in the case of each subsequent Contributor: -i) changes to the Program, and -ii) additions to the Program; -where such changes and/or additions to the Program originate from and are -distributed by that particular Contributor. A Contribution 'originates' from a -Contributor if it was added to the Program by such Contributor itself or anyone -acting on such Contributor's behalf. Contributions do not include additions to -the Program which: (i) are separate modules of software distributed in -conjunction with the Program under their own license agreement, and (ii) are not -derivative works of the Program. -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents" mean patent claims licensable by a Contributor which are -necessarily infringed by the use or sale of its Contribution alone or when -combined with the Program. - -"Program" means the Contributions distributed in accordance with this Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, -including all Contributors. - -2. GRANT OF RIGHTS - -a) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free copyright license to -reproduce, prepare derivative works of, publicly display, publicly perform, -distribute and sublicense the Contribution of such Contributor, if any, and such -derivative works, in source code and object code form. -b) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed -Patents to make, use, sell, offer to sell, import and otherwise transfer the -Contribution of such Contributor, if any, in source code and object code form. -This patent license shall apply to the combination of the Contribution and the -Program if, at the time the Contribution is added by the Contributor, such -addition of the Contribution causes such combination to be covered by the -Licensed Patents. The patent license shall not apply to any other combinations -which include the Contribution. No hardware per se is licensed hereunder. -c) Recipient understands that although each Contributor grants the licenses to -its Contributions set forth herein, no assurances are provided by any -Contributor that the Program does not infringe the patent or other intellectual -property rights of any other entity. Each Contributor disclaims any liability to -Recipient for claims brought by any other entity based on infringement of -intellectual property rights or otherwise. As a condition to exercising the -rights and licenses granted hereunder, each Recipient hereby assumes sole -responsibility to secure any other intellectual property rights needed, if any. -For example, if a third party patent license is required to allow Recipient to -distribute the Program, it is Recipient's responsibility to acquire that license -before distributing the Program. -d) Each Contributor represents that to its knowledge it has sufficient copyright -rights in its Contribution, if any, to grant the copyright license set forth in -this Agreement. -3. REQUIREMENTS - -A Contributor may choose to distribute the Program in object code form under its -own license agreement, provided that: - -a) it complies with the terms and conditions of this Agreement; and -b) its license agreement: -i) effectively disclaims on behalf of all Contributors all warranties and -conditions, express and implied, including warranties or conditions of title and -non-infringement, and implied warranties or conditions of merchantability and -fitness for a particular purpose; -ii) effectively excludes on behalf of all Contributors all liability for -damages, including direct, indirect, special, incidental and consequential -damages, such as lost profits; -iii) states that any provisions which differ from this Agreement are offered by -that Contributor alone and not by any other party; and -iv) states that source code for the Program is available from such Contributor, -and informs licensees how to obtain it in a reasonable manner on or through a -medium customarily used for software exchange. -When the Program is made available in source code form: - -a) it must be made available under this Agreement; and -b) a copy of this Agreement must be included with each copy of the Program. -Contributors may not remove or alter any copyright notices contained within the -Program. - -Each Contributor must identify itself as the originator of its Contribution, if -any, in a manner that reasonably allows subsequent Recipients to identify the -originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION - -Commercial distributors of software may accept certain responsibilities with -respect to end users, business partners and the like. While this license is -intended to facilitate the commercial use of the Program, the Contributor who -includes the Program in a commercial product offering should do so in a manner -which does not create potential liability for other Contributors. Therefore, if -a Contributor includes the Program in a commercial product offering, such -Contributor ("Commercial Contributor") hereby agrees to defend and indemnify -every other Contributor ("Indemnified Contributor") against any losses, damages -and costs (collectively "Losses") arising from claims, lawsuits and other legal -actions brought by a third party against the Indemnified Contributor to the -extent caused by the acts or omissions of such Commercial Contributor in -connection with its distribution of the Program in a commercial product -offering. The obligations in this section do not apply to any claims or Losses -relating to any actual or alleged intellectual property infringement. In order -to qualify, an Indemnified Contributor must: a) promptly notify the Commercial -Contributor in writing of such claim, and b) allow the Commercial Contributor to -control, and cooperate with the Commercial Contributor in, the defense and any -related settlement negotiations. The Indemnified Contributor may participate in -any such claim at its own expense. - -For example, a Contributor might include the Program in a commercial product -offering, Product X. That Contributor is then a Commercial Contributor. If that -Commercial Contributor then makes performance claims, or offers warranties -related to Product X, those performance claims and warranties are such -Commercial Contributor's responsibility alone. Under this section, the -Commercial Contributor would have to defend claims against the other -Contributors related to those performance claims and warranties, and if a court -requires any other Contributor to pay any damages as a result, the Commercial -Contributor must pay those damages. - -5. NO WARRANTY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR -IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, -NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each -Recipient is solely responsible for determining the appropriateness of using and -distributing the Program and assumes all risks associated with its exercise of -rights under this Agreement , including but not limited to the risks and costs -of program errors, compliance with applicable laws, damage to or loss of data, -programs or equipment, and unavailability or interruption of operations. - -6. DISCLAIMER OF LIABILITY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY -CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST -PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS -GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this Agreement, and without further action by the parties hereto, such -provision shall be reformed to the minimum extent necessary to make such -provision valid and enforceable. - -If Recipient institutes patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Program itself -(excluding combinations of the Program with other software or hardware) -infringes such Recipient's patent(s), then such Recipient's rights granted under -Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to -comply with any of the material terms or conditions of this Agreement and does -not cure such failure in a reasonable period of time after becoming aware of -such noncompliance. If all Recipient's rights under this Agreement terminate, -Recipient agrees to cease use and distribution of the Program as soon as -reasonably practicable. However, Recipient's obligations under this Agreement -and any licenses granted by Recipient relating to the Program shall continue and -survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in -order to avoid inconsistency the Agreement is copyrighted and may only be -modified in the following manner. The Agreement Steward reserves the right to -publish new versions (including revisions) of this Agreement from time to time. -No one other than the Agreement Steward has the right to modify this Agreement. -The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation -may assign the responsibility to serve as the Agreement Steward to a suitable -separate entity. Each new version of the Agreement will be given a -distinguishing version number. The Program (including Contributions) may always -be distributed subject to the version of the Agreement under which it was -received. In addition, after a new version of the Agreement is published, -Contributor may elect to distribute the Program (including its Contributions) -under the new version. Except as expressly stated in Sections 2(a) and 2(b) -above, Recipient receives no rights or licenses to the intellectual property of -any Contributor under this Agreement, whether expressly, by implication, -estoppel or otherwise. All rights in the Program not expressly granted under -this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the -intellectual property laws of the United States of America. No party to this -Agreement will bring a legal action under this Agreement more than one year -after the cause of action arose. Each party waives its rights to a jury trial in -any resulting litigation. - -The binary distribution of this product bundles these dependencies under the -following license: -JSch 0.1.51 -ParaNamer Core 2.3 -JLine 0.9.94 -leveldbjni-all 1.8 -Hamcrest Core 1.3 -ASM Core 5.0.4 -ASM Commons 5.0.2 -ASM Tree 5.0.2 --------------------------------------------------------------------------------- -(3-clause BSD) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles these dependencies under the -following license: -FindBugs-jsr305 3.0.0 -dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved. --------------------------------------------------------------------------------- -(2-clause BSD) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those -of the authors and should not be interpreted as representing official policies, -either expressed or implied, of the FreeBSD Project. - -The binary distribution of this product bundles these dependencies under the -following license: -"Java Concurrency in Practice" book annotations 1.0 --------------------------------------------------------------------------------- -(CCAL v2.5) -THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS -PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR -OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS -LICENSE OR COPYRIGHT LAW IS PROHIBITED. - -BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE -BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED -HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. - -1. Definitions - -"Collective Work" means a work, such as a periodical issue, anthology or -encyclopedia, in which the Work in its entirety in unmodified form, along with a -number of other contributions, constituting separate and independent works in -themselves, are assembled into a collective whole. A work that constitutes a -Collective Work will not be considered a Derivative Work (as defined below) for -the purposes of this License. -"Derivative Work" means a work based upon the Work or upon the Work and other -pre-existing works, such as a translation, musical arrangement, dramatization, -fictionalization, motion picture version, sound recording, art reproduction, -abridgment, condensation, or any other form in which the Work may be recast, -transformed, or adapted, except that a work that constitutes a Collective Work -will not be considered a Derivative Work for the purpose of this License. For -the avoidance of doubt, where the Work is a musical composition or sound -recording, the synchronization of the Work in timed-relation with a moving image -("synching") will be considered a Derivative Work for the purpose of this -License. -"Licensor" means the individual or entity that offers the Work under the terms -of this License. -"Original Author" means the individual or entity who created the Work. -"Work" means the copyrightable work of authorship offered under the terms of -this License. -"You" means an individual or entity exercising rights under this License who has -not previously violated the terms of this License with respect to the Work, or -who has received express permission from the Licensor to exercise rights under -this License despite a previous violation. -2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or -restrict any rights arising from fair use, first sale or other limitations on -the exclusive rights of the copyright owner under copyright law or other -applicable laws. - -3. License Grant. Subject to the terms and conditions of this License, Licensor -hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the -duration of the applicable copyright) license to exercise the rights in the Work -as stated below: - -to reproduce the Work, to incorporate the Work into one or more Collective -Works, and to reproduce the Work as incorporated in the Collective Works; -to create and reproduce Derivative Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission the Work including as -incorporated in Collective Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission Derivative Works. -For the avoidance of doubt, where the work is a musical composition: - -Performance Royalties Under Blanket Licenses. Licensor waives the exclusive -right to collect, whether individually or via a performance rights society (e.g. -ASCAP, BMI, SESAC), royalties for the public performance or public digital -performance (e.g. webcast) of the Work. -Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right -to collect, whether individually or via a music rights agency or designated -agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the -Work ("cover version") and distribute, subject to the compulsory license created -by 17 USC Section 115 of the US Copyright Act (or the equivalent in other -jurisdictions). -Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the -Work is a sound recording, Licensor waives the exclusive right to collect, -whether individually or via a performance-rights society (e.g. SoundExchange), -royalties for the public digital performance (e.g. webcast) of the Work, subject -to the compulsory license created by 17 USC Section 114 of the US Copyright Act -(or the equivalent in other jurisdictions). -The above rights may be exercised in all media and formats whether now known or -hereafter devised. The above rights include the right to make such modifications -as are technically necessary to exercise the rights in other media and formats. -All rights not expressly granted by Licensor are hereby reserved. - -4. Restrictions.The license granted in Section 3 above is expressly made subject -to and limited by the following restrictions: - -You may distribute, publicly display, publicly perform, or publicly digitally -perform the Work only under the terms of this License, and You must include a -copy of, or the Uniform Resource Identifier for, this License with every copy or -phonorecord of the Work You distribute, publicly display, publicly perform, or -publicly digitally perform. You may not offer or impose any terms on the Work -that alter or restrict the terms of this License or the recipients' exercise of -the rights granted hereunder. You may not sublicense the Work. You must keep -intact all notices that refer to this License and to the disclaimer of -warranties. You may not distribute, publicly display, publicly perform, or -publicly digitally perform the Work with any technological measures that control -access or use of the Work in a manner inconsistent with the terms of this -License Agreement. The above applies to the Work as incorporated in a Collective -Work, but this does not require the Collective Work apart from the Work itself -to be made subject to the terms of this License. If You create a Collective -Work, upon notice from any Licensor You must, to the extent practicable, remove -from the Collective Work any credit as required by clause 4(b), as requested. If -You create a Derivative Work, upon notice from any Licensor You must, to the -extent practicable, remove from the Derivative Work any credit as required by -clause 4(b), as requested. -If you distribute, publicly display, publicly perform, or publicly digitally -perform the Work or any Derivative Works or Collective Works, You must keep -intact all copyright notices for the Work and provide, reasonable to the medium -or means You are utilizing: (i) the name of the Original Author (or pseudonym, -if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor -designate another party or parties (e.g. a sponsor institute, publishing entity, -journal) for attribution in Licensor's copyright notice, terms of service or by -other reasonable means, the name of such party or parties; the title of the Work -if supplied; to the extent reasonably practicable, the Uniform Resource -Identifier, if any, that Licensor specifies to be associated with the Work, -unless such URI does not refer to the copyright notice or licensing information -for the Work; and in the case of a Derivative Work, a credit identifying the use -of the Work in the Derivative Work (e.g., "French translation of the Work by -Original Author," or "Screenplay based on original Work by Original Author"). -Such credit may be implemented in any reasonable manner; provided, however, that -in the case of a Derivative Work or Collective Work, at a minimum such credit -will appear where any other comparable authorship credit appears and in a manner -at least as prominent as such other comparable authorship credit. -5. Representations, Warranties and Disclaimer - -UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS -THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING -THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT -LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR -PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, -OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME -JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH -EXCLUSION MAY NOT APPLY TO YOU. - -6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN -NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, -INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS -LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - -7. Termination - -This License and the rights granted hereunder will terminate automatically upon -any breach by You of the terms of this License. Individuals or entities who have -received Derivative Works or Collective Works from You under this License, -however, will not have their licenses terminated provided such individuals or -entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, -and 8 will survive any termination of this License. -Subject to the above terms and conditions, the license granted here is perpetual -(for the duration of the applicable copyright in the Work). Notwithstanding the -above, Licensor reserves the right to release the Work under different license -terms or to stop distributing the Work at any time; provided, however that any -such election will not serve to withdraw this License (or any other license that -has been, or is required to be, granted under the terms of this License), and -this License will continue in full force and effect unless terminated as stated -above. -8. Miscellaneous - -Each time You distribute or publicly digitally perform the Work or a Collective -Work, the Licensor offers to the recipient a license to the Work on the same -terms and conditions as the license granted to You under this License. -Each time You distribute or publicly digitally perform a Derivative Work, -Licensor offers to the recipient a license to the original Work on the same -terms and conditions as the license granted to You under this License. -If any provision of this License is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this License, and without further action by the parties to this -agreement, such provision shall be reformed to the minimum extent necessary to -make such provision valid and enforceable. -No term or provision of this License shall be deemed waived and no breach -consented to unless such waiver or consent shall be in writing and signed by the -party to be charged with such waiver or consent. -This License constitutes the entire agreement between the parties with respect -to the Work licensed here. There are no understandings, agreements or -representations with respect to the Work not specified here. Licensor shall not -be bound by any additional provisions that may appear in any communication from -You. This License may not be modified without the mutual written agreement of -the Licensor and You. - -The binary distribution of this product bundles these dependencies under the -following license: -jamon-runtime 2.4.1 --------------------------------------------------------------------------------- -(MPL 2.0) - Mozilla Public License - Version 2.0 - -1.1. “Contributor” -means each individual or legal entity that creates, contributes to the creation -of, or owns Covered Software. - -1.2. “Contributor Version” -means the combination of the Contributions of others (if any) used by a -Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” -means Covered Software of a particular Contributor. - -1.4. “Covered Software” -means Source Code Form to which the initial Contributor has attached the notice -in Exhibit A, the Executable Form of such Source Code Form, and Modifications of -such Source Code Form, in each case including portions thereof. - -1.5. “Incompatible With Secondary Licenses” -means - -that the initial Contributor has attached the notice described in Exhibit B to -the Covered Software; or - -that the Covered Software was made available under the terms of version 1.1 or -earlier of the License, but not also under the terms of a Secondary License. - -1.6. “Executable Form” -means any form of the work other than Source Code Form. - -1.7. “Larger Work” -means a work that combines Covered Software with other material, in a separate -file or files, that is not Covered Software. - -1.8. “License” -means this document. - -1.9. “Licensable” -means having the right to grant, to the maximum extent possible, whether at the -time of the initial grant or subsequently, any and all of the rights conveyed by -this License. - -1.10. “Modifications” -means any of the following: - -any file in Source Code Form that results from an addition to, deletion from, or -modification of the contents of Covered Software; or - -any new file in Source Code Form that contains any Covered Software. - -1.11. “Patent Claims” of a Contributor -means any patent claim(s), including without limitation, method, process, and -apparatus claims, in any patent Licensable by such Contributor that would be -infringed, but for the grant of the License, by the making, using, selling, -offering for sale, having made, import, or transfer of either its Contributions -or its Contributor Version. - -1.12. “Secondary License” -means either the GNU General Public License, Version 2.0, the GNU Lesser General -Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, -or any later versions of those licenses. - -1.13. “Source Code Form” -means the form of the work preferred for making modifications. - -1.14. “You” (or “Your”) -means an individual or a legal entity exercising rights under this License. For -legal entities, “You” includes any entity that controls, is controlled by, -or is under common control with You. For purposes of this definition, -“control” means (a) the power, direct or indirect, to cause the direction or -management of such entity, whether by contract or otherwise, or (b) ownership of -more than fifty percent (50%) of the outstanding shares or beneficial ownership -of such entity. - -2. License Grants and Conditions - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive -license: - -under intellectual property rights (other than patent or trademark) Licensable -by such Contributor to use, reproduce, make available, modify, display, perform, -distribute, and otherwise exploit its Contributions, either on an unmodified -basis, with Modifications, or as part of a Larger Work; and - -under Patent Claims of such Contributor to make, use, sell, offer for sale, have -made, import, and otherwise transfer either its Contributions or its Contributor -Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution become -effective for each Contribution on the date the Contributor first distributes -such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under this -License. No additional rights or licenses will be implied from the distribution -or licensing of Covered Software under this License. Notwithstanding Section -2.1(b) above, no patent license is granted by a Contributor: - -for any code that a Contributor has removed from Covered Software; or - -for infringements caused by: (i) Your and any other third party’s -modifications of Covered Software, or (ii) the combination of its Contributions -with other software (except as part of its Contributor Version); or - -under Patent Claims infringed by Covered Software in the absence of its -Contributions. - -This License does not grant any rights in the trademarks, service marks, or -logos of any Contributor (except as may be necessary to comply with the notice -requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to distribute -the Covered Software under a subsequent version of this License (see Section -10.2) or under the terms of a Secondary License (if permitted under the terms of -Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its Contributions are -its original creation(s) or it has sufficient rights to grant the rights to its -Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under applicable -copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in -Section 2.1. - -3. Responsibilities - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under the -terms of this License. You must inform recipients that the Source Code Form of -the Covered Software is governed by the terms of this License, and how they can -obtain a copy of this License. You may not attempt to alter or restrict the -recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -such Covered Software must also be made available in Source Code Form, as -described in Section 3.1, and You must inform recipients of the Executable Form -how they can obtain a copy of such Source Code Form by reasonable means in a -timely manner, at a charge no more than the cost of distribution to the -recipient; and - -You may distribute such Executable Form under the terms of this License, or -sublicense it under different terms, provided that the license for the -Executable Form does not attempt to limit or alter the recipients’ rights in -the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, provided -that You also comply with the requirements of this License for the Covered -Software. If the Larger Work is a combination of Covered Software with a work -governed by one or more Secondary Licenses, and the Covered Software is not -Incompatible With Secondary Licenses, this License permits You to additionally -distribute such Covered Software under the terms of such Secondary License(s), -so that the recipient of the Larger Work may, at their option, further -distribute the Covered Software under the terms of either this License or such -Secondary License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices (including -copyright notices, patent notices, disclaimers of warranty, or limitations of -liability) contained within the Source Code Form of the Covered Software, except -that You may alter any license notices to the extent required to remedy known -factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, indemnity -or liability obligations to one or more recipients of Covered Software. However, -You may do so only on Your own behalf, and not on behalf of any Contributor. You -must make it absolutely clear that any such warranty, support, indemnity, or -liability obligation is offered by You alone, and You hereby agree to indemnify -every Contributor for any liability incurred by such Contributor as a result of -warranty, support, indemnity or liability terms You offer. You may include -additional disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - -If it is impossible for You to comply with any of the terms of this License with -respect to some or all of the Covered Software due to statute, judicial order, -or regulation then You must: (a) comply with the terms of this License to the -maximum extent possible; and (b) describe the limitations and the code they -affect. Such description must be placed in a text file included with all -distributions of the Covered Software under this License. Except to the extent -prohibited by statute or regulation, such description must be sufficiently -detailed for a recipient of ordinary skill to be able to understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You -fail to comply with any of its terms. However, if You become compliant, then the -rights granted under this License from a particular Contributor are reinstated -(a) provisionally, unless and until such Contributor explicitly and finally -terminates Your grants, and (b) on an ongoing basis, if such Contributor fails -to notify You of the non-compliance by some reasonable means prior to 60 days -after You have come back into compliance. Moreover, Your grants from a -particular Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the first -time You have received notice of non-compliance with this License from such -Contributor, and You become compliant prior to 30 days after Your receipt of the -notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, counter-claims, and -cross-claims) alleging that a Contributor Version directly or indirectly -infringes any patent, then the rights granted to You by any and all Contributors -for the Covered Software under Section 2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user -license agreements (excluding distributors and resellers) which have been -validly granted by You or Your distributors under this License prior to -termination shall survive termination. - -6. Disclaimer of Warranty - -Covered Software is provided under this License on an “as is” basis, without -warranty of any kind, either expressed, implied, or statutory, including, -without limitation, warranties that the Covered Software is free of defects, -merchantable, fit for a particular purpose or non-infringing. The entire risk as -to the quality and performance of the Covered Software is with You. Should any -Covered Software prove defective in any respect, You (not any Contributor) -assume the cost of any necessary servicing, repair, or correction. This -disclaimer of warranty constitutes an essential part of this License. No use of -any Covered Software is authorized under this License except under this -disclaimer. - -7. Limitation of Liability - -Under no circumstances and under no legal theory, whether tort (including -negligence), contract, or otherwise, shall any Contributor, or anyone who -distributes Covered Software as permitted above, be liable to You for any -direct, indirect, special, incidental, or consequential damages of any character -including, without limitation, damages for lost profits, loss of goodwill, work -stoppage, computer failure or malfunction, or any and all other commercial -damages or losses, even if such party shall have been informed of the -possibility of such damages. This limitation of liability shall not apply to -liability for death or personal injury resulting from such party’s negligence -to the extent applicable law prohibits such limitation. Some jurisdictions do -not allow the exclusion or limitation of incidental or consequential damages, so -this exclusion and limitation may not apply to You. - -8. Litigation - -Any litigation relating to this License may be brought only in the courts of a -jurisdiction where the defendant maintains its principal place of business and -such litigation shall be governed by laws of that jurisdiction, without -reference to its conflict-of-law provisions. Nothing in this Section shall -prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - -This License represents the complete agreement concerning the subject matter -hereof. If any provision of this License is held to be unenforceable, such -provision shall be reformed only to the extent necessary to make it enforceable. -Any law or regulation which provides that the language of a contract shall be -construed against the drafter shall not be used to construe this License against -a Contributor. - -10. Versions of the License - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section 10.3, -no one other than the license steward has the right to modify or publish new -versions of this License. Each version will be given a distinguishing version -number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version of the -License under which You originally received the Covered Software, or under the -terms of any subsequent version published by the license steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to create a -new license for such software, you may create and use a modified version of this -License if you rename the license and remove any references to the name of the -license steward (except to note that such modified license differs from this -License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - -If You choose to distribute Source Code Form that is Incompatible With Secondary -Licenses under the terms of this version of the License, the notice described in -Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - -This Source Code Form is subject to the terms of the Mozilla Public License, v. -2.0. If a copy of the MPL was not distributed with this file, You can obtain one -at https://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - -This Source Code Form is “Incompatible With Secondary Licenses”, as defined -by the Mozilla Public License, v. 2.0. - -The binary distribution of this product bundles these dependencies under the -following license: -JDOM 1.1 --------------------------------------------------------------------------------- -/*-- - - Copyright (C) 2000-2004 Jason Hunter & Brett McLaughlin. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions, and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions, and the disclaimer that follows - these conditions in the documentation and/or other materials - provided with the distribution. - - 3. The name "JDOM" must not be used to endorse or promote products - derived from this software without prior written permission. For - written permission, please contact . - - 4. Products derived from this software may not be called "JDOM", nor - may "JDOM" appear in their name, without prior written permission - from the JDOM Project Management . - - In addition, we request (but do not require) that you include in the - end-user documentation provided with the redistribution and/or in the - software itself an acknowledgement equivalent to the following: - "This product includes software developed by the - JDOM Project (http://www.jdom.org/)." - Alternatively, the acknowledgment may be graphical using the logos - available at http://www.jdom.org/images/logos. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED - WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - - This software consists of voluntary contributions made by many - individuals on behalf of the JDOM Project and was originally - created by Jason Hunter and - Brett McLaughlin . For more information - on the JDOM Project, please see . - - */ - -The binary distribution of this product bundles these dependencies under the -following license: -Hbase Server 1.2.4 --------------------------------------------------------------------------------- -This project bundles a derivative image for our Orca Logo. This image is -available under the Creative Commons By Attribution 3.0 License. - - Creative Commons Legal Code - - Attribution 3.0 Unported - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR - DAMAGES RESULTING FROM ITS USE. - - License - - THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE - COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY - COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS - AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. - - BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE - TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY - BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS - CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND - CONDITIONS. - - 1. Definitions - - a. "Adaptation" means a work based upon the Work, or upon the Work and - other pre-existing works, such as a translation, adaptation, - derivative work, arrangement of music or other alterations of a - literary or artistic work, or phonogram or performance and includes - cinematographic adaptations or any other form in which the Work may be - recast, transformed, or adapted including in any form recognizably - derived from the original, except that a work that constitutes a - Collection will not be considered an Adaptation for the purpose of - this License. For the avoidance of doubt, where the Work is a musical - work, performance or phonogram, the synchronization of the Work in - timed-relation with a moving image ("synching") will be considered an - Adaptation for the purpose of this License. - b. "Collection" means a collection of literary or artistic works, such as - encyclopedias and anthologies, or performances, phonograms or - broadcasts, or other works or subject matter other than works listed - in Section 1(f) below, which, by reason of the selection and - arrangement of their contents, constitute intellectual creations, in - which the Work is included in its entirety in unmodified form along - with one or more other contributions, each constituting separate and - independent works in themselves, which together are assembled into a - collective whole. A work that constitutes a Collection will not be - considered an Adaptation (as defined above) for the purposes of this - License. - c. "Distribute" means to make available to the public the original and - copies of the Work or Adaptation, as appropriate, through sale or - other transfer of ownership. - d. "Licensor" means the individual, individuals, entity or entities that - offer(s) the Work under the terms of this License. - e. "Original Author" means, in the case of a literary or artistic work, - the individual, individuals, entity or entities who created the Work - or if no individual or entity can be identified, the publisher; and in - addition (i) in the case of a performance the actors, singers, - musicians, dancers, and other persons who act, sing, deliver, declaim, - play in, interpret or otherwise perform literary or artistic works or - expressions of folklore; (ii) in the case of a phonogram the producer - being the person or legal entity who first fixes the sounds of a - performance or other sounds; and, (iii) in the case of broadcasts, the - organization that transmits the broadcast. - f. "Work" means the literary and/or artistic work offered under the terms - of this License including without limitation any production in the - literary, scientific and artistic domain, whatever may be the mode or - form of its expression including digital form, such as a book, - pamphlet and other writing; a lecture, address, sermon or other work - of the same nature; a dramatic or dramatico-musical work; a - choreographic work or entertainment in dumb show; a musical - composition with or without words; a cinematographic work to which are - assimilated works expressed by a process analogous to cinematography; - a work of drawing, painting, architecture, sculpture, engraving or - lithography; a photographic work to which are assimilated works - expressed by a process analogous to photography; a work of applied - art; an illustration, map, plan, sketch or three-dimensional work - relative to geography, topography, architecture or science; a - performance; a broadcast; a phonogram; a compilation of data to the - extent it is protected as a copyrightable work; or a work performed by - a variety or circus performer to the extent it is not otherwise - considered a literary or artistic work. - g. "You" means an individual or entity exercising rights under this - License who has not previously violated the terms of this License with - respect to the Work, or who has received express permission from the - Licensor to exercise rights under this License despite a previous - violation. - h. "Publicly Perform" means to perform public recitations of the Work and - to communicate to the public those public recitations, by any means or - process, including by wire or wireless means or public digital - performances; to make available to the public Works in such a way that - members of the public may access these Works from a place and at a - place individually chosen by them; to perform the Work to the public - by any means or process and the communication to the public of the - performances of the Work, including by public digital performance; to - broadcast and rebroadcast the Work by any means including signs, - sounds or images. - i. "Reproduce" means to make copies of the Work by any means including - without limitation by sound or visual recordings and the right of - fixation and reproducing fixations of the Work, including storage of a - protected performance or phonogram in digital form or other electronic - medium. - - 2. Fair Dealing Rights. Nothing in this License is intended to reduce, - limit, or restrict any uses free from copyright or rights arising from - limitations or exceptions that are provided for in connection with the - copyright protection under copyright law or other applicable laws. - - 3. License Grant. Subject to the terms and conditions of this License, - Licensor hereby grants You a worldwide, royalty-free, non-exclusive, - perpetual (for the duration of the applicable copyright) license to - exercise the rights in the Work as stated below: - - a. to Reproduce the Work, to incorporate the Work into one or more - Collections, and to Reproduce the Work as incorporated in the - Collections; - b. to create and Reproduce Adaptations provided that any such Adaptation, - including any translation in any medium, takes reasonable steps to - clearly label, demarcate or otherwise identify that changes were made - to the original Work. For example, a translation could be marked "The - original work was translated from English to Spanish," or a - modification could indicate "The original work has been modified."; - c. to Distribute and Publicly Perform the Work including as incorporated - in Collections; and, - d. to Distribute and Publicly Perform Adaptations. - e. For the avoidance of doubt: - - i. Non-waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme cannot be waived, the Licensor - reserves the exclusive right to collect such royalties for any - exercise by You of the rights granted under this License; - ii. Waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme can be waived, the Licensor waives the - exclusive right to collect such royalties for any exercise by You - of the rights granted under this License; and, - iii. Voluntary License Schemes. The Licensor waives the right to - collect royalties, whether individually or, in the event that the - Licensor is a member of a collecting society that administers - voluntary licensing schemes, via that society, from any exercise - by You of the rights granted under this License. - - The above rights may be exercised in all media and formats whether now - known or hereafter devised. The above rights include the right to make - such modifications as are technically necessary to exercise the rights in - other media and formats. Subject to Section 8(f), all rights not expressly - granted by Licensor are hereby reserved. - - 4. Restrictions. The license granted in Section 3 above is expressly made - subject to and limited by the following restrictions: - - a. You may Distribute or Publicly Perform the Work only under the terms - of this License. You must include a copy of, or the Uniform Resource - Identifier (URI) for, this License with every copy of the Work You - Distribute or Publicly Perform. You may not offer or impose any terms - on the Work that restrict the terms of this License or the ability of - the recipient of the Work to exercise the rights granted to that - recipient under the terms of the License. You may not sublicense the - Work. You must keep intact all notices that refer to this License and - to the disclaimer of warranties with every copy of the Work You - Distribute or Publicly Perform. When You Distribute or Publicly - Perform the Work, You may not impose any effective technological - measures on the Work that restrict the ability of a recipient of the - Work from You to exercise the rights granted to that recipient under - the terms of the License. This Section 4(a) applies to the Work as - incorporated in a Collection, but this does not require the Collection - apart from the Work itself to be made subject to the terms of this - License. If You create a Collection, upon notice from any Licensor You - must, to the extent practicable, remove from the Collection any credit - as required by Section 4(b), as requested. If You create an - Adaptation, upon notice from any Licensor You must, to the extent - practicable, remove from the Adaptation any credit as required by - Section 4(b), as requested. - b. If You Distribute, or Publicly Perform the Work or any Adaptations or - Collections, You must, unless a request has been made pursuant to - Section 4(a), keep intact all copyright notices for the Work and - provide, reasonable to the medium or means You are utilizing: (i) the - name of the Original Author (or pseudonym, if applicable) if supplied, - and/or if the Original Author and/or Licensor designate another party - or parties (e.g., a sponsor institute, publishing entity, journal) for - attribution ("Attribution Parties") in Licensor's copyright notice, - terms of service or by other reasonable means, the name of such party - or parties; (ii) the title of the Work if supplied; (iii) to the - extent reasonably practicable, the URI, if any, that Licensor - specifies to be associated with the Work, unless such URI does not - refer to the copyright notice or licensing information for the Work; - and (iv) , consistent with Section 3(b), in the case of an Adaptation, - a credit identifying the use of the Work in the Adaptation (e.g., - "French translation of the Work by Original Author," or "Screenplay - based on original Work by Original Author"). The credit required by - this Section 4 (b) may be implemented in any reasonable manner; - provided, however, that in the case of a Adaptation or Collection, at - a minimum such credit will appear, if a credit for all contributing - authors of the Adaptation or Collection appears, then as part of these - credits and in a manner at least as prominent as the credits for the - other contributing authors. For the avoidance of doubt, You may only - use the credit required by this Section for the purpose of attribution - in the manner set out above and, by exercising Your rights under this - License, You may not implicitly or explicitly assert or imply any - connection with, sponsorship or endorsement by the Original Author, - Licensor and/or Attribution Parties, as appropriate, of You or Your - use of the Work, without the separate, express prior written - permission of the Original Author, Licensor and/or Attribution - Parties. - c. Except as otherwise agreed in writing by the Licensor or as may be - otherwise permitted by applicable law, if You Reproduce, Distribute or - Publicly Perform the Work either by itself or as part of any - Adaptations or Collections, You must not distort, mutilate, modify or - take other derogatory action in relation to the Work which would be - prejudicial to the Original Author's honor or reputation. Licensor - agrees that in those jurisdictions (e.g. Japan), in which any exercise - of the right granted in Section 3(b) of this License (the right to - make Adaptations) would be deemed to be a distortion, mutilation, - modification or other derogatory action prejudicial to the Original - Author's honor and reputation, the Licensor will waive or not assert, - as appropriate, this Section, to the fullest extent permitted by the - applicable national law, to enable You to reasonably exercise Your - right under Section 3(b) of this License (right to make Adaptations) - but not otherwise. - - 5. Representations, Warranties and Disclaimer - - UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR - OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY - KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, - INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, - FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF - LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, - WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION - OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. - - 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE - LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR - ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES - ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS - BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - 7. Termination - - a. This License and the rights granted hereunder will terminate - automatically upon any breach by You of the terms of this License. - Individuals or entities who have received Adaptations or Collections - from You under this License, however, will not have their licenses - terminated provided such individuals or entities remain in full - compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will - survive any termination of this License. - b. Subject to the above terms and conditions, the license granted here is - perpetual (for the duration of the applicable copyright in the Work). - Notwithstanding the above, Licensor reserves the right to release the - Work under different license terms or to stop distributing the Work at - any time; provided, however that any such election will not serve to - withdraw this License (or any other license that has been, or is - required to be, granted under the terms of this License), and this - License will continue in full force and effect unless terminated as - stated above. - - 8. Miscellaneous - - a. Each time You Distribute or Publicly Perform the Work or a Collection, - the Licensor offers to the recipient a license to the Work on the same - terms and conditions as the license granted to You under this License. - b. Each time You Distribute or Publicly Perform an Adaptation, Licensor - offers to the recipient a license to the original Work on the same - terms and conditions as the license granted to You under this License. - c. If any provision of this License is invalid or unenforceable under - applicable law, it shall not affect the validity or enforceability of - the remainder of the terms of this License, and without further action - by the parties to this agreement, such provision shall be reformed to - the minimum extent necessary to make such provision valid and - enforceable. - d. No term or provision of this License shall be deemed waived and no - breach consented to unless such waiver or consent shall be in writing - and signed by the party to be charged with such waiver or consent. - e. This License constitutes the entire agreement between the parties with - respect to the Work licensed here. There are no understandings, - agreements or representations with respect to the Work not specified - here. Licensor shall not be bound by any additional provisions that - may appear in any communication from You. This License may not be - modified without the mutual written agreement of the Licensor and You. - f. The rights granted under, and the subject matter referenced, in this - License were drafted utilizing the terminology of the Berne Convention - for the Protection of Literary and Artistic Works (as amended on - September 28, 1979), the Rome Convention of 1961, the WIPO Copyright - Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 - and the Universal Copyright Convention (as revised on July 24, 1971). - These rights and subject matter take effect in the relevant - jurisdiction in which the License terms are sought to be enforced - according to the corresponding provisions of the implementation of - those treaty provisions in the applicable national law. If the - standard suite of rights granted under applicable copyright law - includes additional rights not granted under this License, such - additional rights are deemed to be included in the License; this - License is not intended to restrict the license of any rights under - applicable law. - - Creative Commons Notice - - Creative Commons is not a party to this License, and makes no warranty - whatsoever in connection with the Work. Creative Commons will not be - liable to You or any party on any legal theory for any damages - whatsoever, including without limitation any general, special, - incidental or consequential damages arising in connection to this - license. Notwithstanding the foregoing two (2) sentences, if Creative - Commons has expressly identified itself as the Licensor hereunder, it - shall have all rights and obligations of Licensor. - - Except for the limited purpose of indicating to the public that the - Work is licensed under the CCPL, Creative Commons does not authorize - the use by either party of the trademark "Creative Commons" or any - related trademark or logo of Creative Commons without the prior - written consent of Creative Commons. Any permitted use will be in - compliance with Creative Commons' then-current trademark usage - guidelines, as may be published on its website or otherwise made - available upon request from time to time. For the avoidance of doubt, - this trademark restriction does not form part of this License. - - Creative Commons may be contacted at https://creativecommons.org/. --------------------------------------------------------------------------------- - -For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs -/server/datanode/checker/AbstractFuture.java and -hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs -/server/datanode/checker/TimeoutFuture.java - -Copyright (C) 2007 The Guava Authors - -Licensed under the Apache License, Version 2.0 (the "License"); you may not -use this file except in compliance with the License. You may obtain a copy of -the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations under -the License. - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). - -The binary distribution of this product bundles binaries of -org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the -following notices: -* Copyright 2011 Dain Sundstrom -* Copyright 2011 FuseSource Corp. http://fusesource.com - -The binary distribution of this product bundles binaries of -AWS SDK for Java - Bundle 1.11.134, -AWS Java SDK for AWS KMS 1.11.134, -AWS Java SDK for Amazon S3 1.11.134, -AWS Java SDK for AWS STS 1.11.134, -JMES Path Query library 1.0, -which has the following notices: - * This software includes third party software subject to the following - copyrights: - XML parsing and utility functions from JetS3t - Copyright - 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - - Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility - functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. - -The binary distribution of this product bundles binaries of -Gson 2.2.4, -which has the following notices: - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'ArrayDeque', written by Josh -Bloch of Google, Inc: - - * LICENSE: - * license/LICENSE.deque.txt (Public Domain) - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified version of Roland Kuhn's ASL2 -AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. -It can be obtained at: - - * LICENSE: - * license/LICENSE.abstractnodequeue.txt (Public Domain) - * HOMEPAGE: - * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/libdivsufsort/ - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/protobuf/ - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/snappy/ - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * http://code.google.com/p/caliper/ - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - -This product contains a modified portion of 'JDOM 1.1', which can be obtained at: - - * LICENSE: - * https://github.com/hunterhacker/jdom/blob/jdom-1.1/core/LICENSE.txt - * HOMEPAGE: - * http://www.jdom.org/ - -The binary distribution of this product bundles binaries of -Commons Codec 1.4, -which has the following notices: - * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - =============================================================================== - The content of package org.apache.commons.codec.language.bm has been translated - from the original php source code available at http://stevemorse.org/phoneticinfo.htm - with permission from the original authors. - Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -The binary distribution of this product bundles binaries of -Commons Lang 2.6, -which has the following notices: - * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -The binary distribution of this product bundles binaries of -Apache Log4j 1.2.17, -which has the following notices: - * ResolverUtil.java - Copyright 2005-2006 Tim Fennell - Dumbster SMTP test server - Copyright 2004 Jason Paul Kitchen - TypeUtil.java - Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams - -The binary distribution of this product bundles binaries of -"Java Concurrency in Practice" book annotations 1.0, -which has the following notices: - * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative - Commons Attribution License (http://creativecommons.org/licenses/by/2.5) - Official home: http://www.jcip.net Any republication or derived work - distributed in source code form must include this copyright and license - notice. - -The binary distribution of this product bundles binaries of -Jetty :: Http Utility 9.3.19., -Jetty :: IO Utility 9.3.19., -Jetty :: Security 9.3.19., -Jetty :: Server Core 9.3.19., -Jetty :: Servlet Handling 9.3.19., -Jetty :: Utilities 9.3.19., -Jetty :: Utilities :: Ajax, -Jetty :: Webapp Application Support 9.3.19., -Jetty :: XML utilities 9.3.19., -which has the following notices: - * ============================================================== - Jetty Web Container - Copyright 1995-2016 Mort Bay Consulting Pty Ltd. - ============================================================== - - The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd - unless otherwise noted. - - Jetty is dual licensed under both - - * The Apache 2.0 License - http://www.apache.org/licenses/LICENSE-2.0.html - - and - - * The Eclipse Public 1.0 License - http://www.eclipse.org/legal/epl-v10.html - - Jetty may be distributed under either license. - - ------ - Eclipse - - The following artifacts are EPL. - * org.eclipse.jetty.orbit:org.eclipse.jdt.core - - The following artifacts are EPL and ASL2. - * org.eclipse.jetty.orbit:javax.security.auth.message - - The following artifacts are EPL and CDDL 1.0. - * org.eclipse.jetty.orbit:javax.mail.glassfish - - ------ - Oracle - - The following artifacts are CDDL + GPLv2 with classpath exception. - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - * javax.servlet:javax.servlet-api - * javax.annotation:javax.annotation-api - * javax.transaction:javax.transaction-api - * javax.websocket:javax.websocket-api - - ------ - Oracle OpenJDK - - If ALPN is used to negotiate HTTP/2 connections, then the following - artifacts may be included in the distribution or downloaded when ALPN - module is selected. - - * java.sun.security.ssl - - These artifacts replace/modify OpenJDK classes. The modififications - are hosted at github and both modified and original are under GPL v2 with - classpath exceptions. - http://openjdk.java.net/legal/gplv2+ce.html - - ------ - OW2 - - The following artifacts are licensed by the OW2 Foundation according to the - terms of http://asm.ow2.org/license.html - - org.ow2.asm:asm-commons - org.ow2.asm:asm - - ------ - Apache - - The following artifacts are ASL2 licensed. - - org.apache.taglibs:taglibs-standard-spec - org.apache.taglibs:taglibs-standard-impl - - ------ - MortBay - - The following artifacts are ASL2 licensed. Based on selected classes from - following Apache Tomcat jars, all ASL2 licensed. - - org.mortbay.jasper:apache-jsp - org.apache.tomcat:tomcat-jasper - org.apache.tomcat:tomcat-juli - org.apache.tomcat:tomcat-jsp-api - org.apache.tomcat:tomcat-el-api - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-api - org.apache.tomcat:tomcat-util-scan - org.apache.tomcat:tomcat-util - - org.mortbay.jasper:apache-el - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-el-api - - ------ - Mortbay - - The following artifacts are CDDL + GPLv2 with classpath exception. - - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - org.eclipse.jetty.toolchain:jetty-schemas - - ------ - Assorted - - The UnixCrypt.java code implements the one way cryptography used by - Unix systems for simple password protection. Copyright 1996 Aki Yoshida, - modified April 2001 by Iris Van den Broeke, Daniel Deville. - Permission to use, copy, modify and distribute UnixCrypt - for non-commercial or commercial purposes and without fee is - granted provided that the copyright notice appears in all copies./ - -The binary distribution of this product bundles binaries of -Snappy for Java 1.0.4.1, -which has the following notices: - * This product includes software developed by Google - Snappy: http://code.google.com/p/snappy/ (New BSD License) - - This product includes software developed by Apache - PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ - (Apache 2.0 license) - - This library containd statically linked libstdc++. This inclusion is allowed by - "GCC RUntime Library Exception" - http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html - - == Contributors == - * Tatu Saloranta - * Providing benchmark suite - * Alec Wysoker - * Performance and memory usage improvement - -The binary distribution of this product bundles binaries of -Xerces2 Java Parser 2.9.1, -which has the following notices: - * ========================================================================= - == NOTICE file corresponding to section 4(d) of the Apache License, == - == Version 2.0, in this case for the Apache Xerces Java distribution. == - ========================================================================= - - Apache Xerces Java - Copyright 1999-2007 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were originally based on the following: - - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. - - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. - - voluntary contributions made by Paul Eng on behalf of the - Apache Software Foundation that were originally developed at iClick, Inc., - software copyright (c) 1999. - -The binary distribution of this product bundles binaries of -Logback Classic Module 1.1.2, -Logback Core Module 1.1.2, -which has the following notices: - * Logback: the reliable, generic, fast and flexible logging framework. - Copyright (C) 1999-2012, QOS.ch. All rights reserved. - -The binary distribution of this product bundles binaries of -Apache HBase - Annotations 1.2.6, -Apache HBase - Client 1.2.6, -Apache HBase - Common 1.2.6, -Apache HBase - Hadoop Compatibility 1.2.6, -Apache HBase - Hadoop Two Compatibility 1.2.6, -Apache HBase - Prefix Tree 1.2.6, -Apache HBase - Procedure 1.2.6, -Apache HBase - Protocol 1.2.6, -Apache HBase - Server 1.2.6, -which has the following notices: - * Apache HBase - Copyright 2007-2015 The Apache Software Foundation - - -- - This product incorporates portions of the 'Hadoop' project - - Copyright 2007-2009 The Apache Software Foundation - - Licensed under the Apache License v2.0 - -- - Our Orca logo we got here: http://www.vectorfree.com/jumping-orca - It is licensed Creative Commons Attribution 3.0. - See https://creativecommons.org/licenses/by/3.0/us/ - We changed the logo by stripping the colored background, inverting - it and then rotating it some. - - Later we found that vectorfree.com image is not properly licensed. - The original is owned by vectorportal.com. The original was - relicensed so we could use it as Creative Commons Attribution 3.0. - The license is bundled with the download available here: - http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp - -- - This product includes portions of the Bootstrap project v3.0.0 - - Copyright 2013 Twitter, Inc. - - Licensed under the Apache License v2.0 - - This product uses the Glyphicons Halflings icon set. - - http://glyphicons.com/ - - Copyright Jan Kovařík - - Licensed under the Apache License v2.0 as a part of the Bootstrap project. - - -- - This product includes portions of the Guava project v14, specifically - 'hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java' - - Copyright (C) 2007 The Guava Authors - - Licensed under the Apache License, Version 2.0 - -The binary distribution of this product bundles binaries of -Phoenix Core 4.7.0, -which has the following notices: - Apache Phoenix - Copyright 2013-2016 The Apache Software Foundation - - This product includes software developed by The Apache Software - Foundation (http://www.apache.org/). - - This also includes: - - The phoenix-spark module has been adapted from the phoenix-spark library - distributed under the terms of the Apache 2 license. Original source copyright: - Copyright 2014 Simply Measured, Inc. - Copyright 2015 Interset Software Inc. - - The file bin/daemon.py is based on the file of the same name in python-daemon 2.0.5 - (https://pypi.python.org/pypi/python-daemon/). Original source copyright: - # Copyright © 2008–2015 Ben Finney - # Copyright © 2007–2008 Robert Niederreiter, Jens Klein - # Copyright © 2004–2005 Chad J. Schroeder - # Copyright © 2003 Clark Evans - # Copyright © 2002 Noah Spurrier - # Copyright © 2001 Jürgen Hermann - -The binary distribution of this product bundles binaries of -Plexus Cipher: encryption/decryption Component 1.4, -which has the following notices: - * The code in this component contains a class - Base64 taken from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/Base64.java - which is Apache license: http://www.apache.org/licenses/LICENSE-2.0 - - The PBE key processing routine PBECipher.createCipher() is adopted from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/OpenSSL.java - which is also Apache APL-2.0 license: http://www.apache.org/licenses/LICENSE-2.0 - -The binary distribution of this product bundles binaries of -software.amazon.ion:ion-java 1.0.1, -which has the following notices: - * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -The binary distribution of this product bundles binaries of -joda-time:joda-time:2.9.9 -which has the following notices: - * ============================================================================= - = NOTICE file corresponding to section 4d of the Apache License Version 2.0 = - ============================================================================= - This product includes software developed by - Joda.org (http://www.joda.org/). - -The binary distribution of this product bundles binaries of -Ehcache 3.3.1, -which has the following notices: - * Ehcache V3 Copyright 2014-2016 Terracotta, Inc. - -The binary distribution of this product bundles binaries of -snakeyaml (https://bitbucket.org/asomov/snakeyaml), -which has the following notices: - * Copyright (c) 2008, http://www.snakeyaml.org - -The binary distribution of this product bundles binaries of -swagger-annotations (https://github.com/swagger-api/swagger-core), -which has the following notices: - * Copyright 2016 SmartBear Software - -The binary distribution of this product bundles binaries of -metrics-core 3.2.4 -which has the following notices: - * Copyright 2010-2013 Coda Hale and Yammer, Inc. - - This product includes software developed by Coda Hale and Yammer, Inc. - - This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, - LongAdder), which was released with the following comments: - - Written by Doug Lea with assistance from members of JCP JSR-166 - Expert Group and released to the public domain, as explained at - http://creativecommons.org/publicdomain/zero/1.0/ - -Apache Commons IO -Copyright 2002-2012 The Apache Software Foundation - -Apache Commons Logging -Copyright 2003-2013 The Apache Software Foundation - -Apache Commons Lang -Copyright 2001-2011 The Apache Software Foundation - -Apache Commons BeanUtils -Copyright 2000-2016 The Apache Software Foundation - -Apache Commons Configuration -Copyright 2001-2017 The Apache Software Foundation - -htrace-core4 -Copyright 2016 The Apache Software Foundation - -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -Apache Commons CLI -Copyright 2001-2015 The Apache Software Foundation - -Apache Commons Math -Copyright 2001-2015 The Apache Software Foundation - -This product includes software developed for Orekit by -CS Systèmes d'Information (http://www.c-s.fr/) -Copyright 2010-2012 CS Systèmes d'Information - -Apache Commons Codec -Copyright 2002-2014 The Apache Software Foundation - -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -Apache Commons Net -Copyright 2001-2017 The Apache Software Foundation - -This product currently only contains code developed by authors -of specific components, as identified by the source code files; -if such notes are missing files have been created by -Tatu Saloranta. - -For additional credits (generally to people who reported problems) -see CREDITS file. - -Apache Avro -Copyright 2009-2017 The Apache Software Foundation - -Curator Framework -Copyright 2011-2017 The Apache Software Foundation - -Curator Client -Copyright 2011-2017 The Apache Software Foundation - -Curator Recipes -Copyright 2011-2017 The Apache Software Foundation - -Kerb Simple Kdc -Copyright 2014-2017 The Apache Software Foundation - -Kerby-kerb Client -Copyright 2014-2017 The Apache Software Foundation - -Kerby Config -Copyright 2014-2017 The Apache Software Foundation - -Kerby-kerb core -Copyright 2014-2017 The Apache Software Foundation - -Kerby PKIX Project -Copyright 2014-2017 The Apache Software Foundation - -Kerby ASN1 Project -Copyright 2014-2017 The Apache Software Foundation - -Kerby Util -Copyright 2014-2017 The Apache Software Foundation - -Kerby-kerb Common -Copyright 2014-2017 The Apache Software Foundation - -Kerby-kerb Crypto -Copyright 2014-2017 The Apache Software Foundation - -Kerby-kerb Util -Copyright 2014-2017 The Apache Software Foundation - -Token provider -Copyright 2014-2017 The Apache Software Foundation - -Kerby-kerb Admin -Copyright 2014-2017 The Apache Software Foundation - -Kerby-kerb Server -Copyright 2014-2017 The Apache Software Foundation - -Kerby-kerb Identity -Copyright 2014-2017 The Apache Software Foundation - -Kerby XDR Project -Copyright 2014-2017 The Apache Software Foundation - -Apache HttpClient -Copyright 1999-2017 The Apache Software Foundation - -Apache HttpCore -Copyright 2005-2017 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-python -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.fasterxml.jackson.core:jackson-annotations:2.9.9 -- com.fasterxml.jackson.core:jackson-core:2.9.9 -- com.fasterxml.jackson.core:jackson-databind:2.9.9 -- com.google.api.grpc:proto-google-common-protos:1.12.0 -- com.google.code.gson:gson:2.7 -- com.google.guava:guava:26.0-jre -- io.grpc:grpc-auth:1.21.0 -- io.grpc:grpc-core:1.21.0 -- io.grpc:grpc-context:1.21.0 -- io.grpc:grpc-netty:1.21.0 -- io.grpc:grpc-protobuf:1.21.0 -- io.grpc:grpc-stub:1.21.0 -- io.grpc:grpc-testing:1.21.0 -- io.netty:netty-buffer:4.1.34.Final -- io.netty:netty-codec:4.1.34.Final -- io.netty:netty-codec-http:4.1.34.Final -- io.netty:netty-codec-http2:4.1.34.Final -- io.netty:netty-codec-socks:4.1.34.Final -- io.netty:netty-common:4.1.34.Final -- io.netty:netty-handler:4.1.34.Final -- io.netty:netty-handler-proxy:4.1.34.Final -- io.netty:netty-resolver:4.1.34.Final -- io.netty:netty-transport:4.1.34.Final -- io.netty:netty-transport-native-epoll:4.1.34.Final -- io.netty:netty-transport-native-unix-common:4.1.34.Final -- io.netty:netty-tcnative-boringssl-static:2.0.22.Final -- io.opencensus:opencensus-api:0.21.0 -- io.opencensus:opencensus-contrib-grpc-metrics:0.21.0 -- joda-time:joda-time:2.5 -- org.apache.beam:beam-model-fn-execution:2.15.0 -- org.apache.beam:beam-model-job-management:2.15.0 -- org.apache.beam:beam-model-pipeline:2.15.0 -- org.apache.beam:beam-runners-core-construction-java:2.15.0 -- org.apache.beam:beam-runners-java-fn-execution:2.15.0 -- org.apache.beam:beam-sdks-java-core:2.15.0 -- org.apache.beam:beam-sdks-java-fn-execution:2.15.0 -- org.apache.beam:beam-vendor-sdks-java-extensions-protobuf:2.15.0 -- org.apache.beam:beam-vendor-guava-26_0-jre:0.1 -- org.apache.beam:beam-vendor-grpc-1_21_0:0.1 - -This project bundles the following dependencies under the BSD license. -See bundled license files for details - -- net.sf.py4j:py4j:0.10.8.1 -- com.google.protobuf:protobuf-java:3.7.1 -- com.google.protobuf:protobuf-java-util:3.7.1 -- com.google.auth:google-auth-library-credentials:0.13.0 - -This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) -See bundled license files for details. - -- net.razorvine:pyrolite:4.13 - -Apache Beam -Copyright 2016-2018 The Apache Software Foundation - -Based on source code originally developed by -Google (http://www.google.com/). - -This product includes software developed at -Google (http://www.google.com/). - -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -============================================================================= -= NOTICE file corresponding to section 4d of the Apache License Version 2.0 = -============================================================================= -This product includes software developed by -Joda.org (http://www.joda.org/). - -Apache Commons Compress -Copyright 2002-2018 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). - -The files in the package org.apache.commons.compress.archivers.sevenz -were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), -which has been placed in the public domain: - -"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-queryable-state-runtime -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-s3-fs-hadoop -Copyright 2014-2019 The Apache Software Foundation - -flink-s3-fs-base -Copyright 2014-2019 The Apache Software Foundation - -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- org.apache.hadoop:hadoop-aws:3.1.0 -- org.apache.httpcomponents:httpcore:4.4.6 -- org.apache.httpcomponents:httpclient:4.5.3 -- commons-codec:commons-codec:1.10 -- commons-logging:commons-logging:1.1.3 -- com.amazonaws:aws-java-sdk-core:1.11.271 -- com.amazonaws:aws-java-sdk-dynamodb:1.11.271 -- com.amazonaws:aws-java-sdk-kms:1.11.271 -- com.amazonaws:aws-java-sdk-s3:1.11.271 -- com.amazonaws:jmespath-java:1.11.271 -- software.amazon.ion:ion-java:1.0.2 -- com.fasterxml.jackson.core:jackson-annotations:2.6.0 -- com.fasterxml.jackson.core:jackson-core:2.6.7 -- com.fasterxml.jackson.core:jackson-databind:2.6.7.1 -- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7 -- joda-time:joda-time:2.5 - -flink-fs-hadoop-shaded -Copyright 2014-2019 The Apache Software Foundation - -- org.apache.hadoop:hadoop-annotations:3.1.0 -- org.apache.hadoop:hadoop-auth:3.1.0 -- org.apache.hadoop:hadoop-common:3.1.0 -- org.apache.htrace:htrace-core4:4.1.0-incubating -- org.apache.commons:commons-configuration2:2.1.1 -- org.apache.commons:commons-lang3:3.3.2 -- commons-lang:commons-lang:2.6 -- commons-collections:commons-collections:3.2.2 -- commons-io:commons-io:2.4 -- commons-logging:commons-logging:1.1.3 -- commons-beanutils:commons-beanutils:1.9.3 -- com.google.guava:guava:11.0.2 -- com.fasterxml.jackson.core:jackson-annotations:2.7.0 -- com.fasterxml.jackson.core:jackson-core:2.7.8 -- com.fasterxml.jackson.core:jackson-databind:2.7.8 -- com.fasterxml.woodstox:woodstox-core:5.0.3 - -This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). -See bundled license files for details. - -- com.google.re2j:re2j:1.1 - -This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). -See bundled license files for details. - -- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) - -This project bundles org.apache.hadoop:*:3.1.0 from which it inherits the following notices: - -The Apache Hadoop project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -For the org.apache.hadoop.util.bloom.* classes: - -/** - * - * Copyright (c) 2005, European Commission project OneLab under contract - * 034819 (http://www.one-lab.org) - * All rights reserved. - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the distribution. - * - Neither the name of the University Catholique de Louvain - UCL - * nor the names of its contributors may be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -For portions of the native implementation of slicing-by-8 CRC calculation -in src/main/native/src/org/apache/hadoop/util: - -Copyright (c) 2008,2009,2010 Massachusetts Institute of Technology. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -* Neither the name of the Massachusetts Institute of Technology nor - the names of its contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other portions are under the same license from Intel: -http://sourceforge.net/projects/slicing-by-8/ -/*++ - * - * Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved - * - * This software program is licensed subject to the BSD License, - * available at http://www.opensource.org/licenses/bsd-license.html - * - * Abstract: The main routine - * - --*/ - -For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, - -/* - LZ4 - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -For hadoop-common-project/hadoop-common/src/main/native/gtest ---------------------------------------------------------------------- -Copyright 2008, Google Inc. -All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -The binary distribution of this product bundles these dependencies under the -following license: -re2j 1.1 ---------------------------------------------------------------------- -(GO license) -This is a work derived from Russ Cox's RE2 in Go, whose license -http://golang.org/LICENSE is as follows: - -Copyright (c) 2009 The Go Authors. All rights reserved. - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - - * Neither the name of Google Inc. nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h ---------------------------------------------------------------------- -Copyright 2002 Niels Provos -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles binaries of leveldbjni -(https://github.com/fusesource/leveldbjni), which is available under the -following license: - -Copyright (c) 2011 FuseSource Corp. All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of FuseSource Corp. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: - -Copyright (c) 2012 The FreeBSD Foundation -All rights reserved. - -This software was developed by Pawel Jakub Dawidek under sponsorship from -the FreeBSD Foundation. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. - -============= - -The binary distribution of this product bundles binaries of leveldb -(http://code.google.com/p/leveldb/), which is available under the following -license: - -Copyright (c) 2011 The LevelDB Authors. All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -The binary distribution of this product bundles binaries of snappy -(http://code.google.com/p/snappy/), which is available under the following -license: - -Copyright 2011, Google Inc. -All rights reserved. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ --------------------------------------------------------------------------------- -Copyright (C) 2008-2016, SpryMedia Ltd. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2010 Aleksander Williams - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -The binary distribution of this product bundles these dependencies under the -following license: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css -bootstrap v3.3.6 -broccoli-asset-rev v2.4.2 -broccoli-funnel v1.0.1 -datatables v1.10.8 -em-helpers v0.5.13 -em-table v0.1.6 -ember v2.2.0 -ember-array-contains-helper v1.0.2 -ember-bootstrap v0.5.1 -ember-cli v1.13.13 -ember-cli-app-version v1.0.0 -ember-cli-babel v5.1.6 -ember-cli-content-security-policy v0.4.0 -ember-cli-dependency-checker v1.2.0 -ember-cli-htmlbars v1.0.2 -ember-cli-htmlbars-inline-precompile v0.3.1 -ember-cli-ic-ajax v0.2.1 -ember-cli-inject-live-reload v1.4.0 -ember-cli-jquery-ui v0.0.20 -ember-cli-qunit v1.2.1 -ember-cli-release v0.2.8 -ember-cli-shims v0.0.6 -ember-cli-sri v1.2.1 -ember-cli-test-loader v0.2.1 -ember-cli-uglify v1.2.0 -ember-d3 v0.1.0 -ember-data v2.1.0 -ember-disable-proxy-controllers v1.0.1 -ember-export-application-global v1.0.5 -ember-load-initializers v0.1.7 -ember-qunit v0.4.16 -ember-qunit-notifications v0.1.0 -ember-resolver v2.0.3 -ember-spin-spinner v0.2.3 -ember-truth-helpers v1.2.0 -jquery v2.1.4 -jquery-ui v1.11.4 -loader.js v3.3.0 -momentjs v2.10.6 -qunit v1.19.0 -select2 v4.0.0 -snippet-ss v1.11.0 -spin.js v2.3.2 -Azure Data Lake Store - Java client SDK 2.0.11 -JCodings 1.0.8 -Joni 2.1.2 -Mockito 1.8.5 -JUL to SLF4J bridge 1.7.25 -SLF4J API Module 1.7.25 -SLF4J LOG4J-12 Binding 1.7.25 --------------------------------------------------------------------------------- - -The MIT License (MIT) - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery -Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 -Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 --------------------------------------------------------------------------------- - -MIT License - -Copyright (c) 2003-2017 Optimatika - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -For: -oj! Algorithms - version 43.0 --------------------------------------------------------------------------------- - -Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/ - -This software consists of voluntary contributions made by many -individuals. For exact contribution history, see the revision history -available at https://github.com/jquery/jquery - -The following license applies to all parts of this software except as -documented below: - -==== - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -All files located in the node_modules and external directories are -externally maintained libraries used by this software which have their -own licenses; we recommend you read them, as their terms may differ from -the terms above. - -For: -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js --------------------------------------------------------------------------------- - -Copyright (c) 2014 Ivan Bozhanov - -For: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js --------------------------------------------------------------------------------- - -D3 is available under a 3-clause BSD license. For details, see: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE - -The binary distribution of this product bundles these dependencies under the -following license: -HSQLDB Database 2.3.4 --------------------------------------------------------------------------------- -(HSQL License) -"COPYRIGHTS AND LICENSES (based on BSD License) - -For work developed by the HSQL Development Group: - -Copyright (c) 2001-2016, The HSQL Development Group -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -Neither the name of the HSQL Development Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -For work originally developed by the Hypersonic SQL Group: - -Copyright (c) 1995-2000 by the Hypersonic SQL Group. -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Neither the name of the Hypersonic SQL Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -This software consists of voluntary contributions made by many individuals on behalf of the -Hypersonic SQL Group." - -The binary distribution of this product bundles these dependencies under the -following license: -Java Servlet API 3.1.0 -servlet-api 2.5 -jsp-api 2.1 -jsr311-api 1.1.1 -Glassfish Jasper 6.1.14 -Servlet Specification 2.5 API 6.1.14 --------------------------------------------------------------------------------- -(CDDL 1.0) -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 -1. Definitions.  - -1.1. Contributor means each individual or entity -that creates or contributes to the creation of -Modifications.  - -1.2. Contributor Version means the combination of the -Original Software, prior Modifications used by a Contributor (if any), and the -Modifications made by that particular Contributor.  - -1.3. Covered -Software means (a) the Original Software, or (b) Modifications, or (c) the -combination of files containing Original Software with files containing -Modifications, in each case including portions -thereof.  - -1.4. Executable means the Covered Software in any form other -than Source Code.  - -1.5. Initial Developer means the individual or entity -that first makes Original Software available under this -License.  - -1.6. Larger Work means a work which combines Covered Software or -portions thereof with code not governed by the terms of this -License.  - -1.7. License means this document.  - -1.8. Licensable means -having the right to grant, to the maximum extent possible, whether at the time -of the initial grant or subsequently acquired, any and all of the rights -conveyed herein.  - -1.9. Modifications means the Source Code and Executable -form of any of the following: -A. Any file that results from an addition to, -deletion from or modification of the contents of a file containing Original -Software or previous Modifications; -B. Any new file that contains any part of the Original Software -or previous Modification; or -C. Any new file that is contributed or otherwise made available -under the terms of this License.  - -1.10. Original Software means the Source Code and Executable form of -computer software code that is originally released under this License.  - -1.11. Patent Claims means any patent claim(s), now owned or -hereafter acquired, including without limitation, method, process, and apparatus -claims, in any patent Licensable by grantor.  - -1.12. Source Code means (a) the common form of computer software code in which -modifications are made and (b) associated documentation included in or -with such code.  - -1.13. You (or Your) means an individual or a legal entity exercising rights -under, and complying with all of the terms of, this License. For legal entities, -You includes any entity which controls, is controlled by, or is under common control -with You. For purposes of this definition, control means (a) the power, direct -or indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (b) ownership of more than fifty percent (50%) of the -outstanding shares or beneficial ownership of such entity.  - -2. License Grants. - -2.1. The Initial Developer Grant. Conditioned upon Your compliance -with Section 3.1 below and subject to third party intellectual property claims, -the Initial Developer hereby grants You a world-wide, royalty-free, -non-exclusive license:  - -(a) under intellectual property rights (other than -patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, -display, perform, sublicense and distribute the Original Software (or portions -thereof), with or without Modifications, and/or as part of a Larger Work; -and  - -(b) under Patent Claims infringed by the making, using or selling of -Original Software, to make, have made, use, practice, sell, and offer for sale, -and/or otherwise dispose of the Original Software (or portions -thereof); - -(c) The licenses granted in Sections 2.1(a) and (b) are -effective on the date Initial Developer first distributes or otherwise makes the -Original Software available to a third party under the terms of this -License; - -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or -(ii) the combination of the Original Software with other software or -devices.  - -2.2. Contributor Grant. Conditioned upon Your compliance with -Section 3.1 below and subject to third party intellectual property claims, each -Contributor hereby grants You a world-wide, royalty-free, non-exclusive -license:  - -(a) under intellectual property rights (other than patent or -trademark) Licensable by Contributor to use, reproduce, modify, display, -perform, sublicense and distribute the Modifications created by such Contributor -(or portions thereof), either on an unmodified basis, with other Modifications, -as Covered Software and/or as part of a Larger Work; and  - -(b) under Patent -Claims infringed by the making, using, or selling of Modifications made by that -Contributor either alone and/or in combination with its Contributor Version (or -portions of such combination), to make, use, sell, offer for sale, have made, -and/or otherwise dispose of: (1) Modifications made by that Contributor (or -portions thereof); and (2) the combination of Modifications made by that -Contributor with its Contributor Version (or portions of such -combination).  - -(c) The licenses granted in Sections 2.2(a) and 2.2(b) are -effective on the date Contributor first distributes or otherwise makes the -Modifications available to a third party. - -(d) Notwithstanding Section 2.2(b) -above, no patent license is granted: (1) for any code that Contributor has -deleted from the Contributor Version; (2) for infringements caused by: -(i) third party modifications of Contributor Version, or (ii) the combination -of Modifications made by that Contributor with other software (except as part of -the Contributor Version) or other devices; or (3) under Patent Claims infringed -by Covered Software in the absence of Modifications made by that -Contributor.  - -3. Distribution Obligations.  - -3.1. Availability of Source -Code. Any Covered Software that You distribute or otherwise make available in -Executable form must also be made available in Source Code form and that Source -Code form must be distributed only under the terms of this License. You must -include a copy of this License with every copy of the Source Code form of the -Covered Software You distribute or otherwise make available. You must inform -recipients of any such Covered Software in Executable form as to how they can -obtain such Covered Software in Source Code form in a reasonable manner on or -through a medium customarily used for software exchange.  - -3.2. -Modifications. The Modifications that You create or to which You contribute are -governed by the terms of this License. You represent that You believe Your -Modifications are Your original creation(s) and/or You have sufficient rights to -grant the rights conveyed by this License.  - -3.3. Required Notices. You must -include a notice in each of Your Modifications that identifies You as the -Contributor of the Modification. You may not remove or alter any copyright, -patent or trademark notices contained within the Covered Software, or any -notices of licensing or any descriptive text giving attribution to any -Contributor or the Initial Developer.  - -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source Code -form that alters or restricts the applicable version of this License or the -recipients rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. - -3.5. Distribution of Executable Versions. You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipients rights in the Source Code form from the rights set forth in this -License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer.  - -3.6. Larger Works. You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software.  - -4. Versions of the License.  - -4.1. -New Versions. Sun Microsystems, Inc. is the initial license steward and may -publish revised and/or new versions of this License from time to time. Each -version will be given a distinguishing version number. Except as provided in -Section 4.3, no one other than the license steward has the right to modify this -License.  - -4.2. Effect of New Versions. You may always continue to use, -distribute or otherwise make the Covered Software available under the terms of -the version of the License under which You originally received the Covered -Software. If the Initial Developer includes a notice in the Original Software -prohibiting it from being distributed or otherwise made available under any -subsequent version of the License, You must distribute and make the Covered -Software available under the terms of the version of the License under which You -originally received the Covered Software. Otherwise, You may also choose to use, -distribute or otherwise make the Covered Software available under the terms of -any subsequent version of the License published by the license -steward.  - -4.3. Modified Versions. When You are an Initial Developer and You -want to create a new license for Your Original Software, You may create and use -a modified version of this License if You: (a) rename the license and remove -any references to the name of the license steward (except to note that the -license differs from this License); and (b) otherwise make it clear that the -license contains terms which differ from this License.  - -5. DISCLAIMER OF WARRANTY. - -COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, -WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT -LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, -MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY -COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER -OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR -CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS -LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER -THIS DISCLAIMER.  - -6. TERMINATION.  - -6.1. This License and the rights -granted hereunder will terminate automatically if You fail to comply with terms -herein and fail to cure such breach within 30 days of becoming aware of the -breach. Provisions which, by their nature, must remain in effect beyond the -termination of this License shall survive.  - -6.2. If You assert a patent -infringement claim (excluding declaratory judgment actions) against Initial -Developer or a Contributor (the Initial Developer or Contributor against whom -You assert such claim is referred to as Participant) alleging that the -Participant Software (meaning the Contributor Version where the Participant is a -Contributor or the Original Software where the Participant is the Initial -Developer) directly or indirectly infringes any patent, then any and all rights -granted directly or indirectly to You by such Participant, the Initial Developer -(if the Initial Developer is not the Participant) and all Contributors under -Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from -Participant terminate prospectively and automatically at the expiration of such -60 day notice period, unless if within such 60 day period You withdraw Your -claim with respect to the Participant Software against such Participant either -unilaterally or pursuant to a written agreement with Participant.  - -6.3. In -the event of termination under Sections 6.1 or 6.2 above, all end user licenses -that have been validly granted by You or any distributor hereunder prior to -termination (excluding licenses granted to You by any distributor) shall survive -termination.  - -7. LIMITATION OF LIABILITY. -UNDER NO CIRCUMSTANCES AND UNDER -NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, -SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF -COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY -PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY -CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF -GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER -COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE -POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO -LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO -THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT -ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO -THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  - -8. U.S. GOVERNMENT END USERS. - -The Covered Software is a commercial item, as that term is defined in -48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as -that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer -software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. -1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through -227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software -with only those rights set forth herein. This U.S. Government Rights clause is -in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision -that addresses Government rights in computer software under this -License.  - -9. MISCELLANEOUS. -This License represents the complete agreement -concerning subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdictions conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software.  - -10. RESPONSIBILITY FOR CLAIMS. -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability.  - -The binary distribution of this product bundles these dependencies under the -following license: -jersey-client 1.19 -jersey-core 1.19 -jersey-grizzly2 1.19 -jersey-grizzly2-servlet 1.19 -jersey-json 1.19 -jersey-server 1.19 -jersey-servlet 1.19 -jersey-guice 1.19 -Jersey Test Framework - Grizzly 2 Module 1.19 -JAXB RI 2.2.3 -Java Architecture for XML Binding 2.2.11 -grizzly-framework 2.2.21 -grizzly-http 2.2.21 -grizzly-http-server 2.2.21 -grizzly-http-servlet 2.2.21 -grizzly-rcm 2.2.21 --------------------------------------------------------------------------------- -(CDDL 1.1) -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 - -1. Definitions. - -1.1. “Contributor” means each individual or entity that creates or -contributes to the creation of Modifications. -1.2. “Contributor Version” means the combination of the Original Software, -prior Modifications used by a Contributor (if any), and the Modifications made -by that particular Contributor. -1.3. “Covered Software” means (a) the Original Software, or (b) -Modifications, or (c) the combination of files containing Original Software with -files containing Modifications, in each case including portions thereof. -1.4. “Executable” means the Covered Software in any form other than Source -Code. -1.5. “Initial Developer” means the individual or entity that first makes -Original Software available under this License. -1.6. “Larger Work” means a work which combines Covered Software or portions -thereof with code not governed by the terms of this License. -1.7. “License” means this document. -1.8. “Licensable” means having the right to grant, to the maximum extent -possible, whether at the time of the initial grant or subsequently acquired, any -and all of the rights conveyed herein. -1.9. “Modifications” means the Source Code and Executable form of any of the -following: -A. Any file that results from an addition to, deletion from or modification of -the contents of a file containing Original Software or previous Modifications; -B. Any new file that contains any part of the Original Software or previous -Modification; or -C. Any new file that is contributed or otherwise made available under the terms -of this License. -1.10. “Original Software” means the Source Code and Executable form of -computer software code that is originally released under this License. -1.11. “Patent Claims” means any patent claim(s), now owned or hereafter -acquired, including without limitation, method, process, and apparatus claims, -in any patent Licensable by grantor. -1.12. “Source Code” means (a) the common form of computer software code in -which modifications are made and (b) associated documentation included in or -with such code. -1.13. “You” (or “Your”) means an individual or a legal entity exercising -rights under, and complying with all of the terms of, this License. For legal -entities, “You” includes any entity which controls, is controlled by, or is -under common control with You. For purposes of this definition, “control” -means (a) the power, direct or indirect, to cause the direction or management of -such entity, whether by contract or otherwise, or (b) ownership of more than -fifty percent (50%) of the outstanding shares or beneficial ownership of such -entity. - -2.1. The Initial Developer Grant. - -Conditioned upon Your compliance with Section 3.1 below and subject to -third party intellectual property claims, the Initial Developer hereby grants -You a world-wide, royalty-free, non-exclusive license: -(a) under intellectual -property rights (other than patent or trademark) Licensable by Initial -Developer, to use, reproduce, modify, display, perform, sublicense and -distribute the Original Software (or portions thereof), with or without -Modifications, and/or as part of a Larger Work; and -(b) under Patent Claims -infringed by the making, using or selling of Original Software, to make, have -made, use, practice, sell, and offer for sale, and/or otherwise dispose of the -Original Software (or portions thereof). -(c) The licenses granted in Sections -2.1(a) and (b) are effective on the date Initial Developer first distributes or -otherwise makes the Original Software available to a third party under the terms -of this License. -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or (ii) -the combination of the Original Software with other software or devices. - -2.2. Contributor Grant. - -Conditioned upon Your compliance with Section 3.1 below and -subject to third party intellectual property claims, each Contributor hereby -grants You a world-wide, royalty-free, non-exclusive license: -(a) under -intellectual property rights (other than patent or trademark) Licensable by -Contributor to use, reproduce, modify, display, perform, sublicense and -distribute the Modifications created by such Contributor (or portions thereof), -either on an unmodified basis, with other Modifications, as Covered Software -and/or as part of a Larger Work; and -(b) under Patent Claims infringed by the -making, using, or selling of Modifications made by that Contributor either alone -and/or in combination with its Contributor Version (or portions of such -combination), to make, use, sell, offer for sale, have made, and/or otherwise -dispose of: (1) Modifications made by that Contributor (or portions thereof); -and (2) the combination of Modifications made by that Contributor with its -Contributor Version (or portions of such combination). -(c) The licenses granted -in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first -distributes or otherwise makes the Modifications available to a third -party. -(d) Notwithstanding Section 2.2(b) above, no patent license is granted: -(1) for any code that Contributor has deleted from the Contributor Version; (2) -for infringements caused by: (i) third party modifications of Contributor -Version, or (ii) the combination of Modifications made by that Contributor with -other software (except as part of the Contributor Version) or other devices; or -(3) under Patent Claims infringed by Covered Software in the absence of -Modifications made by that Contributor. - -3. Distribution Obligations. - -3.1. Availability of Source Code. -Any Covered Software that You distribute or -otherwise make available in Executable form must also be made available in -Source Code form and that Source Code form must be distributed only under the -terms of this License. You must include a copy of this License with every copy -of the Source Code form of the Covered Software You distribute or otherwise make -available. You must inform recipients of any such Covered Software in Executable -form as to how they can obtain such Covered Software in Source Code form in a -reasonable manner on or through a medium customarily used for software -exchange. -3.2. Modifications. -The Modifications that You create or to which -You contribute are governed by the terms of this License. You represent that You -believe Your Modifications are Your original creation(s) and/or You have -sufficient rights to grant the rights conveyed by this License. -3.3. Required Notices. -You must include a notice in each of Your Modifications that -identifies You as the Contributor of the Modification. You may not remove or -alter any copyright, patent or trademark notices contained within the Covered -Software, or any notices of licensing or any descriptive text giving attribution -to any Contributor or the Initial Developer. -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source -Code form that alters or restricts the applicable version of this License or the -recipients' rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. -3.5. Distribution of Executable Versions. -You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipient's rights in the Source Code form from the rights set forth in -this License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer. -3.6. Larger Works. -You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software. - -4. Versions of the License. - -4.1. New Versions. -Oracle is the initial license steward and may publish revised and/or -new versions of this License from time to time. Each version will be given a -distinguishing version number. Except as provided in Section 4.3, no one other -than the license steward has the right to modify this License. -4.2. Effect of New Versions. -You may always continue to use, distribute or otherwise make the -Covered Software available under the terms of the version of the License under -which You originally received the Covered Software. If the Initial Developer -includes a notice in the Original Software prohibiting it from being distributed -or otherwise made available under any subsequent version of the License, You -must distribute and make the Covered Software available under the terms of the -version of the License under which You originally received the Covered Software. -Otherwise, You may also choose to use, distribute or otherwise make the Covered -Software available under the terms of any subsequent version of the License -published by the license steward. -4.3. Modified Versions. -When You are an -Initial Developer and You want to create a new license for Your Original -Software, You may create and use a modified version of this License if You: (a) -rename the license and remove any references to the name of the license steward -(except to note that the license differs from this License); and (b) otherwise -make it clear that the license contains terms which differ from this -License. - -COVERED SOFTWARE IS PROVIDED UNDER THIS -LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE -IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR -NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED -SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY -RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE -COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF -WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED -SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - -6.1. This License and the rights granted hereunder will -terminate automatically if You fail to comply with terms herein and fail to cure -such breach within 30 days of becoming aware of the breach. Provisions which, by -their nature, must remain in effect beyond the termination of this License shall -survive. -6.2. If You assert a patent infringement claim (excluding declaratory -judgment actions) against Initial Developer or a Contributor (the Initial -Developer or Contributor against whom You assert such claim is referred to as -“Participant”) alleging that the Participant Software (meaning the -Contributor Version where the Participant is a Contributor or the Original -Software where the Participant is the Initial Developer) directly or indirectly -infringes any patent, then any and all rights granted directly or indirectly to -You by such Participant, the Initial Developer (if the Initial Developer is not -the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this -License shall, upon 60 days notice from Participant terminate prospectively and -automatically at the expiration of such 60 day notice period, unless if within -such 60 day period You withdraw Your claim with respect to the Participant -Software against such Participant either unilaterally or pursuant to a written -agreement with Participant. -6.3. If You assert a patent infringement claim -against Participant alleging that the Participant Software directly or -indirectly infringes any patent where such claim is resolved (such as by license -or settlement) prior to the initiation of patent infringement litigation, then -the reasonable value of the licenses granted by such Participant under Sections -2.1 or 2.2 shall be taken into account in determining the amount or value of any -payment or license. -6.4. In the event of termination under Sections 6.1 or 6.2 -above, all end user licenses that have been validly granted by You or any -distributor hereunder prior to termination (excluding licenses granted to You by -any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. - -UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT -(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL -DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY -SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, -SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, -WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER -FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN -IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS -LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL -INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW -PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR -LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND -LIMITATION MAY NOT APPLY TO YOU. - -The Covered -Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 -(Oct. 1995), consisting of “commercial computer software” (as that term is -defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software -documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). -Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 -(June 1995), all U.S. Government End Users acquire Covered Software with only -those rights set forth herein. This U.S. Government Rights clause is in lieu of, -and supersedes, any other FAR, DFAR, or other clause or provision that addresses -Government rights in computer software under this License. - -9. MISCELLANEOUS. - -This License represents the complete agreement concerning -subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdiction's conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys' fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. - -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability. - -The binary distribution of this product bundles these dependencies under the -following license: -Protocol Buffer Java API 2.5.0 --------------------------------------------------------------------------------- -This license applies to all parts of Protocol Buffers except the following: - - - Atomicops support for generic gcc, located in - src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. - This file is copyrighted by Red Hat Inc. - - - Atomicops support for AIX/POWER, located in - src/google/protobuf/stubs/atomicops_internals_power.h. - This file is copyrighted by Bloomberg Finance LP. - -Copyright 2014, Google Inc. All rights reserved. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - -For: -XML Commons External Components XML APIs 1.3.04 --------------------------------------------------------------------------------- -By obtaining, using and/or copying this work, you (the licensee) agree that you -have read, understood, and will comply with the following terms and conditions. - -Permission to copy, modify, and distribute this software and its documentation, -with or without modification, for any purpose and without fee or royalty is -hereby granted, provided that you include the following on ALL copies of the -software and documentation or portions thereof, including modifications: -- The full text of this NOTICE in a location viewable to users of the -redistributed or derivative work. -- Any pre-existing intellectual property disclaimers, notices, or terms and -conditions. If none exist, the W3C Software Short Notice should be included -(hypertext is preferred, text is permitted) within the body of any redistributed -or derivative code. -- Notice of any changes or modifications to the files, including the date changes -were made. (We recommend you provide URIs to the location from which the code is -derived.) - -The binary distribution of this product bundles these dependencies under the -following license: -JUnit 4.11 -Eclipse JDT Core 3.1.1 --------------------------------------------------------------------------------- -(EPL v1.0) -Eclipse Public License - v 1.0 - -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC -LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM -CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: - -a) in the case of the initial Contributor, the initial code and documentation -distributed under this Agreement, and -b) in the case of each subsequent Contributor: -i) changes to the Program, and -ii) additions to the Program; -where such changes and/or additions to the Program originate from and are -distributed by that particular Contributor. A Contribution 'originates' from a -Contributor if it was added to the Program by such Contributor itself or anyone -acting on such Contributor's behalf. Contributions do not include additions to -the Program which: (i) are separate modules of software distributed in -conjunction with the Program under their own license agreement, and (ii) are not -derivative works of the Program. -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents" mean patent claims licensable by a Contributor which are -necessarily infringed by the use or sale of its Contribution alone or when -combined with the Program. - -"Program" means the Contributions distributed in accordance with this Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, -including all Contributors. - -2. GRANT OF RIGHTS - -a) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free copyright license to -reproduce, prepare derivative works of, publicly display, publicly perform, -distribute and sublicense the Contribution of such Contributor, if any, and such -derivative works, in source code and object code form. -b) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed -Patents to make, use, sell, offer to sell, import and otherwise transfer the -Contribution of such Contributor, if any, in source code and object code form. -This patent license shall apply to the combination of the Contribution and the -Program if, at the time the Contribution is added by the Contributor, such -addition of the Contribution causes such combination to be covered by the -Licensed Patents. The patent license shall not apply to any other combinations -which include the Contribution. No hardware per se is licensed hereunder. -c) Recipient understands that although each Contributor grants the licenses to -its Contributions set forth herein, no assurances are provided by any -Contributor that the Program does not infringe the patent or other intellectual -property rights of any other entity. Each Contributor disclaims any liability to -Recipient for claims brought by any other entity based on infringement of -intellectual property rights or otherwise. As a condition to exercising the -rights and licenses granted hereunder, each Recipient hereby assumes sole -responsibility to secure any other intellectual property rights needed, if any. -For example, if a third party patent license is required to allow Recipient to -distribute the Program, it is Recipient's responsibility to acquire that license -before distributing the Program. -d) Each Contributor represents that to its knowledge it has sufficient copyright -rights in its Contribution, if any, to grant the copyright license set forth in -this Agreement. -3. REQUIREMENTS - -A Contributor may choose to distribute the Program in object code form under its -own license agreement, provided that: - -a) it complies with the terms and conditions of this Agreement; and -b) its license agreement: -i) effectively disclaims on behalf of all Contributors all warranties and -conditions, express and implied, including warranties or conditions of title and -non-infringement, and implied warranties or conditions of merchantability and -fitness for a particular purpose; -ii) effectively excludes on behalf of all Contributors all liability for -damages, including direct, indirect, special, incidental and consequential -damages, such as lost profits; -iii) states that any provisions which differ from this Agreement are offered by -that Contributor alone and not by any other party; and -iv) states that source code for the Program is available from such Contributor, -and informs licensees how to obtain it in a reasonable manner on or through a -medium customarily used for software exchange. -When the Program is made available in source code form: - -a) it must be made available under this Agreement; and -b) a copy of this Agreement must be included with each copy of the Program. -Contributors may not remove or alter any copyright notices contained within the -Program. - -Each Contributor must identify itself as the originator of its Contribution, if -any, in a manner that reasonably allows subsequent Recipients to identify the -originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION - -Commercial distributors of software may accept certain responsibilities with -respect to end users, business partners and the like. While this license is -intended to facilitate the commercial use of the Program, the Contributor who -includes the Program in a commercial product offering should do so in a manner -which does not create potential liability for other Contributors. Therefore, if -a Contributor includes the Program in a commercial product offering, such -Contributor ("Commercial Contributor") hereby agrees to defend and indemnify -every other Contributor ("Indemnified Contributor") against any losses, damages -and costs (collectively "Losses") arising from claims, lawsuits and other legal -actions brought by a third party against the Indemnified Contributor to the -extent caused by the acts or omissions of such Commercial Contributor in -connection with its distribution of the Program in a commercial product -offering. The obligations in this section do not apply to any claims or Losses -relating to any actual or alleged intellectual property infringement. In order -to qualify, an Indemnified Contributor must: a) promptly notify the Commercial -Contributor in writing of such claim, and b) allow the Commercial Contributor to -control, and cooperate with the Commercial Contributor in, the defense and any -related settlement negotiations. The Indemnified Contributor may participate in -any such claim at its own expense. - -For example, a Contributor might include the Program in a commercial product -offering, Product X. That Contributor is then a Commercial Contributor. If that -Commercial Contributor then makes performance claims, or offers warranties -related to Product X, those performance claims and warranties are such -Commercial Contributor's responsibility alone. Under this section, the -Commercial Contributor would have to defend claims against the other -Contributors related to those performance claims and warranties, and if a court -requires any other Contributor to pay any damages as a result, the Commercial -Contributor must pay those damages. - -5. NO WARRANTY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR -IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, -NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each -Recipient is solely responsible for determining the appropriateness of using and -distributing the Program and assumes all risks associated with its exercise of -rights under this Agreement , including but not limited to the risks and costs -of program errors, compliance with applicable laws, damage to or loss of data, -programs or equipment, and unavailability or interruption of operations. - -6. DISCLAIMER OF LIABILITY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY -CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST -PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS -GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this Agreement, and without further action by the parties hereto, such -provision shall be reformed to the minimum extent necessary to make such -provision valid and enforceable. - -If Recipient institutes patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Program itself -(excluding combinations of the Program with other software or hardware) -infringes such Recipient's patent(s), then such Recipient's rights granted under -Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to -comply with any of the material terms or conditions of this Agreement and does -not cure such failure in a reasonable period of time after becoming aware of -such noncompliance. If all Recipient's rights under this Agreement terminate, -Recipient agrees to cease use and distribution of the Program as soon as -reasonably practicable. However, Recipient's obligations under this Agreement -and any licenses granted by Recipient relating to the Program shall continue and -survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in -order to avoid inconsistency the Agreement is copyrighted and may only be -modified in the following manner. The Agreement Steward reserves the right to -publish new versions (including revisions) of this Agreement from time to time. -No one other than the Agreement Steward has the right to modify this Agreement. -The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation -may assign the responsibility to serve as the Agreement Steward to a suitable -separate entity. Each new version of the Agreement will be given a -distinguishing version number. The Program (including Contributions) may always -be distributed subject to the version of the Agreement under which it was -received. In addition, after a new version of the Agreement is published, -Contributor may elect to distribute the Program (including its Contributions) -under the new version. Except as expressly stated in Sections 2(a) and 2(b) -above, Recipient receives no rights or licenses to the intellectual property of -any Contributor under this Agreement, whether expressly, by implication, -estoppel or otherwise. All rights in the Program not expressly granted under -this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the -intellectual property laws of the United States of America. No party to this -Agreement will bring a legal action under this Agreement more than one year -after the cause of action arose. Each party waives its rights to a jury trial in -any resulting litigation. - -The binary distribution of this product bundles these dependencies under the -following license: -JSch 0.1.51 -ParaNamer Core 2.3 -JLine 0.9.94 -leveldbjni-all 1.8 -Hamcrest Core 1.3 -ASM Core 5.0.4 -ASM Commons 5.0.2 -ASM Tree 5.0.2 --------------------------------------------------------------------------------- -(3-clause BSD) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles these dependencies under the -following license: -FindBugs-jsr305 3.0.0 -dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved. --------------------------------------------------------------------------------- -(2-clause BSD) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those -of the authors and should not be interpreted as representing official policies, -either expressed or implied, of the FreeBSD Project. - -The binary distribution of this product bundles these dependencies under the -following license: -"Java Concurrency in Practice" book annotations 1.0 --------------------------------------------------------------------------------- -(CCAL v2.5) -THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS -PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR -OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS -LICENSE OR COPYRIGHT LAW IS PROHIBITED. - -BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE -BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED -HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. - -1. Definitions - -"Collective Work" means a work, such as a periodical issue, anthology or -encyclopedia, in which the Work in its entirety in unmodified form, along with a -number of other contributions, constituting separate and independent works in -themselves, are assembled into a collective whole. A work that constitutes a -Collective Work will not be considered a Derivative Work (as defined below) for -the purposes of this License. -"Derivative Work" means a work based upon the Work or upon the Work and other -pre-existing works, such as a translation, musical arrangement, dramatization, -fictionalization, motion picture version, sound recording, art reproduction, -abridgment, condensation, or any other form in which the Work may be recast, -transformed, or adapted, except that a work that constitutes a Collective Work -will not be considered a Derivative Work for the purpose of this License. For -the avoidance of doubt, where the Work is a musical composition or sound -recording, the synchronization of the Work in timed-relation with a moving image -("synching") will be considered a Derivative Work for the purpose of this -License. -"Licensor" means the individual or entity that offers the Work under the terms -of this License. -"Original Author" means the individual or entity who created the Work. -"Work" means the copyrightable work of authorship offered under the terms of -this License. -"You" means an individual or entity exercising rights under this License who has -not previously violated the terms of this License with respect to the Work, or -who has received express permission from the Licensor to exercise rights under -this License despite a previous violation. -2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or -restrict any rights arising from fair use, first sale or other limitations on -the exclusive rights of the copyright owner under copyright law or other -applicable laws. - -3. License Grant. Subject to the terms and conditions of this License, Licensor -hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the -duration of the applicable copyright) license to exercise the rights in the Work -as stated below: - -to reproduce the Work, to incorporate the Work into one or more Collective -Works, and to reproduce the Work as incorporated in the Collective Works; -to create and reproduce Derivative Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission the Work including as -incorporated in Collective Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission Derivative Works. -For the avoidance of doubt, where the work is a musical composition: - -Performance Royalties Under Blanket Licenses. Licensor waives the exclusive -right to collect, whether individually or via a performance rights society (e.g. -ASCAP, BMI, SESAC), royalties for the public performance or public digital -performance (e.g. webcast) of the Work. -Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right -to collect, whether individually or via a music rights agency or designated -agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the -Work ("cover version") and distribute, subject to the compulsory license created -by 17 USC Section 115 of the US Copyright Act (or the equivalent in other -jurisdictions). -Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the -Work is a sound recording, Licensor waives the exclusive right to collect, -whether individually or via a performance-rights society (e.g. SoundExchange), -royalties for the public digital performance (e.g. webcast) of the Work, subject -to the compulsory license created by 17 USC Section 114 of the US Copyright Act -(or the equivalent in other jurisdictions). -The above rights may be exercised in all media and formats whether now known or -hereafter devised. The above rights include the right to make such modifications -as are technically necessary to exercise the rights in other media and formats. -All rights not expressly granted by Licensor are hereby reserved. - -4. Restrictions.The license granted in Section 3 above is expressly made subject -to and limited by the following restrictions: - -You may distribute, publicly display, publicly perform, or publicly digitally -perform the Work only under the terms of this License, and You must include a -copy of, or the Uniform Resource Identifier for, this License with every copy or -phonorecord of the Work You distribute, publicly display, publicly perform, or -publicly digitally perform. You may not offer or impose any terms on the Work -that alter or restrict the terms of this License or the recipients' exercise of -the rights granted hereunder. You may not sublicense the Work. You must keep -intact all notices that refer to this License and to the disclaimer of -warranties. You may not distribute, publicly display, publicly perform, or -publicly digitally perform the Work with any technological measures that control -access or use of the Work in a manner inconsistent with the terms of this -License Agreement. The above applies to the Work as incorporated in a Collective -Work, but this does not require the Collective Work apart from the Work itself -to be made subject to the terms of this License. If You create a Collective -Work, upon notice from any Licensor You must, to the extent practicable, remove -from the Collective Work any credit as required by clause 4(b), as requested. If -You create a Derivative Work, upon notice from any Licensor You must, to the -extent practicable, remove from the Derivative Work any credit as required by -clause 4(b), as requested. -If you distribute, publicly display, publicly perform, or publicly digitally -perform the Work or any Derivative Works or Collective Works, You must keep -intact all copyright notices for the Work and provide, reasonable to the medium -or means You are utilizing: (i) the name of the Original Author (or pseudonym, -if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor -designate another party or parties (e.g. a sponsor institute, publishing entity, -journal) for attribution in Licensor's copyright notice, terms of service or by -other reasonable means, the name of such party or parties; the title of the Work -if supplied; to the extent reasonably practicable, the Uniform Resource -Identifier, if any, that Licensor specifies to be associated with the Work, -unless such URI does not refer to the copyright notice or licensing information -for the Work; and in the case of a Derivative Work, a credit identifying the use -of the Work in the Derivative Work (e.g., "French translation of the Work by -Original Author," or "Screenplay based on original Work by Original Author"). -Such credit may be implemented in any reasonable manner; provided, however, that -in the case of a Derivative Work or Collective Work, at a minimum such credit -will appear where any other comparable authorship credit appears and in a manner -at least as prominent as such other comparable authorship credit. -5. Representations, Warranties and Disclaimer - -UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS -THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING -THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT -LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR -PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, -OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME -JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH -EXCLUSION MAY NOT APPLY TO YOU. - -6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN -NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, -INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS -LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - -7. Termination - -This License and the rights granted hereunder will terminate automatically upon -any breach by You of the terms of this License. Individuals or entities who have -received Derivative Works or Collective Works from You under this License, -however, will not have their licenses terminated provided such individuals or -entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, -and 8 will survive any termination of this License. -Subject to the above terms and conditions, the license granted here is perpetual -(for the duration of the applicable copyright in the Work). Notwithstanding the -above, Licensor reserves the right to release the Work under different license -terms or to stop distributing the Work at any time; provided, however that any -such election will not serve to withdraw this License (or any other license that -has been, or is required to be, granted under the terms of this License), and -this License will continue in full force and effect unless terminated as stated -above. -8. Miscellaneous - -Each time You distribute or publicly digitally perform the Work or a Collective -Work, the Licensor offers to the recipient a license to the Work on the same -terms and conditions as the license granted to You under this License. -Each time You distribute or publicly digitally perform a Derivative Work, -Licensor offers to the recipient a license to the original Work on the same -terms and conditions as the license granted to You under this License. -If any provision of this License is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this License, and without further action by the parties to this -agreement, such provision shall be reformed to the minimum extent necessary to -make such provision valid and enforceable. -No term or provision of this License shall be deemed waived and no breach -consented to unless such waiver or consent shall be in writing and signed by the -party to be charged with such waiver or consent. -This License constitutes the entire agreement between the parties with respect -to the Work licensed here. There are no understandings, agreements or -representations with respect to the Work not specified here. Licensor shall not -be bound by any additional provisions that may appear in any communication from -You. This License may not be modified without the mutual written agreement of -the Licensor and You. - -The binary distribution of this product bundles these dependencies under the -following license: -jamon-runtime 2.4.1 --------------------------------------------------------------------------------- -(MPL 2.0) - Mozilla Public License - Version 2.0 - -1.1. “Contributor” -means each individual or legal entity that creates, contributes to the creation -of, or owns Covered Software. - -1.2. “Contributor Version” -means the combination of the Contributions of others (if any) used by a -Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” -means Covered Software of a particular Contributor. - -1.4. “Covered Software” -means Source Code Form to which the initial Contributor has attached the notice -in Exhibit A, the Executable Form of such Source Code Form, and Modifications of -such Source Code Form, in each case including portions thereof. - -1.5. “Incompatible With Secondary Licenses” -means - -that the initial Contributor has attached the notice described in Exhibit B to -the Covered Software; or - -that the Covered Software was made available under the terms of version 1.1 or -earlier of the License, but not also under the terms of a Secondary License. - -1.6. “Executable Form” -means any form of the work other than Source Code Form. - -1.7. “Larger Work” -means a work that combines Covered Software with other material, in a separate -file or files, that is not Covered Software. - -1.8. “License” -means this document. - -1.9. “Licensable” -means having the right to grant, to the maximum extent possible, whether at the -time of the initial grant or subsequently, any and all of the rights conveyed by -this License. - -1.10. “Modifications” -means any of the following: - -any file in Source Code Form that results from an addition to, deletion from, or -modification of the contents of Covered Software; or - -any new file in Source Code Form that contains any Covered Software. - -1.11. “Patent Claims” of a Contributor -means any patent claim(s), including without limitation, method, process, and -apparatus claims, in any patent Licensable by such Contributor that would be -infringed, but for the grant of the License, by the making, using, selling, -offering for sale, having made, import, or transfer of either its Contributions -or its Contributor Version. - -1.12. “Secondary License” -means either the GNU General Public License, Version 2.0, the GNU Lesser General -Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, -or any later versions of those licenses. - -1.13. “Source Code Form” -means the form of the work preferred for making modifications. - -1.14. “You” (or “Your”) -means an individual or a legal entity exercising rights under this License. For -legal entities, “You” includes any entity that controls, is controlled by, -or is under common control with You. For purposes of this definition, -“control” means (a) the power, direct or indirect, to cause the direction or -management of such entity, whether by contract or otherwise, or (b) ownership of -more than fifty percent (50%) of the outstanding shares or beneficial ownership -of such entity. - -2. License Grants and Conditions - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive -license: - -under intellectual property rights (other than patent or trademark) Licensable -by such Contributor to use, reproduce, make available, modify, display, perform, -distribute, and otherwise exploit its Contributions, either on an unmodified -basis, with Modifications, or as part of a Larger Work; and - -under Patent Claims of such Contributor to make, use, sell, offer for sale, have -made, import, and otherwise transfer either its Contributions or its Contributor -Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution become -effective for each Contribution on the date the Contributor first distributes -such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under this -License. No additional rights or licenses will be implied from the distribution -or licensing of Covered Software under this License. Notwithstanding Section -2.1(b) above, no patent license is granted by a Contributor: - -for any code that a Contributor has removed from Covered Software; or - -for infringements caused by: (i) Your and any other third party’s -modifications of Covered Software, or (ii) the combination of its Contributions -with other software (except as part of its Contributor Version); or - -under Patent Claims infringed by Covered Software in the absence of its -Contributions. - -This License does not grant any rights in the trademarks, service marks, or -logos of any Contributor (except as may be necessary to comply with the notice -requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to distribute -the Covered Software under a subsequent version of this License (see Section -10.2) or under the terms of a Secondary License (if permitted under the terms of -Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its Contributions are -its original creation(s) or it has sufficient rights to grant the rights to its -Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under applicable -copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in -Section 2.1. - -3. Responsibilities - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under the -terms of this License. You must inform recipients that the Source Code Form of -the Covered Software is governed by the terms of this License, and how they can -obtain a copy of this License. You may not attempt to alter or restrict the -recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -such Covered Software must also be made available in Source Code Form, as -described in Section 3.1, and You must inform recipients of the Executable Form -how they can obtain a copy of such Source Code Form by reasonable means in a -timely manner, at a charge no more than the cost of distribution to the -recipient; and - -You may distribute such Executable Form under the terms of this License, or -sublicense it under different terms, provided that the license for the -Executable Form does not attempt to limit or alter the recipients’ rights in -the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, provided -that You also comply with the requirements of this License for the Covered -Software. If the Larger Work is a combination of Covered Software with a work -governed by one or more Secondary Licenses, and the Covered Software is not -Incompatible With Secondary Licenses, this License permits You to additionally -distribute such Covered Software under the terms of such Secondary License(s), -so that the recipient of the Larger Work may, at their option, further -distribute the Covered Software under the terms of either this License or such -Secondary License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices (including -copyright notices, patent notices, disclaimers of warranty, or limitations of -liability) contained within the Source Code Form of the Covered Software, except -that You may alter any license notices to the extent required to remedy known -factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, indemnity -or liability obligations to one or more recipients of Covered Software. However, -You may do so only on Your own behalf, and not on behalf of any Contributor. You -must make it absolutely clear that any such warranty, support, indemnity, or -liability obligation is offered by You alone, and You hereby agree to indemnify -every Contributor for any liability incurred by such Contributor as a result of -warranty, support, indemnity or liability terms You offer. You may include -additional disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - -If it is impossible for You to comply with any of the terms of this License with -respect to some or all of the Covered Software due to statute, judicial order, -or regulation then You must: (a) comply with the terms of this License to the -maximum extent possible; and (b) describe the limitations and the code they -affect. Such description must be placed in a text file included with all -distributions of the Covered Software under this License. Except to the extent -prohibited by statute or regulation, such description must be sufficiently -detailed for a recipient of ordinary skill to be able to understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You -fail to comply with any of its terms. However, if You become compliant, then the -rights granted under this License from a particular Contributor are reinstated -(a) provisionally, unless and until such Contributor explicitly and finally -terminates Your grants, and (b) on an ongoing basis, if such Contributor fails -to notify You of the non-compliance by some reasonable means prior to 60 days -after You have come back into compliance. Moreover, Your grants from a -particular Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the first -time You have received notice of non-compliance with this License from such -Contributor, and You become compliant prior to 30 days after Your receipt of the -notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, counter-claims, and -cross-claims) alleging that a Contributor Version directly or indirectly -infringes any patent, then the rights granted to You by any and all Contributors -for the Covered Software under Section 2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user -license agreements (excluding distributors and resellers) which have been -validly granted by You or Your distributors under this License prior to -termination shall survive termination. - -6. Disclaimer of Warranty - -Covered Software is provided under this License on an “as is” basis, without -warranty of any kind, either expressed, implied, or statutory, including, -without limitation, warranties that the Covered Software is free of defects, -merchantable, fit for a particular purpose or non-infringing. The entire risk as -to the quality and performance of the Covered Software is with You. Should any -Covered Software prove defective in any respect, You (not any Contributor) -assume the cost of any necessary servicing, repair, or correction. This -disclaimer of warranty constitutes an essential part of this License. No use of -any Covered Software is authorized under this License except under this -disclaimer. - -7. Limitation of Liability - -Under no circumstances and under no legal theory, whether tort (including -negligence), contract, or otherwise, shall any Contributor, or anyone who -distributes Covered Software as permitted above, be liable to You for any -direct, indirect, special, incidental, or consequential damages of any character -including, without limitation, damages for lost profits, loss of goodwill, work -stoppage, computer failure or malfunction, or any and all other commercial -damages or losses, even if such party shall have been informed of the -possibility of such damages. This limitation of liability shall not apply to -liability for death or personal injury resulting from such party’s negligence -to the extent applicable law prohibits such limitation. Some jurisdictions do -not allow the exclusion or limitation of incidental or consequential damages, so -this exclusion and limitation may not apply to You. - -8. Litigation - -Any litigation relating to this License may be brought only in the courts of a -jurisdiction where the defendant maintains its principal place of business and -such litigation shall be governed by laws of that jurisdiction, without -reference to its conflict-of-law provisions. Nothing in this Section shall -prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - -This License represents the complete agreement concerning the subject matter -hereof. If any provision of this License is held to be unenforceable, such -provision shall be reformed only to the extent necessary to make it enforceable. -Any law or regulation which provides that the language of a contract shall be -construed against the drafter shall not be used to construe this License against -a Contributor. - -10. Versions of the License - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section 10.3, -no one other than the license steward has the right to modify or publish new -versions of this License. Each version will be given a distinguishing version -number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version of the -License under which You originally received the Covered Software, or under the -terms of any subsequent version published by the license steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to create a -new license for such software, you may create and use a modified version of this -License if you rename the license and remove any references to the name of the -license steward (except to note that such modified license differs from this -License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - -If You choose to distribute Source Code Form that is Incompatible With Secondary -Licenses under the terms of this version of the License, the notice described in -Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - -This Source Code Form is subject to the terms of the Mozilla Public License, v. -2.0. If a copy of the MPL was not distributed with this file, You can obtain one -at https://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - -This Source Code Form is “Incompatible With Secondary Licenses”, as defined -by the Mozilla Public License, v. 2.0. - -The binary distribution of this product bundles these dependencies under the -following license: -JDOM 1.1 --------------------------------------------------------------------------------- -/*-- - - Copyright (C) 2000-2004 Jason Hunter & Brett McLaughlin. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions, and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions, and the disclaimer that follows - these conditions in the documentation and/or other materials - provided with the distribution. - - 3. The name "JDOM" must not be used to endorse or promote products - derived from this software without prior written permission. For - written permission, please contact . - - 4. Products derived from this software may not be called "JDOM", nor - may "JDOM" appear in their name, without prior written permission - from the JDOM Project Management . - - In addition, we request (but do not require) that you include in the - end-user documentation provided with the redistribution and/or in the - software itself an acknowledgement equivalent to the following: - "This product includes software developed by the - JDOM Project (http://www.jdom.org/)." - Alternatively, the acknowledgment may be graphical using the logos - available at http://www.jdom.org/images/logos. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED - WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - - This software consists of voluntary contributions made by many - individuals on behalf of the JDOM Project and was originally - created by Jason Hunter and - Brett McLaughlin . For more information - on the JDOM Project, please see . - - */ - -The binary distribution of this product bundles these dependencies under the -following license: -Hbase Server 1.2.4 --------------------------------------------------------------------------------- -This project bundles a derivative image for our Orca Logo. This image is -available under the Creative Commons By Attribution 3.0 License. - - Creative Commons Legal Code - - Attribution 3.0 Unported - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR - DAMAGES RESULTING FROM ITS USE. - - License - - THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE - COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY - COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS - AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. - - BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE - TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY - BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS - CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND - CONDITIONS. - - 1. Definitions - - a. "Adaptation" means a work based upon the Work, or upon the Work and - other pre-existing works, such as a translation, adaptation, - derivative work, arrangement of music or other alterations of a - literary or artistic work, or phonogram or performance and includes - cinematographic adaptations or any other form in which the Work may be - recast, transformed, or adapted including in any form recognizably - derived from the original, except that a work that constitutes a - Collection will not be considered an Adaptation for the purpose of - this License. For the avoidance of doubt, where the Work is a musical - work, performance or phonogram, the synchronization of the Work in - timed-relation with a moving image ("synching") will be considered an - Adaptation for the purpose of this License. - b. "Collection" means a collection of literary or artistic works, such as - encyclopedias and anthologies, or performances, phonograms or - broadcasts, or other works or subject matter other than works listed - in Section 1(f) below, which, by reason of the selection and - arrangement of their contents, constitute intellectual creations, in - which the Work is included in its entirety in unmodified form along - with one or more other contributions, each constituting separate and - independent works in themselves, which together are assembled into a - collective whole. A work that constitutes a Collection will not be - considered an Adaptation (as defined above) for the purposes of this - License. - c. "Distribute" means to make available to the public the original and - copies of the Work or Adaptation, as appropriate, through sale or - other transfer of ownership. - d. "Licensor" means the individual, individuals, entity or entities that - offer(s) the Work under the terms of this License. - e. "Original Author" means, in the case of a literary or artistic work, - the individual, individuals, entity or entities who created the Work - or if no individual or entity can be identified, the publisher; and in - addition (i) in the case of a performance the actors, singers, - musicians, dancers, and other persons who act, sing, deliver, declaim, - play in, interpret or otherwise perform literary or artistic works or - expressions of folklore; (ii) in the case of a phonogram the producer - being the person or legal entity who first fixes the sounds of a - performance or other sounds; and, (iii) in the case of broadcasts, the - organization that transmits the broadcast. - f. "Work" means the literary and/or artistic work offered under the terms - of this License including without limitation any production in the - literary, scientific and artistic domain, whatever may be the mode or - form of its expression including digital form, such as a book, - pamphlet and other writing; a lecture, address, sermon or other work - of the same nature; a dramatic or dramatico-musical work; a - choreographic work or entertainment in dumb show; a musical - composition with or without words; a cinematographic work to which are - assimilated works expressed by a process analogous to cinematography; - a work of drawing, painting, architecture, sculpture, engraving or - lithography; a photographic work to which are assimilated works - expressed by a process analogous to photography; a work of applied - art; an illustration, map, plan, sketch or three-dimensional work - relative to geography, topography, architecture or science; a - performance; a broadcast; a phonogram; a compilation of data to the - extent it is protected as a copyrightable work; or a work performed by - a variety or circus performer to the extent it is not otherwise - considered a literary or artistic work. - g. "You" means an individual or entity exercising rights under this - License who has not previously violated the terms of this License with - respect to the Work, or who has received express permission from the - Licensor to exercise rights under this License despite a previous - violation. - h. "Publicly Perform" means to perform public recitations of the Work and - to communicate to the public those public recitations, by any means or - process, including by wire or wireless means or public digital - performances; to make available to the public Works in such a way that - members of the public may access these Works from a place and at a - place individually chosen by them; to perform the Work to the public - by any means or process and the communication to the public of the - performances of the Work, including by public digital performance; to - broadcast and rebroadcast the Work by any means including signs, - sounds or images. - i. "Reproduce" means to make copies of the Work by any means including - without limitation by sound or visual recordings and the right of - fixation and reproducing fixations of the Work, including storage of a - protected performance or phonogram in digital form or other electronic - medium. - - 2. Fair Dealing Rights. Nothing in this License is intended to reduce, - limit, or restrict any uses free from copyright or rights arising from - limitations or exceptions that are provided for in connection with the - copyright protection under copyright law or other applicable laws. - - 3. License Grant. Subject to the terms and conditions of this License, - Licensor hereby grants You a worldwide, royalty-free, non-exclusive, - perpetual (for the duration of the applicable copyright) license to - exercise the rights in the Work as stated below: - - a. to Reproduce the Work, to incorporate the Work into one or more - Collections, and to Reproduce the Work as incorporated in the - Collections; - b. to create and Reproduce Adaptations provided that any such Adaptation, - including any translation in any medium, takes reasonable steps to - clearly label, demarcate or otherwise identify that changes were made - to the original Work. For example, a translation could be marked "The - original work was translated from English to Spanish," or a - modification could indicate "The original work has been modified."; - c. to Distribute and Publicly Perform the Work including as incorporated - in Collections; and, - d. to Distribute and Publicly Perform Adaptations. - e. For the avoidance of doubt: - - i. Non-waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme cannot be waived, the Licensor - reserves the exclusive right to collect such royalties for any - exercise by You of the rights granted under this License; - ii. Waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme can be waived, the Licensor waives the - exclusive right to collect such royalties for any exercise by You - of the rights granted under this License; and, - iii. Voluntary License Schemes. The Licensor waives the right to - collect royalties, whether individually or, in the event that the - Licensor is a member of a collecting society that administers - voluntary licensing schemes, via that society, from any exercise - by You of the rights granted under this License. - - The above rights may be exercised in all media and formats whether now - known or hereafter devised. The above rights include the right to make - such modifications as are technically necessary to exercise the rights in - other media and formats. Subject to Section 8(f), all rights not expressly - granted by Licensor are hereby reserved. - - 4. Restrictions. The license granted in Section 3 above is expressly made - subject to and limited by the following restrictions: - - a. You may Distribute or Publicly Perform the Work only under the terms - of this License. You must include a copy of, or the Uniform Resource - Identifier (URI) for, this License with every copy of the Work You - Distribute or Publicly Perform. You may not offer or impose any terms - on the Work that restrict the terms of this License or the ability of - the recipient of the Work to exercise the rights granted to that - recipient under the terms of the License. You may not sublicense the - Work. You must keep intact all notices that refer to this License and - to the disclaimer of warranties with every copy of the Work You - Distribute or Publicly Perform. When You Distribute or Publicly - Perform the Work, You may not impose any effective technological - measures on the Work that restrict the ability of a recipient of the - Work from You to exercise the rights granted to that recipient under - the terms of the License. This Section 4(a) applies to the Work as - incorporated in a Collection, but this does not require the Collection - apart from the Work itself to be made subject to the terms of this - License. If You create a Collection, upon notice from any Licensor You - must, to the extent practicable, remove from the Collection any credit - as required by Section 4(b), as requested. If You create an - Adaptation, upon notice from any Licensor You must, to the extent - practicable, remove from the Adaptation any credit as required by - Section 4(b), as requested. - b. If You Distribute, or Publicly Perform the Work or any Adaptations or - Collections, You must, unless a request has been made pursuant to - Section 4(a), keep intact all copyright notices for the Work and - provide, reasonable to the medium or means You are utilizing: (i) the - name of the Original Author (or pseudonym, if applicable) if supplied, - and/or if the Original Author and/or Licensor designate another party - or parties (e.g., a sponsor institute, publishing entity, journal) for - attribution ("Attribution Parties") in Licensor's copyright notice, - terms of service or by other reasonable means, the name of such party - or parties; (ii) the title of the Work if supplied; (iii) to the - extent reasonably practicable, the URI, if any, that Licensor - specifies to be associated with the Work, unless such URI does not - refer to the copyright notice or licensing information for the Work; - and (iv) , consistent with Section 3(b), in the case of an Adaptation, - a credit identifying the use of the Work in the Adaptation (e.g., - "French translation of the Work by Original Author," or "Screenplay - based on original Work by Original Author"). The credit required by - this Section 4 (b) may be implemented in any reasonable manner; - provided, however, that in the case of a Adaptation or Collection, at - a minimum such credit will appear, if a credit for all contributing - authors of the Adaptation or Collection appears, then as part of these - credits and in a manner at least as prominent as the credits for the - other contributing authors. For the avoidance of doubt, You may only - use the credit required by this Section for the purpose of attribution - in the manner set out above and, by exercising Your rights under this - License, You may not implicitly or explicitly assert or imply any - connection with, sponsorship or endorsement by the Original Author, - Licensor and/or Attribution Parties, as appropriate, of You or Your - use of the Work, without the separate, express prior written - permission of the Original Author, Licensor and/or Attribution - Parties. - c. Except as otherwise agreed in writing by the Licensor or as may be - otherwise permitted by applicable law, if You Reproduce, Distribute or - Publicly Perform the Work either by itself or as part of any - Adaptations or Collections, You must not distort, mutilate, modify or - take other derogatory action in relation to the Work which would be - prejudicial to the Original Author's honor or reputation. Licensor - agrees that in those jurisdictions (e.g. Japan), in which any exercise - of the right granted in Section 3(b) of this License (the right to - make Adaptations) would be deemed to be a distortion, mutilation, - modification or other derogatory action prejudicial to the Original - Author's honor and reputation, the Licensor will waive or not assert, - as appropriate, this Section, to the fullest extent permitted by the - applicable national law, to enable You to reasonably exercise Your - right under Section 3(b) of this License (right to make Adaptations) - but not otherwise. - - 5. Representations, Warranties and Disclaimer - - UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR - OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY - KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, - INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, - FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF - LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, - WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION - OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. - - 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE - LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR - ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES - ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS - BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - 7. Termination - - a. This License and the rights granted hereunder will terminate - automatically upon any breach by You of the terms of this License. - Individuals or entities who have received Adaptations or Collections - from You under this License, however, will not have their licenses - terminated provided such individuals or entities remain in full - compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will - survive any termination of this License. - b. Subject to the above terms and conditions, the license granted here is - perpetual (for the duration of the applicable copyright in the Work). - Notwithstanding the above, Licensor reserves the right to release the - Work under different license terms or to stop distributing the Work at - any time; provided, however that any such election will not serve to - withdraw this License (or any other license that has been, or is - required to be, granted under the terms of this License), and this - License will continue in full force and effect unless terminated as - stated above. - - 8. Miscellaneous - - a. Each time You Distribute or Publicly Perform the Work or a Collection, - the Licensor offers to the recipient a license to the Work on the same - terms and conditions as the license granted to You under this License. - b. Each time You Distribute or Publicly Perform an Adaptation, Licensor - offers to the recipient a license to the original Work on the same - terms and conditions as the license granted to You under this License. - c. If any provision of this License is invalid or unenforceable under - applicable law, it shall not affect the validity or enforceability of - the remainder of the terms of this License, and without further action - by the parties to this agreement, such provision shall be reformed to - the minimum extent necessary to make such provision valid and - enforceable. - d. No term or provision of this License shall be deemed waived and no - breach consented to unless such waiver or consent shall be in writing - and signed by the party to be charged with such waiver or consent. - e. This License constitutes the entire agreement between the parties with - respect to the Work licensed here. There are no understandings, - agreements or representations with respect to the Work not specified - here. Licensor shall not be bound by any additional provisions that - may appear in any communication from You. This License may not be - modified without the mutual written agreement of the Licensor and You. - f. The rights granted under, and the subject matter referenced, in this - License were drafted utilizing the terminology of the Berne Convention - for the Protection of Literary and Artistic Works (as amended on - September 28, 1979), the Rome Convention of 1961, the WIPO Copyright - Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 - and the Universal Copyright Convention (as revised on July 24, 1971). - These rights and subject matter take effect in the relevant - jurisdiction in which the License terms are sought to be enforced - according to the corresponding provisions of the implementation of - those treaty provisions in the applicable national law. If the - standard suite of rights granted under applicable copyright law - includes additional rights not granted under this License, such - additional rights are deemed to be included in the License; this - License is not intended to restrict the license of any rights under - applicable law. - - Creative Commons Notice - - Creative Commons is not a party to this License, and makes no warranty - whatsoever in connection with the Work. Creative Commons will not be - liable to You or any party on any legal theory for any damages - whatsoever, including without limitation any general, special, - incidental or consequential damages arising in connection to this - license. Notwithstanding the foregoing two (2) sentences, if Creative - Commons has expressly identified itself as the Licensor hereunder, it - shall have all rights and obligations of Licensor. - - Except for the limited purpose of indicating to the public that the - Work is licensed under the CCPL, Creative Commons does not authorize - the use by either party of the trademark "Creative Commons" or any - related trademark or logo of Creative Commons without the prior - written consent of Creative Commons. Any permitted use will be in - compliance with Creative Commons' then-current trademark usage - guidelines, as may be published on its website or otherwise made - available upon request from time to time. For the avoidance of doubt, - this trademark restriction does not form part of this License. - - Creative Commons may be contacted at https://creativecommons.org/. --------------------------------------------------------------------------------- - -For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs -/server/datanode/checker/AbstractFuture.java and -hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs -/server/datanode/checker/TimeoutFuture.java - -Copyright (C) 2007 The Guava Authors - -Licensed under the Apache License, Version 2.0 (the "License"); you may not -use this file except in compliance with the License. You may obtain a copy of -the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations under -the License. - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). - -The binary distribution of this product bundles binaries of -org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the -following notices: -* Copyright 2011 Dain Sundstrom -* Copyright 2011 FuseSource Corp. http://fusesource.com - -The binary distribution of this product bundles binaries of -AWS SDK for Java - Bundle 1.11.134, -AWS Java SDK for AWS KMS 1.11.134, -AWS Java SDK for Amazon S3 1.11.134, -AWS Java SDK for AWS STS 1.11.134, -JMES Path Query library 1.0, -which has the following notices: - * This software includes third party software subject to the following - copyrights: - XML parsing and utility functions from JetS3t - Copyright - 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - - Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility - functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. - -The binary distribution of this product bundles binaries of -Gson 2.2.4, -which has the following notices: - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'ArrayDeque', written by Josh -Bloch of Google, Inc: - - * LICENSE: - * license/LICENSE.deque.txt (Public Domain) - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified version of Roland Kuhn's ASL2 -AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. -It can be obtained at: - - * LICENSE: - * license/LICENSE.abstractnodequeue.txt (Public Domain) - * HOMEPAGE: - * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/libdivsufsort/ - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/protobuf/ - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/snappy/ - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * http://code.google.com/p/caliper/ - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - -This product contains a modified portion of 'JDOM 1.1', which can be obtained at: - - * LICENSE: - * https://github.com/hunterhacker/jdom/blob/jdom-1.1/core/LICENSE.txt - * HOMEPAGE: - * http://www.jdom.org/ - -The binary distribution of this product bundles binaries of -Commons Codec 1.4, -which has the following notices: - * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - =============================================================================== - The content of package org.apache.commons.codec.language.bm has been translated - from the original php source code available at http://stevemorse.org/phoneticinfo.htm - with permission from the original authors. - Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -The binary distribution of this product bundles binaries of -Commons Lang 2.6, -which has the following notices: - * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -The binary distribution of this product bundles binaries of -Apache Log4j 1.2.17, -which has the following notices: - * ResolverUtil.java - Copyright 2005-2006 Tim Fennell - Dumbster SMTP test server - Copyright 2004 Jason Paul Kitchen - TypeUtil.java - Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams - -The binary distribution of this product bundles binaries of -"Java Concurrency in Practice" book annotations 1.0, -which has the following notices: - * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative - Commons Attribution License (http://creativecommons.org/licenses/by/2.5) - Official home: http://www.jcip.net Any republication or derived work - distributed in source code form must include this copyright and license - notice. - -The binary distribution of this product bundles binaries of -Jetty :: Http Utility 9.3.19., -Jetty :: IO Utility 9.3.19., -Jetty :: Security 9.3.19., -Jetty :: Server Core 9.3.19., -Jetty :: Servlet Handling 9.3.19., -Jetty :: Utilities 9.3.19., -Jetty :: Utilities :: Ajax, -Jetty :: Webapp Application Support 9.3.19., -Jetty :: XML utilities 9.3.19., -which has the following notices: - * ============================================================== - Jetty Web Container - Copyright 1995-2016 Mort Bay Consulting Pty Ltd. - ============================================================== - - The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd - unless otherwise noted. - - Jetty is dual licensed under both - - * The Apache 2.0 License - http://www.apache.org/licenses/LICENSE-2.0.html - - and - - * The Eclipse Public 1.0 License - http://www.eclipse.org/legal/epl-v10.html - - Jetty may be distributed under either license. - - ------ - Eclipse - - The following artifacts are EPL. - * org.eclipse.jetty.orbit:org.eclipse.jdt.core - - The following artifacts are EPL and ASL2. - * org.eclipse.jetty.orbit:javax.security.auth.message - - The following artifacts are EPL and CDDL 1.0. - * org.eclipse.jetty.orbit:javax.mail.glassfish - - ------ - Oracle - - The following artifacts are CDDL + GPLv2 with classpath exception. - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - * javax.servlet:javax.servlet-api - * javax.annotation:javax.annotation-api - * javax.transaction:javax.transaction-api - * javax.websocket:javax.websocket-api - - ------ - Oracle OpenJDK - - If ALPN is used to negotiate HTTP/2 connections, then the following - artifacts may be included in the distribution or downloaded when ALPN - module is selected. - - * java.sun.security.ssl - - These artifacts replace/modify OpenJDK classes. The modififications - are hosted at github and both modified and original are under GPL v2 with - classpath exceptions. - http://openjdk.java.net/legal/gplv2+ce.html - - ------ - OW2 - - The following artifacts are licensed by the OW2 Foundation according to the - terms of http://asm.ow2.org/license.html - - org.ow2.asm:asm-commons - org.ow2.asm:asm - - ------ - Apache - - The following artifacts are ASL2 licensed. - - org.apache.taglibs:taglibs-standard-spec - org.apache.taglibs:taglibs-standard-impl - - ------ - MortBay - - The following artifacts are ASL2 licensed. Based on selected classes from - following Apache Tomcat jars, all ASL2 licensed. - - org.mortbay.jasper:apache-jsp - org.apache.tomcat:tomcat-jasper - org.apache.tomcat:tomcat-juli - org.apache.tomcat:tomcat-jsp-api - org.apache.tomcat:tomcat-el-api - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-api - org.apache.tomcat:tomcat-util-scan - org.apache.tomcat:tomcat-util - - org.mortbay.jasper:apache-el - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-el-api - - ------ - Mortbay - - The following artifacts are CDDL + GPLv2 with classpath exception. - - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - org.eclipse.jetty.toolchain:jetty-schemas - - ------ - Assorted - - The UnixCrypt.java code implements the one way cryptography used by - Unix systems for simple password protection. Copyright 1996 Aki Yoshida, - modified April 2001 by Iris Van den Broeke, Daniel Deville. - Permission to use, copy, modify and distribute UnixCrypt - for non-commercial or commercial purposes and without fee is - granted provided that the copyright notice appears in all copies./ - -The binary distribution of this product bundles binaries of -Snappy for Java 1.0.4.1, -which has the following notices: - * This product includes software developed by Google - Snappy: http://code.google.com/p/snappy/ (New BSD License) - - This product includes software developed by Apache - PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ - (Apache 2.0 license) - - This library containd statically linked libstdc++. This inclusion is allowed by - "GCC RUntime Library Exception" - http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html - - == Contributors == - * Tatu Saloranta - * Providing benchmark suite - * Alec Wysoker - * Performance and memory usage improvement - -The binary distribution of this product bundles binaries of -Xerces2 Java Parser 2.9.1, -which has the following notices: - * ========================================================================= - == NOTICE file corresponding to section 4(d) of the Apache License, == - == Version 2.0, in this case for the Apache Xerces Java distribution. == - ========================================================================= - - Apache Xerces Java - Copyright 1999-2007 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were originally based on the following: - - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. - - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. - - voluntary contributions made by Paul Eng on behalf of the - Apache Software Foundation that were originally developed at iClick, Inc., - software copyright (c) 1999. - -The binary distribution of this product bundles binaries of -Logback Classic Module 1.1.2, -Logback Core Module 1.1.2, -which has the following notices: - * Logback: the reliable, generic, fast and flexible logging framework. - Copyright (C) 1999-2012, QOS.ch. All rights reserved. - -The binary distribution of this product bundles binaries of -Apache HBase - Annotations 1.2.6, -Apache HBase - Client 1.2.6, -Apache HBase - Common 1.2.6, -Apache HBase - Hadoop Compatibility 1.2.6, -Apache HBase - Hadoop Two Compatibility 1.2.6, -Apache HBase - Prefix Tree 1.2.6, -Apache HBase - Procedure 1.2.6, -Apache HBase - Protocol 1.2.6, -Apache HBase - Server 1.2.6, -which has the following notices: - * Apache HBase - Copyright 2007-2015 The Apache Software Foundation - - -- - This product incorporates portions of the 'Hadoop' project - - Copyright 2007-2009 The Apache Software Foundation - - Licensed under the Apache License v2.0 - -- - Our Orca logo we got here: http://www.vectorfree.com/jumping-orca - It is licensed Creative Commons Attribution 3.0. - See https://creativecommons.org/licenses/by/3.0/us/ - We changed the logo by stripping the colored background, inverting - it and then rotating it some. - - Later we found that vectorfree.com image is not properly licensed. - The original is owned by vectorportal.com. The original was - relicensed so we could use it as Creative Commons Attribution 3.0. - The license is bundled with the download available here: - http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp - -- - This product includes portions of the Bootstrap project v3.0.0 - - Copyright 2013 Twitter, Inc. - - Licensed under the Apache License v2.0 - - This product uses the Glyphicons Halflings icon set. - - http://glyphicons.com/ - - Copyright Jan Kovařík - - Licensed under the Apache License v2.0 as a part of the Bootstrap project. - - -- - This product includes portions of the Guava project v14, specifically - 'hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java' - - Copyright (C) 2007 The Guava Authors - - Licensed under the Apache License, Version 2.0 - -The binary distribution of this product bundles binaries of -Phoenix Core 4.7.0, -which has the following notices: - Apache Phoenix - Copyright 2013-2016 The Apache Software Foundation - - This product includes software developed by The Apache Software - Foundation (http://www.apache.org/). - - This also includes: - - The phoenix-spark module has been adapted from the phoenix-spark library - distributed under the terms of the Apache 2 license. Original source copyright: - Copyright 2014 Simply Measured, Inc. - Copyright 2015 Interset Software Inc. - - The file bin/daemon.py is based on the file of the same name in python-daemon 2.0.5 - (https://pypi.python.org/pypi/python-daemon/). Original source copyright: - # Copyright © 2008–2015 Ben Finney - # Copyright © 2007–2008 Robert Niederreiter, Jens Klein - # Copyright © 2004–2005 Chad J. Schroeder - # Copyright © 2003 Clark Evans - # Copyright © 2002 Noah Spurrier - # Copyright © 2001 Jürgen Hermann - -The binary distribution of this product bundles binaries of -Plexus Cipher: encryption/decryption Component 1.4, -which has the following notices: - * The code in this component contains a class - Base64 taken from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/Base64.java - which is Apache license: http://www.apache.org/licenses/LICENSE-2.0 - - The PBE key processing routine PBECipher.createCipher() is adopted from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/OpenSSL.java - which is also Apache APL-2.0 license: http://www.apache.org/licenses/LICENSE-2.0 - -The binary distribution of this product bundles binaries of -software.amazon.ion:ion-java 1.0.1, -which has the following notices: - * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -The binary distribution of this product bundles binaries of -joda-time:joda-time:2.9.9 -which has the following notices: - * ============================================================================= - = NOTICE file corresponding to section 4d of the Apache License Version 2.0 = - ============================================================================= - This product includes software developed by - Joda.org (http://www.joda.org/). - -The binary distribution of this product bundles binaries of -Ehcache 3.3.1, -which has the following notices: - * Ehcache V3 Copyright 2014-2016 Terracotta, Inc. - -The binary distribution of this product bundles binaries of -snakeyaml (https://bitbucket.org/asomov/snakeyaml), -which has the following notices: - * Copyright (c) 2008, http://www.snakeyaml.org - -The binary distribution of this product bundles binaries of -swagger-annotations (https://github.com/swagger-api/swagger-core), -which has the following notices: - * Copyright 2016 SmartBear Software - -The binary distribution of this product bundles binaries of -metrics-core 3.2.4 -which has the following notices: - * Copyright 2010-2013 Coda Hale and Yammer, Inc. - - This product includes software developed by Coda Hale and Yammer, Inc. - - This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, - LongAdder), which was released with the following comments: - - Written by Doug Lea with assistance from members of JCP JSR-166 - Expert Group and released to the public domain, as explained at - http://creativecommons.org/publicdomain/zero/1.0/ - -Apache Commons IO -Copyright 2002-2012 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -Apache Commons Collections -Copyright 2001-2015 The Apache Software Foundation - -Apache Commons Logging -Copyright 2003-2013 The Apache Software Foundation - -Apache Commons Lang -Copyright 2001-2011 The Apache Software Foundation - -Apache Commons BeanUtils -Copyright 2000-2016 The Apache Software Foundation - -Apache Commons Configuration -Copyright 2001-2017 The Apache Software Foundation - -Apache Commons Lang -Copyright 2001-2014 The Apache Software Foundation - -This product includes software from the Spring Framework, -under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -htrace-core4 -Copyright 2016 The Apache Software Foundation - -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -flink-hadoop-fs -Copyright 2014-2019 The Apache Software Foundation - -Apache HttpClient -Copyright 1999-2017 The Apache Software Foundation - -Apache HttpCore -Copyright 2005-2017 The Apache Software Foundation - -Apache Commons Codec -Copyright 2002-2014 The Apache Software Foundation - -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -============================================================================= -= NOTICE file corresponding to section 4d of the Apache License Version 2.0 = -============================================================================= -This product includes software developed by -Joda.org (http://www.joda.org/). - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-s3-fs-presto -Copyright 2014-2019 The Apache Software Foundation - -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.facebook.presto:presto-hive:0.187 -- com.facebook.presto.hadoop:hadoop-apache2:2.7.3-1 -- com.google.guava:guava:21.0 -- io.airlift:configuration:0.153 -- io.airlift:log:0.153 -- io.airlift:stats:0.153 -- io.airlift:units:1.0 -- io.airlift:slice:0.31 -- com.fasterxml.jackson.core:jackson-annotations:2.8.1 -- com.fasterxml.jackson.core:jackson-core:2.8.1 -- com.fasterxml.jackson.core:jackson-databind:2.8.1 -- joda-time:joda-time:2.5 -- org.weakref:jmxutils:1.19 - -This project bundles the following dependencies under the Creative Commons CC0 1.0 Universal Public Domain Dedication License (http://creativecommons.org/publicdomain/zero/1.0/) -See bundled license files for details. - -- org.hdrhistogram:HdrHistogram:2.1.9 - - -flink-s3-fs-base -Copyright 2014-2019 The Apache Software Foundation - -- org.apache.hadoop:hadoop-aws:3.1.0 -- org.apache.httpcomponents:httpcore:4.4.6 -- org.apache.httpcomponents:httpclient:4.5.3 -- commons-codec:commons-codec:1.10 -- commons-logging:commons-logging:1.1.3 -- com.amazonaws:aws-java-sdk-core:1.11.271 -- com.amazonaws:aws-java-sdk-dynamodb:1.11.271 -- com.amazonaws:aws-java-sdk-kms:1.11.271 -- com.amazonaws:aws-java-sdk-s3:1.11.271 -- com.amazonaws:jmespath-java:1.11.271 -- software.amazon.ion:ion-java:1.0.2 -- com.fasterxml.jackson.core:jackson-annotations:2.6.0 -- com.fasterxml.jackson.core:jackson-core:2.6.7 -- com.fasterxml.jackson.core:jackson-databind:2.6.7.1 -- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7 -- joda-time:joda-time:2.5 - -flink-fs-hadoop-shaded -Copyright 2014-2019 The Apache Software Foundation - -- org.apache.hadoop:hadoop-annotations:3.1.0 -- org.apache.hadoop:hadoop-auth:3.1.0 -- org.apache.hadoop:hadoop-common:3.1.0 -- org.apache.htrace:htrace-core4:4.1.0-incubating -- org.apache.commons:commons-configuration2:2.1.1 -- org.apache.commons:commons-lang3:3.3.2 -- commons-lang:commons-lang:2.6 -- commons-collections:commons-collections:3.2.2 -- commons-io:commons-io:2.4 -- commons-logging:commons-logging:1.1.3 -- commons-beanutils:commons-beanutils:1.9.3 -- com.google.guava:guava:11.0.2 -- com.fasterxml.jackson.core:jackson-annotations:2.7.0 -- com.fasterxml.jackson.core:jackson-core:2.7.8 -- com.fasterxml.jackson.core:jackson-databind:2.7.8 -- com.fasterxml.woodstox:woodstox-core:5.0.3 - -This project bundles the following dependencies under the Go License (https://golang.org/LICENSE). -See bundled license files for details. - -- com.google.re2j:re2j:1.1 - -This project bundles the following dependencies under BSD License (https://opensource.org/licenses/bsd-license.php). -See bundled license files for details. - -- org.codehaus.woodstox:stax2-api:3.1.4 (https://github.com/FasterXML/stax2-api/tree/stax2-api-3.1.4) - -This project bundles org.apache.hadoop:*:3.1.0 from which it inherits the following notices: - -The Apache Hadoop project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -For the org.apache.hadoop.util.bloom.* classes: - -/** - * - * Copyright (c) 2005, European Commission project OneLab under contract - * 034819 (http://www.one-lab.org) - * All rights reserved. - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the distribution. - * - Neither the name of the University Catholique de Louvain - UCL - * nor the names of its contributors may be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -For portions of the native implementation of slicing-by-8 CRC calculation -in src/main/native/src/org/apache/hadoop/util: - -Copyright (c) 2008,2009,2010 Massachusetts Institute of Technology. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. -* Neither the name of the Massachusetts Institute of Technology nor - the names of its contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Other portions are under the same license from Intel: -http://sourceforge.net/projects/slicing-by-8/ -/*++ - * - * Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved - * - * This software program is licensed subject to the BSD License, - * available at http://www.opensource.org/licenses/bsd-license.html - * - * Abstract: The main routine - * - --*/ - -For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, - -/* - LZ4 - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -For hadoop-common-project/hadoop-common/src/main/native/gtest ---------------------------------------------------------------------- -Copyright 2008, Google Inc. -All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -The binary distribution of this product bundles these dependencies under the -following license: -re2j 1.1 ---------------------------------------------------------------------- -(GO license) -This is a work derived from Russ Cox's RE2 in Go, whose license -http://golang.org/LICENSE is as follows: - -Copyright (c) 2009 The Go Authors. All rights reserved. - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - - * Neither the name of Google Inc. nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h ---------------------------------------------------------------------- -Copyright 2002 Niels Provos -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles binaries of leveldbjni -(https://github.com/fusesource/leveldbjni), which is available under the -following license: - -Copyright (c) 2011 FuseSource Corp. All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of FuseSource Corp. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: - -Copyright (c) 2012 The FreeBSD Foundation -All rights reserved. - -This software was developed by Pawel Jakub Dawidek under sponsorship from -the FreeBSD Foundation. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. - -============= - -The binary distribution of this product bundles binaries of leveldb -(http://code.google.com/p/leveldb/), which is available under the following -license: - -Copyright (c) 2011 The LevelDB Authors. All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -The binary distribution of this product bundles binaries of snappy -(http://code.google.com/p/snappy/), which is available under the following -license: - -Copyright 2011, Google Inc. -All rights reserved. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ --------------------------------------------------------------------------------- -Copyright (C) 2008-2016, SpryMedia Ltd. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2010 Aleksander Williams - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -The binary distribution of this product bundles these dependencies under the -following license: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css -bootstrap v3.3.6 -broccoli-asset-rev v2.4.2 -broccoli-funnel v1.0.1 -datatables v1.10.8 -em-helpers v0.5.13 -em-table v0.1.6 -ember v2.2.0 -ember-array-contains-helper v1.0.2 -ember-bootstrap v0.5.1 -ember-cli v1.13.13 -ember-cli-app-version v1.0.0 -ember-cli-babel v5.1.6 -ember-cli-content-security-policy v0.4.0 -ember-cli-dependency-checker v1.2.0 -ember-cli-htmlbars v1.0.2 -ember-cli-htmlbars-inline-precompile v0.3.1 -ember-cli-ic-ajax v0.2.1 -ember-cli-inject-live-reload v1.4.0 -ember-cli-jquery-ui v0.0.20 -ember-cli-qunit v1.2.1 -ember-cli-release v0.2.8 -ember-cli-shims v0.0.6 -ember-cli-sri v1.2.1 -ember-cli-test-loader v0.2.1 -ember-cli-uglify v1.2.0 -ember-d3 v0.1.0 -ember-data v2.1.0 -ember-disable-proxy-controllers v1.0.1 -ember-export-application-global v1.0.5 -ember-load-initializers v0.1.7 -ember-qunit v0.4.16 -ember-qunit-notifications v0.1.0 -ember-resolver v2.0.3 -ember-spin-spinner v0.2.3 -ember-truth-helpers v1.2.0 -jquery v2.1.4 -jquery-ui v1.11.4 -loader.js v3.3.0 -momentjs v2.10.6 -qunit v1.19.0 -select2 v4.0.0 -snippet-ss v1.11.0 -spin.js v2.3.2 -Azure Data Lake Store - Java client SDK 2.0.11 -JCodings 1.0.8 -Joni 2.1.2 -Mockito 1.8.5 -JUL to SLF4J bridge 1.7.25 -SLF4J API Module 1.7.25 -SLF4J LOG4J-12 Binding 1.7.25 --------------------------------------------------------------------------------- - -The MIT License (MIT) - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery -Apache HBase - Server which contains JQuery minified javascript library version 1.8.3 -Microsoft JDBC Driver for SQLServer - version 6.2.1.jre7 --------------------------------------------------------------------------------- - -MIT License - -Copyright (c) 2003-2017 Optimatika - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -For: -oj! Algorithms - version 43.0 --------------------------------------------------------------------------------- - -Copyright 2005, 2012, 2013 jQuery Foundation and other contributors, https://jquery.org/ - -This software consists of voluntary contributions made by many -individuals. For exact contribution history, see the revision history -available at https://github.com/jquery/jquery - -The following license applies to all parts of this software except as -documented below: - -==== - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -All files located in the node_modules and external directories are -externally maintained libraries used by this software which have their -own licenses; we recommend you read them, as their terms may differ from -the terms above. - -For: -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js --------------------------------------------------------------------------------- - -Copyright (c) 2014 Ivan Bozhanov - -For: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js --------------------------------------------------------------------------------- - -D3 is available under a 3-clause BSD license. For details, see: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE - -The binary distribution of this product bundles these dependencies under the -following license: -HSQLDB Database 2.3.4 --------------------------------------------------------------------------------- -(HSQL License) -"COPYRIGHTS AND LICENSES (based on BSD License) - -For work developed by the HSQL Development Group: - -Copyright (c) 2001-2016, The HSQL Development Group -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -Neither the name of the HSQL Development Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -For work originally developed by the Hypersonic SQL Group: - -Copyright (c) 1995-2000 by the Hypersonic SQL Group. -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Neither the name of the Hypersonic SQL Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -This software consists of voluntary contributions made by many individuals on behalf of the -Hypersonic SQL Group." - -The binary distribution of this product bundles these dependencies under the -following license: -Java Servlet API 3.1.0 -servlet-api 2.5 -jsp-api 2.1 -jsr311-api 1.1.1 -Glassfish Jasper 6.1.14 -Servlet Specification 2.5 API 6.1.14 --------------------------------------------------------------------------------- -(CDDL 1.0) -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 -1. Definitions.  - -1.1. Contributor means each individual or entity -that creates or contributes to the creation of -Modifications.  - -1.2. Contributor Version means the combination of the -Original Software, prior Modifications used by a Contributor (if any), and the -Modifications made by that particular Contributor.  - -1.3. Covered -Software means (a) the Original Software, or (b) Modifications, or (c) the -combination of files containing Original Software with files containing -Modifications, in each case including portions -thereof.  - -1.4. Executable means the Covered Software in any form other -than Source Code.  - -1.5. Initial Developer means the individual or entity -that first makes Original Software available under this -License.  - -1.6. Larger Work means a work which combines Covered Software or -portions thereof with code not governed by the terms of this -License.  - -1.7. License means this document.  - -1.8. Licensable means -having the right to grant, to the maximum extent possible, whether at the time -of the initial grant or subsequently acquired, any and all of the rights -conveyed herein.  - -1.9. Modifications means the Source Code and Executable -form of any of the following: -A. Any file that results from an addition to, -deletion from or modification of the contents of a file containing Original -Software or previous Modifications; -B. Any new file that contains any part of the Original Software -or previous Modification; or -C. Any new file that is contributed or otherwise made available -under the terms of this License.  - -1.10. Original Software means the Source Code and Executable form of -computer software code that is originally released under this License.  - -1.11. Patent Claims means any patent claim(s), now owned or -hereafter acquired, including without limitation, method, process, and apparatus -claims, in any patent Licensable by grantor.  - -1.12. Source Code means (a) the common form of computer software code in which -modifications are made and (b) associated documentation included in or -with such code.  - -1.13. You (or Your) means an individual or a legal entity exercising rights -under, and complying with all of the terms of, this License. For legal entities, -You includes any entity which controls, is controlled by, or is under common control -with You. For purposes of this definition, control means (a) the power, direct -or indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (b) ownership of more than fifty percent (50%) of the -outstanding shares or beneficial ownership of such entity.  - -2. License Grants. - -2.1. The Initial Developer Grant. Conditioned upon Your compliance -with Section 3.1 below and subject to third party intellectual property claims, -the Initial Developer hereby grants You a world-wide, royalty-free, -non-exclusive license:  - -(a) under intellectual property rights (other than -patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, -display, perform, sublicense and distribute the Original Software (or portions -thereof), with or without Modifications, and/or as part of a Larger Work; -and  - -(b) under Patent Claims infringed by the making, using or selling of -Original Software, to make, have made, use, practice, sell, and offer for sale, -and/or otherwise dispose of the Original Software (or portions -thereof); - -(c) The licenses granted in Sections 2.1(a) and (b) are -effective on the date Initial Developer first distributes or otherwise makes the -Original Software available to a third party under the terms of this -License; - -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or -(ii) the combination of the Original Software with other software or -devices.  - -2.2. Contributor Grant. Conditioned upon Your compliance with -Section 3.1 below and subject to third party intellectual property claims, each -Contributor hereby grants You a world-wide, royalty-free, non-exclusive -license:  - -(a) under intellectual property rights (other than patent or -trademark) Licensable by Contributor to use, reproduce, modify, display, -perform, sublicense and distribute the Modifications created by such Contributor -(or portions thereof), either on an unmodified basis, with other Modifications, -as Covered Software and/or as part of a Larger Work; and  - -(b) under Patent -Claims infringed by the making, using, or selling of Modifications made by that -Contributor either alone and/or in combination with its Contributor Version (or -portions of such combination), to make, use, sell, offer for sale, have made, -and/or otherwise dispose of: (1) Modifications made by that Contributor (or -portions thereof); and (2) the combination of Modifications made by that -Contributor with its Contributor Version (or portions of such -combination).  - -(c) The licenses granted in Sections 2.2(a) and 2.2(b) are -effective on the date Contributor first distributes or otherwise makes the -Modifications available to a third party. - -(d) Notwithstanding Section 2.2(b) -above, no patent license is granted: (1) for any code that Contributor has -deleted from the Contributor Version; (2) for infringements caused by: -(i) third party modifications of Contributor Version, or (ii) the combination -of Modifications made by that Contributor with other software (except as part of -the Contributor Version) or other devices; or (3) under Patent Claims infringed -by Covered Software in the absence of Modifications made by that -Contributor.  - -3. Distribution Obligations.  - -3.1. Availability of Source -Code. Any Covered Software that You distribute or otherwise make available in -Executable form must also be made available in Source Code form and that Source -Code form must be distributed only under the terms of this License. You must -include a copy of this License with every copy of the Source Code form of the -Covered Software You distribute or otherwise make available. You must inform -recipients of any such Covered Software in Executable form as to how they can -obtain such Covered Software in Source Code form in a reasonable manner on or -through a medium customarily used for software exchange.  - -3.2. -Modifications. The Modifications that You create or to which You contribute are -governed by the terms of this License. You represent that You believe Your -Modifications are Your original creation(s) and/or You have sufficient rights to -grant the rights conveyed by this License.  - -3.3. Required Notices. You must -include a notice in each of Your Modifications that identifies You as the -Contributor of the Modification. You may not remove or alter any copyright, -patent or trademark notices contained within the Covered Software, or any -notices of licensing or any descriptive text giving attribution to any -Contributor or the Initial Developer.  - -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source Code -form that alters or restricts the applicable version of this License or the -recipients rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. - -3.5. Distribution of Executable Versions. You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipients rights in the Source Code form from the rights set forth in this -License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer.  - -3.6. Larger Works. You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software.  - -4. Versions of the License.  - -4.1. -New Versions. Sun Microsystems, Inc. is the initial license steward and may -publish revised and/or new versions of this License from time to time. Each -version will be given a distinguishing version number. Except as provided in -Section 4.3, no one other than the license steward has the right to modify this -License.  - -4.2. Effect of New Versions. You may always continue to use, -distribute or otherwise make the Covered Software available under the terms of -the version of the License under which You originally received the Covered -Software. If the Initial Developer includes a notice in the Original Software -prohibiting it from being distributed or otherwise made available under any -subsequent version of the License, You must distribute and make the Covered -Software available under the terms of the version of the License under which You -originally received the Covered Software. Otherwise, You may also choose to use, -distribute or otherwise make the Covered Software available under the terms of -any subsequent version of the License published by the license -steward.  - -4.3. Modified Versions. When You are an Initial Developer and You -want to create a new license for Your Original Software, You may create and use -a modified version of this License if You: (a) rename the license and remove -any references to the name of the license steward (except to note that the -license differs from this License); and (b) otherwise make it clear that the -license contains terms which differ from this License.  - -5. DISCLAIMER OF WARRANTY. - -COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, -WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT -LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, -MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY -COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER -OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR -CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS -LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER -THIS DISCLAIMER.  - -6. TERMINATION.  - -6.1. This License and the rights -granted hereunder will terminate automatically if You fail to comply with terms -herein and fail to cure such breach within 30 days of becoming aware of the -breach. Provisions which, by their nature, must remain in effect beyond the -termination of this License shall survive.  - -6.2. If You assert a patent -infringement claim (excluding declaratory judgment actions) against Initial -Developer or a Contributor (the Initial Developer or Contributor against whom -You assert such claim is referred to as Participant) alleging that the -Participant Software (meaning the Contributor Version where the Participant is a -Contributor or the Original Software where the Participant is the Initial -Developer) directly or indirectly infringes any patent, then any and all rights -granted directly or indirectly to You by such Participant, the Initial Developer -(if the Initial Developer is not the Participant) and all Contributors under -Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from -Participant terminate prospectively and automatically at the expiration of such -60 day notice period, unless if within such 60 day period You withdraw Your -claim with respect to the Participant Software against such Participant either -unilaterally or pursuant to a written agreement with Participant.  - -6.3. In -the event of termination under Sections 6.1 or 6.2 above, all end user licenses -that have been validly granted by You or any distributor hereunder prior to -termination (excluding licenses granted to You by any distributor) shall survive -termination.  - -7. LIMITATION OF LIABILITY. -UNDER NO CIRCUMSTANCES AND UNDER -NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, -SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF -COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY -PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY -CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF -GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER -COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE -POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO -LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO -THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT -ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO -THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  - -8. U.S. GOVERNMENT END USERS. - -The Covered Software is a commercial item, as that term is defined in -48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as -that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer -software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. -1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through -227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software -with only those rights set forth herein. This U.S. Government Rights clause is -in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision -that addresses Government rights in computer software under this -License.  - -9. MISCELLANEOUS. -This License represents the complete agreement -concerning subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdictions conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software.  - -10. RESPONSIBILITY FOR CLAIMS. -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability.  - -The binary distribution of this product bundles these dependencies under the -following license: -jersey-client 1.19 -jersey-core 1.19 -jersey-grizzly2 1.19 -jersey-grizzly2-servlet 1.19 -jersey-json 1.19 -jersey-server 1.19 -jersey-servlet 1.19 -jersey-guice 1.19 -Jersey Test Framework - Grizzly 2 Module 1.19 -JAXB RI 2.2.3 -Java Architecture for XML Binding 2.2.11 -grizzly-framework 2.2.21 -grizzly-http 2.2.21 -grizzly-http-server 2.2.21 -grizzly-http-servlet 2.2.21 -grizzly-rcm 2.2.21 --------------------------------------------------------------------------------- -(CDDL 1.1) -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 - -1. Definitions. - -1.1. “Contributor” means each individual or entity that creates or -contributes to the creation of Modifications. -1.2. “Contributor Version” means the combination of the Original Software, -prior Modifications used by a Contributor (if any), and the Modifications made -by that particular Contributor. -1.3. “Covered Software” means (a) the Original Software, or (b) -Modifications, or (c) the combination of files containing Original Software with -files containing Modifications, in each case including portions thereof. -1.4. “Executable” means the Covered Software in any form other than Source -Code. -1.5. “Initial Developer” means the individual or entity that first makes -Original Software available under this License. -1.6. “Larger Work” means a work which combines Covered Software or portions -thereof with code not governed by the terms of this License. -1.7. “License” means this document. -1.8. “Licensable” means having the right to grant, to the maximum extent -possible, whether at the time of the initial grant or subsequently acquired, any -and all of the rights conveyed herein. -1.9. “Modifications” means the Source Code and Executable form of any of the -following: -A. Any file that results from an addition to, deletion from or modification of -the contents of a file containing Original Software or previous Modifications; -B. Any new file that contains any part of the Original Software or previous -Modification; or -C. Any new file that is contributed or otherwise made available under the terms -of this License. -1.10. “Original Software” means the Source Code and Executable form of -computer software code that is originally released under this License. -1.11. “Patent Claims” means any patent claim(s), now owned or hereafter -acquired, including without limitation, method, process, and apparatus claims, -in any patent Licensable by grantor. -1.12. “Source Code” means (a) the common form of computer software code in -which modifications are made and (b) associated documentation included in or -with such code. -1.13. “You” (or “Your”) means an individual or a legal entity exercising -rights under, and complying with all of the terms of, this License. For legal -entities, “You” includes any entity which controls, is controlled by, or is -under common control with You. For purposes of this definition, “control” -means (a) the power, direct or indirect, to cause the direction or management of -such entity, whether by contract or otherwise, or (b) ownership of more than -fifty percent (50%) of the outstanding shares or beneficial ownership of such -entity. - -2.1. The Initial Developer Grant. - -Conditioned upon Your compliance with Section 3.1 below and subject to -third party intellectual property claims, the Initial Developer hereby grants -You a world-wide, royalty-free, non-exclusive license: -(a) under intellectual -property rights (other than patent or trademark) Licensable by Initial -Developer, to use, reproduce, modify, display, perform, sublicense and -distribute the Original Software (or portions thereof), with or without -Modifications, and/or as part of a Larger Work; and -(b) under Patent Claims -infringed by the making, using or selling of Original Software, to make, have -made, use, practice, sell, and offer for sale, and/or otherwise dispose of the -Original Software (or portions thereof). -(c) The licenses granted in Sections -2.1(a) and (b) are effective on the date Initial Developer first distributes or -otherwise makes the Original Software available to a third party under the terms -of this License. -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or (ii) -the combination of the Original Software with other software or devices. - -2.2. Contributor Grant. - -Conditioned upon Your compliance with Section 3.1 below and -subject to third party intellectual property claims, each Contributor hereby -grants You a world-wide, royalty-free, non-exclusive license: -(a) under -intellectual property rights (other than patent or trademark) Licensable by -Contributor to use, reproduce, modify, display, perform, sublicense and -distribute the Modifications created by such Contributor (or portions thereof), -either on an unmodified basis, with other Modifications, as Covered Software -and/or as part of a Larger Work; and -(b) under Patent Claims infringed by the -making, using, or selling of Modifications made by that Contributor either alone -and/or in combination with its Contributor Version (or portions of such -combination), to make, use, sell, offer for sale, have made, and/or otherwise -dispose of: (1) Modifications made by that Contributor (or portions thereof); -and (2) the combination of Modifications made by that Contributor with its -Contributor Version (or portions of such combination). -(c) The licenses granted -in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first -distributes or otherwise makes the Modifications available to a third -party. -(d) Notwithstanding Section 2.2(b) above, no patent license is granted: -(1) for any code that Contributor has deleted from the Contributor Version; (2) -for infringements caused by: (i) third party modifications of Contributor -Version, or (ii) the combination of Modifications made by that Contributor with -other software (except as part of the Contributor Version) or other devices; or -(3) under Patent Claims infringed by Covered Software in the absence of -Modifications made by that Contributor. - -3. Distribution Obligations. - -3.1. Availability of Source Code. -Any Covered Software that You distribute or -otherwise make available in Executable form must also be made available in -Source Code form and that Source Code form must be distributed only under the -terms of this License. You must include a copy of this License with every copy -of the Source Code form of the Covered Software You distribute or otherwise make -available. You must inform recipients of any such Covered Software in Executable -form as to how they can obtain such Covered Software in Source Code form in a -reasonable manner on or through a medium customarily used for software -exchange. -3.2. Modifications. -The Modifications that You create or to which -You contribute are governed by the terms of this License. You represent that You -believe Your Modifications are Your original creation(s) and/or You have -sufficient rights to grant the rights conveyed by this License. -3.3. Required Notices. -You must include a notice in each of Your Modifications that -identifies You as the Contributor of the Modification. You may not remove or -alter any copyright, patent or trademark notices contained within the Covered -Software, or any notices of licensing or any descriptive text giving attribution -to any Contributor or the Initial Developer. -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source -Code form that alters or restricts the applicable version of this License or the -recipients' rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. -3.5. Distribution of Executable Versions. -You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipient's rights in the Source Code form from the rights set forth in -this License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer. -3.6. Larger Works. -You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software. - -4. Versions of the License. - -4.1. New Versions. -Oracle is the initial license steward and may publish revised and/or -new versions of this License from time to time. Each version will be given a -distinguishing version number. Except as provided in Section 4.3, no one other -than the license steward has the right to modify this License. -4.2. Effect of New Versions. -You may always continue to use, distribute or otherwise make the -Covered Software available under the terms of the version of the License under -which You originally received the Covered Software. If the Initial Developer -includes a notice in the Original Software prohibiting it from being distributed -or otherwise made available under any subsequent version of the License, You -must distribute and make the Covered Software available under the terms of the -version of the License under which You originally received the Covered Software. -Otherwise, You may also choose to use, distribute or otherwise make the Covered -Software available under the terms of any subsequent version of the License -published by the license steward. -4.3. Modified Versions. -When You are an -Initial Developer and You want to create a new license for Your Original -Software, You may create and use a modified version of this License if You: (a) -rename the license and remove any references to the name of the license steward -(except to note that the license differs from this License); and (b) otherwise -make it clear that the license contains terms which differ from this -License. - -COVERED SOFTWARE IS PROVIDED UNDER THIS -LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE -IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR -NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED -SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY -RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE -COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF -WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED -SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - -6.1. This License and the rights granted hereunder will -terminate automatically if You fail to comply with terms herein and fail to cure -such breach within 30 days of becoming aware of the breach. Provisions which, by -their nature, must remain in effect beyond the termination of this License shall -survive. -6.2. If You assert a patent infringement claim (excluding declaratory -judgment actions) against Initial Developer or a Contributor (the Initial -Developer or Contributor against whom You assert such claim is referred to as -“Participant”) alleging that the Participant Software (meaning the -Contributor Version where the Participant is a Contributor or the Original -Software where the Participant is the Initial Developer) directly or indirectly -infringes any patent, then any and all rights granted directly or indirectly to -You by such Participant, the Initial Developer (if the Initial Developer is not -the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this -License shall, upon 60 days notice from Participant terminate prospectively and -automatically at the expiration of such 60 day notice period, unless if within -such 60 day period You withdraw Your claim with respect to the Participant -Software against such Participant either unilaterally or pursuant to a written -agreement with Participant. -6.3. If You assert a patent infringement claim -against Participant alleging that the Participant Software directly or -indirectly infringes any patent where such claim is resolved (such as by license -or settlement) prior to the initiation of patent infringement litigation, then -the reasonable value of the licenses granted by such Participant under Sections -2.1 or 2.2 shall be taken into account in determining the amount or value of any -payment or license. -6.4. In the event of termination under Sections 6.1 or 6.2 -above, all end user licenses that have been validly granted by You or any -distributor hereunder prior to termination (excluding licenses granted to You by -any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. - -UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT -(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL -DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY -SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, -SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, -WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER -FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN -IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS -LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL -INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW -PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR -LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND -LIMITATION MAY NOT APPLY TO YOU. - -The Covered -Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 -(Oct. 1995), consisting of “commercial computer software” (as that term is -defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software -documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). -Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 -(June 1995), all U.S. Government End Users acquire Covered Software with only -those rights set forth herein. This U.S. Government Rights clause is in lieu of, -and supersedes, any other FAR, DFAR, or other clause or provision that addresses -Government rights in computer software under this License. - -9. MISCELLANEOUS. - -This License represents the complete agreement concerning -subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdiction's conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys' fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. - -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability. - -The binary distribution of this product bundles these dependencies under the -following license: -Protocol Buffer Java API 2.5.0 --------------------------------------------------------------------------------- -This license applies to all parts of Protocol Buffers except the following: - - - Atomicops support for generic gcc, located in - src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. - This file is copyrighted by Red Hat Inc. - - - Atomicops support for AIX/POWER, located in - src/google/protobuf/stubs/atomicops_internals_power.h. - This file is copyrighted by Bloomberg Finance LP. - -Copyright 2014, Google Inc. All rights reserved. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - -For: -XML Commons External Components XML APIs 1.3.04 --------------------------------------------------------------------------------- -By obtaining, using and/or copying this work, you (the licensee) agree that you -have read, understood, and will comply with the following terms and conditions. - -Permission to copy, modify, and distribute this software and its documentation, -with or without modification, for any purpose and without fee or royalty is -hereby granted, provided that you include the following on ALL copies of the -software and documentation or portions thereof, including modifications: -- The full text of this NOTICE in a location viewable to users of the -redistributed or derivative work. -- Any pre-existing intellectual property disclaimers, notices, or terms and -conditions. If none exist, the W3C Software Short Notice should be included -(hypertext is preferred, text is permitted) within the body of any redistributed -or derivative code. -- Notice of any changes or modifications to the files, including the date changes -were made. (We recommend you provide URIs to the location from which the code is -derived.) - -The binary distribution of this product bundles these dependencies under the -following license: -JUnit 4.11 -Eclipse JDT Core 3.1.1 --------------------------------------------------------------------------------- -(EPL v1.0) -Eclipse Public License - v 1.0 - -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC -LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM -CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: - -a) in the case of the initial Contributor, the initial code and documentation -distributed under this Agreement, and -b) in the case of each subsequent Contributor: -i) changes to the Program, and -ii) additions to the Program; -where such changes and/or additions to the Program originate from and are -distributed by that particular Contributor. A Contribution 'originates' from a -Contributor if it was added to the Program by such Contributor itself or anyone -acting on such Contributor's behalf. Contributions do not include additions to -the Program which: (i) are separate modules of software distributed in -conjunction with the Program under their own license agreement, and (ii) are not -derivative works of the Program. -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents" mean patent claims licensable by a Contributor which are -necessarily infringed by the use or sale of its Contribution alone or when -combined with the Program. - -"Program" means the Contributions distributed in accordance with this Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, -including all Contributors. - -2. GRANT OF RIGHTS - -a) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free copyright license to -reproduce, prepare derivative works of, publicly display, publicly perform, -distribute and sublicense the Contribution of such Contributor, if any, and such -derivative works, in source code and object code form. -b) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed -Patents to make, use, sell, offer to sell, import and otherwise transfer the -Contribution of such Contributor, if any, in source code and object code form. -This patent license shall apply to the combination of the Contribution and the -Program if, at the time the Contribution is added by the Contributor, such -addition of the Contribution causes such combination to be covered by the -Licensed Patents. The patent license shall not apply to any other combinations -which include the Contribution. No hardware per se is licensed hereunder. -c) Recipient understands that although each Contributor grants the licenses to -its Contributions set forth herein, no assurances are provided by any -Contributor that the Program does not infringe the patent or other intellectual -property rights of any other entity. Each Contributor disclaims any liability to -Recipient for claims brought by any other entity based on infringement of -intellectual property rights or otherwise. As a condition to exercising the -rights and licenses granted hereunder, each Recipient hereby assumes sole -responsibility to secure any other intellectual property rights needed, if any. -For example, if a third party patent license is required to allow Recipient to -distribute the Program, it is Recipient's responsibility to acquire that license -before distributing the Program. -d) Each Contributor represents that to its knowledge it has sufficient copyright -rights in its Contribution, if any, to grant the copyright license set forth in -this Agreement. -3. REQUIREMENTS - -A Contributor may choose to distribute the Program in object code form under its -own license agreement, provided that: - -a) it complies with the terms and conditions of this Agreement; and -b) its license agreement: -i) effectively disclaims on behalf of all Contributors all warranties and -conditions, express and implied, including warranties or conditions of title and -non-infringement, and implied warranties or conditions of merchantability and -fitness for a particular purpose; -ii) effectively excludes on behalf of all Contributors all liability for -damages, including direct, indirect, special, incidental and consequential -damages, such as lost profits; -iii) states that any provisions which differ from this Agreement are offered by -that Contributor alone and not by any other party; and -iv) states that source code for the Program is available from such Contributor, -and informs licensees how to obtain it in a reasonable manner on or through a -medium customarily used for software exchange. -When the Program is made available in source code form: - -a) it must be made available under this Agreement; and -b) a copy of this Agreement must be included with each copy of the Program. -Contributors may not remove or alter any copyright notices contained within the -Program. - -Each Contributor must identify itself as the originator of its Contribution, if -any, in a manner that reasonably allows subsequent Recipients to identify the -originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION - -Commercial distributors of software may accept certain responsibilities with -respect to end users, business partners and the like. While this license is -intended to facilitate the commercial use of the Program, the Contributor who -includes the Program in a commercial product offering should do so in a manner -which does not create potential liability for other Contributors. Therefore, if -a Contributor includes the Program in a commercial product offering, such -Contributor ("Commercial Contributor") hereby agrees to defend and indemnify -every other Contributor ("Indemnified Contributor") against any losses, damages -and costs (collectively "Losses") arising from claims, lawsuits and other legal -actions brought by a third party against the Indemnified Contributor to the -extent caused by the acts or omissions of such Commercial Contributor in -connection with its distribution of the Program in a commercial product -offering. The obligations in this section do not apply to any claims or Losses -relating to any actual or alleged intellectual property infringement. In order -to qualify, an Indemnified Contributor must: a) promptly notify the Commercial -Contributor in writing of such claim, and b) allow the Commercial Contributor to -control, and cooperate with the Commercial Contributor in, the defense and any -related settlement negotiations. The Indemnified Contributor may participate in -any such claim at its own expense. - -For example, a Contributor might include the Program in a commercial product -offering, Product X. That Contributor is then a Commercial Contributor. If that -Commercial Contributor then makes performance claims, or offers warranties -related to Product X, those performance claims and warranties are such -Commercial Contributor's responsibility alone. Under this section, the -Commercial Contributor would have to defend claims against the other -Contributors related to those performance claims and warranties, and if a court -requires any other Contributor to pay any damages as a result, the Commercial -Contributor must pay those damages. - -5. NO WARRANTY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR -IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, -NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each -Recipient is solely responsible for determining the appropriateness of using and -distributing the Program and assumes all risks associated with its exercise of -rights under this Agreement , including but not limited to the risks and costs -of program errors, compliance with applicable laws, damage to or loss of data, -programs or equipment, and unavailability or interruption of operations. - -6. DISCLAIMER OF LIABILITY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY -CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST -PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS -GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this Agreement, and without further action by the parties hereto, such -provision shall be reformed to the minimum extent necessary to make such -provision valid and enforceable. - -If Recipient institutes patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Program itself -(excluding combinations of the Program with other software or hardware) -infringes such Recipient's patent(s), then such Recipient's rights granted under -Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to -comply with any of the material terms or conditions of this Agreement and does -not cure such failure in a reasonable period of time after becoming aware of -such noncompliance. If all Recipient's rights under this Agreement terminate, -Recipient agrees to cease use and distribution of the Program as soon as -reasonably practicable. However, Recipient's obligations under this Agreement -and any licenses granted by Recipient relating to the Program shall continue and -survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in -order to avoid inconsistency the Agreement is copyrighted and may only be -modified in the following manner. The Agreement Steward reserves the right to -publish new versions (including revisions) of this Agreement from time to time. -No one other than the Agreement Steward has the right to modify this Agreement. -The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation -may assign the responsibility to serve as the Agreement Steward to a suitable -separate entity. Each new version of the Agreement will be given a -distinguishing version number. The Program (including Contributions) may always -be distributed subject to the version of the Agreement under which it was -received. In addition, after a new version of the Agreement is published, -Contributor may elect to distribute the Program (including its Contributions) -under the new version. Except as expressly stated in Sections 2(a) and 2(b) -above, Recipient receives no rights or licenses to the intellectual property of -any Contributor under this Agreement, whether expressly, by implication, -estoppel or otherwise. All rights in the Program not expressly granted under -this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the -intellectual property laws of the United States of America. No party to this -Agreement will bring a legal action under this Agreement more than one year -after the cause of action arose. Each party waives its rights to a jury trial in -any resulting litigation. - -The binary distribution of this product bundles these dependencies under the -following license: -JSch 0.1.51 -ParaNamer Core 2.3 -JLine 0.9.94 -leveldbjni-all 1.8 -Hamcrest Core 1.3 -ASM Core 5.0.4 -ASM Commons 5.0.2 -ASM Tree 5.0.2 --------------------------------------------------------------------------------- -(3-clause BSD) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles these dependencies under the -following license: -FindBugs-jsr305 3.0.0 -dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved. --------------------------------------------------------------------------------- -(2-clause BSD) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those -of the authors and should not be interpreted as representing official policies, -either expressed or implied, of the FreeBSD Project. - -The binary distribution of this product bundles these dependencies under the -following license: -"Java Concurrency in Practice" book annotations 1.0 --------------------------------------------------------------------------------- -(CCAL v2.5) -THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS -PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR -OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS -LICENSE OR COPYRIGHT LAW IS PROHIBITED. - -BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE -BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED -HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. - -1. Definitions - -"Collective Work" means a work, such as a periodical issue, anthology or -encyclopedia, in which the Work in its entirety in unmodified form, along with a -number of other contributions, constituting separate and independent works in -themselves, are assembled into a collective whole. A work that constitutes a -Collective Work will not be considered a Derivative Work (as defined below) for -the purposes of this License. -"Derivative Work" means a work based upon the Work or upon the Work and other -pre-existing works, such as a translation, musical arrangement, dramatization, -fictionalization, motion picture version, sound recording, art reproduction, -abridgment, condensation, or any other form in which the Work may be recast, -transformed, or adapted, except that a work that constitutes a Collective Work -will not be considered a Derivative Work for the purpose of this License. For -the avoidance of doubt, where the Work is a musical composition or sound -recording, the synchronization of the Work in timed-relation with a moving image -("synching") will be considered a Derivative Work for the purpose of this -License. -"Licensor" means the individual or entity that offers the Work under the terms -of this License. -"Original Author" means the individual or entity who created the Work. -"Work" means the copyrightable work of authorship offered under the terms of -this License. -"You" means an individual or entity exercising rights under this License who has -not previously violated the terms of this License with respect to the Work, or -who has received express permission from the Licensor to exercise rights under -this License despite a previous violation. -2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or -restrict any rights arising from fair use, first sale or other limitations on -the exclusive rights of the copyright owner under copyright law or other -applicable laws. - -3. License Grant. Subject to the terms and conditions of this License, Licensor -hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the -duration of the applicable copyright) license to exercise the rights in the Work -as stated below: - -to reproduce the Work, to incorporate the Work into one or more Collective -Works, and to reproduce the Work as incorporated in the Collective Works; -to create and reproduce Derivative Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission the Work including as -incorporated in Collective Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission Derivative Works. -For the avoidance of doubt, where the work is a musical composition: - -Performance Royalties Under Blanket Licenses. Licensor waives the exclusive -right to collect, whether individually or via a performance rights society (e.g. -ASCAP, BMI, SESAC), royalties for the public performance or public digital -performance (e.g. webcast) of the Work. -Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right -to collect, whether individually or via a music rights agency or designated -agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the -Work ("cover version") and distribute, subject to the compulsory license created -by 17 USC Section 115 of the US Copyright Act (or the equivalent in other -jurisdictions). -Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the -Work is a sound recording, Licensor waives the exclusive right to collect, -whether individually or via a performance-rights society (e.g. SoundExchange), -royalties for the public digital performance (e.g. webcast) of the Work, subject -to the compulsory license created by 17 USC Section 114 of the US Copyright Act -(or the equivalent in other jurisdictions). -The above rights may be exercised in all media and formats whether now known or -hereafter devised. The above rights include the right to make such modifications -as are technically necessary to exercise the rights in other media and formats. -All rights not expressly granted by Licensor are hereby reserved. - -4. Restrictions.The license granted in Section 3 above is expressly made subject -to and limited by the following restrictions: - -You may distribute, publicly display, publicly perform, or publicly digitally -perform the Work only under the terms of this License, and You must include a -copy of, or the Uniform Resource Identifier for, this License with every copy or -phonorecord of the Work You distribute, publicly display, publicly perform, or -publicly digitally perform. You may not offer or impose any terms on the Work -that alter or restrict the terms of this License or the recipients' exercise of -the rights granted hereunder. You may not sublicense the Work. You must keep -intact all notices that refer to this License and to the disclaimer of -warranties. You may not distribute, publicly display, publicly perform, or -publicly digitally perform the Work with any technological measures that control -access or use of the Work in a manner inconsistent with the terms of this -License Agreement. The above applies to the Work as incorporated in a Collective -Work, but this does not require the Collective Work apart from the Work itself -to be made subject to the terms of this License. If You create a Collective -Work, upon notice from any Licensor You must, to the extent practicable, remove -from the Collective Work any credit as required by clause 4(b), as requested. If -You create a Derivative Work, upon notice from any Licensor You must, to the -extent practicable, remove from the Derivative Work any credit as required by -clause 4(b), as requested. -If you distribute, publicly display, publicly perform, or publicly digitally -perform the Work or any Derivative Works or Collective Works, You must keep -intact all copyright notices for the Work and provide, reasonable to the medium -or means You are utilizing: (i) the name of the Original Author (or pseudonym, -if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor -designate another party or parties (e.g. a sponsor institute, publishing entity, -journal) for attribution in Licensor's copyright notice, terms of service or by -other reasonable means, the name of such party or parties; the title of the Work -if supplied; to the extent reasonably practicable, the Uniform Resource -Identifier, if any, that Licensor specifies to be associated with the Work, -unless such URI does not refer to the copyright notice or licensing information -for the Work; and in the case of a Derivative Work, a credit identifying the use -of the Work in the Derivative Work (e.g., "French translation of the Work by -Original Author," or "Screenplay based on original Work by Original Author"). -Such credit may be implemented in any reasonable manner; provided, however, that -in the case of a Derivative Work or Collective Work, at a minimum such credit -will appear where any other comparable authorship credit appears and in a manner -at least as prominent as such other comparable authorship credit. -5. Representations, Warranties and Disclaimer - -UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS -THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING -THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT -LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR -PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, -OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME -JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH -EXCLUSION MAY NOT APPLY TO YOU. - -6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN -NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, -INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS -LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - -7. Termination - -This License and the rights granted hereunder will terminate automatically upon -any breach by You of the terms of this License. Individuals or entities who have -received Derivative Works or Collective Works from You under this License, -however, will not have their licenses terminated provided such individuals or -entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, -and 8 will survive any termination of this License. -Subject to the above terms and conditions, the license granted here is perpetual -(for the duration of the applicable copyright in the Work). Notwithstanding the -above, Licensor reserves the right to release the Work under different license -terms or to stop distributing the Work at any time; provided, however that any -such election will not serve to withdraw this License (or any other license that -has been, or is required to be, granted under the terms of this License), and -this License will continue in full force and effect unless terminated as stated -above. -8. Miscellaneous - -Each time You distribute or publicly digitally perform the Work or a Collective -Work, the Licensor offers to the recipient a license to the Work on the same -terms and conditions as the license granted to You under this License. -Each time You distribute or publicly digitally perform a Derivative Work, -Licensor offers to the recipient a license to the original Work on the same -terms and conditions as the license granted to You under this License. -If any provision of this License is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this License, and without further action by the parties to this -agreement, such provision shall be reformed to the minimum extent necessary to -make such provision valid and enforceable. -No term or provision of this License shall be deemed waived and no breach -consented to unless such waiver or consent shall be in writing and signed by the -party to be charged with such waiver or consent. -This License constitutes the entire agreement between the parties with respect -to the Work licensed here. There are no understandings, agreements or -representations with respect to the Work not specified here. Licensor shall not -be bound by any additional provisions that may appear in any communication from -You. This License may not be modified without the mutual written agreement of -the Licensor and You. - -The binary distribution of this product bundles these dependencies under the -following license: -jamon-runtime 2.4.1 --------------------------------------------------------------------------------- -(MPL 2.0) - Mozilla Public License - Version 2.0 - -1.1. “Contributor” -means each individual or legal entity that creates, contributes to the creation -of, or owns Covered Software. - -1.2. “Contributor Version” -means the combination of the Contributions of others (if any) used by a -Contributor and that particular Contributor’s Contribution. - -1.3. “Contribution” -means Covered Software of a particular Contributor. - -1.4. “Covered Software” -means Source Code Form to which the initial Contributor has attached the notice -in Exhibit A, the Executable Form of such Source Code Form, and Modifications of -such Source Code Form, in each case including portions thereof. - -1.5. “Incompatible With Secondary Licenses” -means - -that the initial Contributor has attached the notice described in Exhibit B to -the Covered Software; or - -that the Covered Software was made available under the terms of version 1.1 or -earlier of the License, but not also under the terms of a Secondary License. - -1.6. “Executable Form” -means any form of the work other than Source Code Form. - -1.7. “Larger Work” -means a work that combines Covered Software with other material, in a separate -file or files, that is not Covered Software. - -1.8. “License” -means this document. - -1.9. “Licensable” -means having the right to grant, to the maximum extent possible, whether at the -time of the initial grant or subsequently, any and all of the rights conveyed by -this License. - -1.10. “Modifications” -means any of the following: - -any file in Source Code Form that results from an addition to, deletion from, or -modification of the contents of Covered Software; or - -any new file in Source Code Form that contains any Covered Software. - -1.11. “Patent Claims” of a Contributor -means any patent claim(s), including without limitation, method, process, and -apparatus claims, in any patent Licensable by such Contributor that would be -infringed, but for the grant of the License, by the making, using, selling, -offering for sale, having made, import, or transfer of either its Contributions -or its Contributor Version. - -1.12. “Secondary License” -means either the GNU General Public License, Version 2.0, the GNU Lesser General -Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, -or any later versions of those licenses. - -1.13. “Source Code Form” -means the form of the work preferred for making modifications. - -1.14. “You” (or “Your”) -means an individual or a legal entity exercising rights under this License. For -legal entities, “You” includes any entity that controls, is controlled by, -or is under common control with You. For purposes of this definition, -“control” means (a) the power, direct or indirect, to cause the direction or -management of such entity, whether by contract or otherwise, or (b) ownership of -more than fifty percent (50%) of the outstanding shares or beneficial ownership -of such entity. - -2. License Grants and Conditions - -2.1. Grants - -Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive -license: - -under intellectual property rights (other than patent or trademark) Licensable -by such Contributor to use, reproduce, make available, modify, display, perform, -distribute, and otherwise exploit its Contributions, either on an unmodified -basis, with Modifications, or as part of a Larger Work; and - -under Patent Claims of such Contributor to make, use, sell, offer for sale, have -made, import, and otherwise transfer either its Contributions or its Contributor -Version. - -2.2. Effective Date - -The licenses granted in Section 2.1 with respect to any Contribution become -effective for each Contribution on the date the Contributor first distributes -such Contribution. - -2.3. Limitations on Grant Scope - -The licenses granted in this Section 2 are the only rights granted under this -License. No additional rights or licenses will be implied from the distribution -or licensing of Covered Software under this License. Notwithstanding Section -2.1(b) above, no patent license is granted by a Contributor: - -for any code that a Contributor has removed from Covered Software; or - -for infringements caused by: (i) Your and any other third party’s -modifications of Covered Software, or (ii) the combination of its Contributions -with other software (except as part of its Contributor Version); or - -under Patent Claims infringed by Covered Software in the absence of its -Contributions. - -This License does not grant any rights in the trademarks, service marks, or -logos of any Contributor (except as may be necessary to comply with the notice -requirements in Section 3.4). - -2.4. Subsequent Licenses - -No Contributor makes additional grants as a result of Your choice to distribute -the Covered Software under a subsequent version of this License (see Section -10.2) or under the terms of a Secondary License (if permitted under the terms of -Section 3.3). - -2.5. Representation - -Each Contributor represents that the Contributor believes its Contributions are -its original creation(s) or it has sufficient rights to grant the rights to its -Contributions conveyed by this License. - -2.6. Fair Use - -This License is not intended to limit any rights You have under applicable -copyright doctrines of fair use, fair dealing, or other equivalents. - -2.7. Conditions - -Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in -Section 2.1. - -3. Responsibilities - -3.1. Distribution of Source Form - -All distribution of Covered Software in Source Code Form, including any -Modifications that You create or to which You contribute, must be under the -terms of this License. You must inform recipients that the Source Code Form of -the Covered Software is governed by the terms of this License, and how they can -obtain a copy of this License. You may not attempt to alter or restrict the -recipients’ rights in the Source Code Form. - -3.2. Distribution of Executable Form - -If You distribute Covered Software in Executable Form then: - -such Covered Software must also be made available in Source Code Form, as -described in Section 3.1, and You must inform recipients of the Executable Form -how they can obtain a copy of such Source Code Form by reasonable means in a -timely manner, at a charge no more than the cost of distribution to the -recipient; and - -You may distribute such Executable Form under the terms of this License, or -sublicense it under different terms, provided that the license for the -Executable Form does not attempt to limit or alter the recipients’ rights in -the Source Code Form under this License. - -3.3. Distribution of a Larger Work - -You may create and distribute a Larger Work under terms of Your choice, provided -that You also comply with the requirements of this License for the Covered -Software. If the Larger Work is a combination of Covered Software with a work -governed by one or more Secondary Licenses, and the Covered Software is not -Incompatible With Secondary Licenses, this License permits You to additionally -distribute such Covered Software under the terms of such Secondary License(s), -so that the recipient of the Larger Work may, at their option, further -distribute the Covered Software under the terms of either this License or such -Secondary License(s). - -3.4. Notices - -You may not remove or alter the substance of any license notices (including -copyright notices, patent notices, disclaimers of warranty, or limitations of -liability) contained within the Source Code Form of the Covered Software, except -that You may alter any license notices to the extent required to remedy known -factual inaccuracies. - -3.5. Application of Additional Terms - -You may choose to offer, and to charge a fee for, warranty, support, indemnity -or liability obligations to one or more recipients of Covered Software. However, -You may do so only on Your own behalf, and not on behalf of any Contributor. You -must make it absolutely clear that any such warranty, support, indemnity, or -liability obligation is offered by You alone, and You hereby agree to indemnify -every Contributor for any liability incurred by such Contributor as a result of -warranty, support, indemnity or liability terms You offer. You may include -additional disclaimers of warranty and limitations of liability specific to any -jurisdiction. - -4. Inability to Comply Due to Statute or Regulation - -If it is impossible for You to comply with any of the terms of this License with -respect to some or all of the Covered Software due to statute, judicial order, -or regulation then You must: (a) comply with the terms of this License to the -maximum extent possible; and (b) describe the limitations and the code they -affect. Such description must be placed in a text file included with all -distributions of the Covered Software under this License. Except to the extent -prohibited by statute or regulation, such description must be sufficiently -detailed for a recipient of ordinary skill to be able to understand it. - -5. Termination - -5.1. The rights granted under this License will terminate automatically if You -fail to comply with any of its terms. However, if You become compliant, then the -rights granted under this License from a particular Contributor are reinstated -(a) provisionally, unless and until such Contributor explicitly and finally -terminates Your grants, and (b) on an ongoing basis, if such Contributor fails -to notify You of the non-compliance by some reasonable means prior to 60 days -after You have come back into compliance. Moreover, Your grants from a -particular Contributor are reinstated on an ongoing basis if such Contributor -notifies You of the non-compliance by some reasonable means, this is the first -time You have received notice of non-compliance with this License from such -Contributor, and You become compliant prior to 30 days after Your receipt of the -notice. - -5.2. If You initiate litigation against any entity by asserting a patent -infringement claim (excluding declaratory judgment actions, counter-claims, and -cross-claims) alleging that a Contributor Version directly or indirectly -infringes any patent, then the rights granted to You by any and all Contributors -for the Covered Software under Section 2.1 of this License shall terminate. - -5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user -license agreements (excluding distributors and resellers) which have been -validly granted by You or Your distributors under this License prior to -termination shall survive termination. - -6. Disclaimer of Warranty - -Covered Software is provided under this License on an “as is” basis, without -warranty of any kind, either expressed, implied, or statutory, including, -without limitation, warranties that the Covered Software is free of defects, -merchantable, fit for a particular purpose or non-infringing. The entire risk as -to the quality and performance of the Covered Software is with You. Should any -Covered Software prove defective in any respect, You (not any Contributor) -assume the cost of any necessary servicing, repair, or correction. This -disclaimer of warranty constitutes an essential part of this License. No use of -any Covered Software is authorized under this License except under this -disclaimer. - -7. Limitation of Liability - -Under no circumstances and under no legal theory, whether tort (including -negligence), contract, or otherwise, shall any Contributor, or anyone who -distributes Covered Software as permitted above, be liable to You for any -direct, indirect, special, incidental, or consequential damages of any character -including, without limitation, damages for lost profits, loss of goodwill, work -stoppage, computer failure or malfunction, or any and all other commercial -damages or losses, even if such party shall have been informed of the -possibility of such damages. This limitation of liability shall not apply to -liability for death or personal injury resulting from such party’s negligence -to the extent applicable law prohibits such limitation. Some jurisdictions do -not allow the exclusion or limitation of incidental or consequential damages, so -this exclusion and limitation may not apply to You. - -8. Litigation - -Any litigation relating to this License may be brought only in the courts of a -jurisdiction where the defendant maintains its principal place of business and -such litigation shall be governed by laws of that jurisdiction, without -reference to its conflict-of-law provisions. Nothing in this Section shall -prevent a party’s ability to bring cross-claims or counter-claims. - -9. Miscellaneous - -This License represents the complete agreement concerning the subject matter -hereof. If any provision of this License is held to be unenforceable, such -provision shall be reformed only to the extent necessary to make it enforceable. -Any law or regulation which provides that the language of a contract shall be -construed against the drafter shall not be used to construe this License against -a Contributor. - -10. Versions of the License - -10.1. New Versions - -Mozilla Foundation is the license steward. Except as provided in Section 10.3, -no one other than the license steward has the right to modify or publish new -versions of this License. Each version will be given a distinguishing version -number. - -10.2. Effect of New Versions - -You may distribute the Covered Software under the terms of the version of the -License under which You originally received the Covered Software, or under the -terms of any subsequent version published by the license steward. - -10.3. Modified Versions - -If you create software not governed by this License, and you want to create a -new license for such software, you may create and use a modified version of this -License if you rename the license and remove any references to the name of the -license steward (except to note that such modified license differs from this -License). - -10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses - -If You choose to distribute Source Code Form that is Incompatible With Secondary -Licenses under the terms of this version of the License, the notice described in -Exhibit B of this License must be attached. - -Exhibit A - Source Code Form License Notice - -This Source Code Form is subject to the terms of the Mozilla Public License, v. -2.0. If a copy of the MPL was not distributed with this file, You can obtain one -at https://mozilla.org/MPL/2.0/. - -If it is not possible or desirable to put the notice in a particular file, then -You may include the notice in a location (such as a LICENSE file in a relevant -directory) where a recipient would be likely to look for such a notice. - -You may add additional accurate notices of copyright ownership. - -Exhibit B - “Incompatible With Secondary Licenses” Notice - -This Source Code Form is “Incompatible With Secondary Licenses”, as defined -by the Mozilla Public License, v. 2.0. - -The binary distribution of this product bundles these dependencies under the -following license: -JDOM 1.1 --------------------------------------------------------------------------------- -/*-- - - Copyright (C) 2000-2004 Jason Hunter & Brett McLaughlin. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions, and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions, and the disclaimer that follows - these conditions in the documentation and/or other materials - provided with the distribution. - - 3. The name "JDOM" must not be used to endorse or promote products - derived from this software without prior written permission. For - written permission, please contact . - - 4. Products derived from this software may not be called "JDOM", nor - may "JDOM" appear in their name, without prior written permission - from the JDOM Project Management . - - In addition, we request (but do not require) that you include in the - end-user documentation provided with the redistribution and/or in the - software itself an acknowledgement equivalent to the following: - "This product includes software developed by the - JDOM Project (http://www.jdom.org/)." - Alternatively, the acknowledgment may be graphical using the logos - available at http://www.jdom.org/images/logos. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED - WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE JDOM AUTHORS OR THE PROJECT - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - - This software consists of voluntary contributions made by many - individuals on behalf of the JDOM Project and was originally - created by Jason Hunter and - Brett McLaughlin . For more information - on the JDOM Project, please see . - - */ - -The binary distribution of this product bundles these dependencies under the -following license: -Hbase Server 1.2.4 --------------------------------------------------------------------------------- -This project bundles a derivative image for our Orca Logo. This image is -available under the Creative Commons By Attribution 3.0 License. - - Creative Commons Legal Code - - Attribution 3.0 Unported - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS LICENSE DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE INFORMATION PROVIDED, AND DISCLAIMS LIABILITY FOR - DAMAGES RESULTING FROM ITS USE. - - License - - THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE - COMMONS PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY - COPYRIGHT AND/OR OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS - AUTHORIZED UNDER THIS LICENSE OR COPYRIGHT LAW IS PROHIBITED. - - BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE - TO BE BOUND BY THE TERMS OF THIS LICENSE. TO THE EXTENT THIS LICENSE MAY - BE CONSIDERED TO BE A CONTRACT, THE LICENSOR GRANTS YOU THE RIGHTS - CONTAINED HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND - CONDITIONS. - - 1. Definitions - - a. "Adaptation" means a work based upon the Work, or upon the Work and - other pre-existing works, such as a translation, adaptation, - derivative work, arrangement of music or other alterations of a - literary or artistic work, or phonogram or performance and includes - cinematographic adaptations or any other form in which the Work may be - recast, transformed, or adapted including in any form recognizably - derived from the original, except that a work that constitutes a - Collection will not be considered an Adaptation for the purpose of - this License. For the avoidance of doubt, where the Work is a musical - work, performance or phonogram, the synchronization of the Work in - timed-relation with a moving image ("synching") will be considered an - Adaptation for the purpose of this License. - b. "Collection" means a collection of literary or artistic works, such as - encyclopedias and anthologies, or performances, phonograms or - broadcasts, or other works or subject matter other than works listed - in Section 1(f) below, which, by reason of the selection and - arrangement of their contents, constitute intellectual creations, in - which the Work is included in its entirety in unmodified form along - with one or more other contributions, each constituting separate and - independent works in themselves, which together are assembled into a - collective whole. A work that constitutes a Collection will not be - considered an Adaptation (as defined above) for the purposes of this - License. - c. "Distribute" means to make available to the public the original and - copies of the Work or Adaptation, as appropriate, through sale or - other transfer of ownership. - d. "Licensor" means the individual, individuals, entity or entities that - offer(s) the Work under the terms of this License. - e. "Original Author" means, in the case of a literary or artistic work, - the individual, individuals, entity or entities who created the Work - or if no individual or entity can be identified, the publisher; and in - addition (i) in the case of a performance the actors, singers, - musicians, dancers, and other persons who act, sing, deliver, declaim, - play in, interpret or otherwise perform literary or artistic works or - expressions of folklore; (ii) in the case of a phonogram the producer - being the person or legal entity who first fixes the sounds of a - performance or other sounds; and, (iii) in the case of broadcasts, the - organization that transmits the broadcast. - f. "Work" means the literary and/or artistic work offered under the terms - of this License including without limitation any production in the - literary, scientific and artistic domain, whatever may be the mode or - form of its expression including digital form, such as a book, - pamphlet and other writing; a lecture, address, sermon or other work - of the same nature; a dramatic or dramatico-musical work; a - choreographic work or entertainment in dumb show; a musical - composition with or without words; a cinematographic work to which are - assimilated works expressed by a process analogous to cinematography; - a work of drawing, painting, architecture, sculpture, engraving or - lithography; a photographic work to which are assimilated works - expressed by a process analogous to photography; a work of applied - art; an illustration, map, plan, sketch or three-dimensional work - relative to geography, topography, architecture or science; a - performance; a broadcast; a phonogram; a compilation of data to the - extent it is protected as a copyrightable work; or a work performed by - a variety or circus performer to the extent it is not otherwise - considered a literary or artistic work. - g. "You" means an individual or entity exercising rights under this - License who has not previously violated the terms of this License with - respect to the Work, or who has received express permission from the - Licensor to exercise rights under this License despite a previous - violation. - h. "Publicly Perform" means to perform public recitations of the Work and - to communicate to the public those public recitations, by any means or - process, including by wire or wireless means or public digital - performances; to make available to the public Works in such a way that - members of the public may access these Works from a place and at a - place individually chosen by them; to perform the Work to the public - by any means or process and the communication to the public of the - performances of the Work, including by public digital performance; to - broadcast and rebroadcast the Work by any means including signs, - sounds or images. - i. "Reproduce" means to make copies of the Work by any means including - without limitation by sound or visual recordings and the right of - fixation and reproducing fixations of the Work, including storage of a - protected performance or phonogram in digital form or other electronic - medium. - - 2. Fair Dealing Rights. Nothing in this License is intended to reduce, - limit, or restrict any uses free from copyright or rights arising from - limitations or exceptions that are provided for in connection with the - copyright protection under copyright law or other applicable laws. - - 3. License Grant. Subject to the terms and conditions of this License, - Licensor hereby grants You a worldwide, royalty-free, non-exclusive, - perpetual (for the duration of the applicable copyright) license to - exercise the rights in the Work as stated below: - - a. to Reproduce the Work, to incorporate the Work into one or more - Collections, and to Reproduce the Work as incorporated in the - Collections; - b. to create and Reproduce Adaptations provided that any such Adaptation, - including any translation in any medium, takes reasonable steps to - clearly label, demarcate or otherwise identify that changes were made - to the original Work. For example, a translation could be marked "The - original work was translated from English to Spanish," or a - modification could indicate "The original work has been modified."; - c. to Distribute and Publicly Perform the Work including as incorporated - in Collections; and, - d. to Distribute and Publicly Perform Adaptations. - e. For the avoidance of doubt: - - i. Non-waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme cannot be waived, the Licensor - reserves the exclusive right to collect such royalties for any - exercise by You of the rights granted under this License; - ii. Waivable Compulsory License Schemes. In those jurisdictions in - which the right to collect royalties through any statutory or - compulsory licensing scheme can be waived, the Licensor waives the - exclusive right to collect such royalties for any exercise by You - of the rights granted under this License; and, - iii. Voluntary License Schemes. The Licensor waives the right to - collect royalties, whether individually or, in the event that the - Licensor is a member of a collecting society that administers - voluntary licensing schemes, via that society, from any exercise - by You of the rights granted under this License. - - The above rights may be exercised in all media and formats whether now - known or hereafter devised. The above rights include the right to make - such modifications as are technically necessary to exercise the rights in - other media and formats. Subject to Section 8(f), all rights not expressly - granted by Licensor are hereby reserved. - - 4. Restrictions. The license granted in Section 3 above is expressly made - subject to and limited by the following restrictions: - - a. You may Distribute or Publicly Perform the Work only under the terms - of this License. You must include a copy of, or the Uniform Resource - Identifier (URI) for, this License with every copy of the Work You - Distribute or Publicly Perform. You may not offer or impose any terms - on the Work that restrict the terms of this License or the ability of - the recipient of the Work to exercise the rights granted to that - recipient under the terms of the License. You may not sublicense the - Work. You must keep intact all notices that refer to this License and - to the disclaimer of warranties with every copy of the Work You - Distribute or Publicly Perform. When You Distribute or Publicly - Perform the Work, You may not impose any effective technological - measures on the Work that restrict the ability of a recipient of the - Work from You to exercise the rights granted to that recipient under - the terms of the License. This Section 4(a) applies to the Work as - incorporated in a Collection, but this does not require the Collection - apart from the Work itself to be made subject to the terms of this - License. If You create a Collection, upon notice from any Licensor You - must, to the extent practicable, remove from the Collection any credit - as required by Section 4(b), as requested. If You create an - Adaptation, upon notice from any Licensor You must, to the extent - practicable, remove from the Adaptation any credit as required by - Section 4(b), as requested. - b. If You Distribute, or Publicly Perform the Work or any Adaptations or - Collections, You must, unless a request has been made pursuant to - Section 4(a), keep intact all copyright notices for the Work and - provide, reasonable to the medium or means You are utilizing: (i) the - name of the Original Author (or pseudonym, if applicable) if supplied, - and/or if the Original Author and/or Licensor designate another party - or parties (e.g., a sponsor institute, publishing entity, journal) for - attribution ("Attribution Parties") in Licensor's copyright notice, - terms of service or by other reasonable means, the name of such party - or parties; (ii) the title of the Work if supplied; (iii) to the - extent reasonably practicable, the URI, if any, that Licensor - specifies to be associated with the Work, unless such URI does not - refer to the copyright notice or licensing information for the Work; - and (iv) , consistent with Section 3(b), in the case of an Adaptation, - a credit identifying the use of the Work in the Adaptation (e.g., - "French translation of the Work by Original Author," or "Screenplay - based on original Work by Original Author"). The credit required by - this Section 4 (b) may be implemented in any reasonable manner; - provided, however, that in the case of a Adaptation or Collection, at - a minimum such credit will appear, if a credit for all contributing - authors of the Adaptation or Collection appears, then as part of these - credits and in a manner at least as prominent as the credits for the - other contributing authors. For the avoidance of doubt, You may only - use the credit required by this Section for the purpose of attribution - in the manner set out above and, by exercising Your rights under this - License, You may not implicitly or explicitly assert or imply any - connection with, sponsorship or endorsement by the Original Author, - Licensor and/or Attribution Parties, as appropriate, of You or Your - use of the Work, without the separate, express prior written - permission of the Original Author, Licensor and/or Attribution - Parties. - c. Except as otherwise agreed in writing by the Licensor or as may be - otherwise permitted by applicable law, if You Reproduce, Distribute or - Publicly Perform the Work either by itself or as part of any - Adaptations or Collections, You must not distort, mutilate, modify or - take other derogatory action in relation to the Work which would be - prejudicial to the Original Author's honor or reputation. Licensor - agrees that in those jurisdictions (e.g. Japan), in which any exercise - of the right granted in Section 3(b) of this License (the right to - make Adaptations) would be deemed to be a distortion, mutilation, - modification or other derogatory action prejudicial to the Original - Author's honor and reputation, the Licensor will waive or not assert, - as appropriate, this Section, to the fullest extent permitted by the - applicable national law, to enable You to reasonably exercise Your - right under Section 3(b) of this License (right to make Adaptations) - but not otherwise. - - 5. Representations, Warranties and Disclaimer - - UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR - OFFERS THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY - KIND CONCERNING THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, - INCLUDING, WITHOUT LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, - FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF - LATENT OR OTHER DEFECTS, ACCURACY, OR THE PRESENCE OF ABSENCE OF ERRORS, - WHETHER OR NOT DISCOVERABLE. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION - OF IMPLIED WARRANTIES, SO SUCH EXCLUSION MAY NOT APPLY TO YOU. - - 6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE - LAW, IN NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR - ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES - ARISING OUT OF THIS LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS - BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - 7. Termination - - a. This License and the rights granted hereunder will terminate - automatically upon any breach by You of the terms of this License. - Individuals or entities who have received Adaptations or Collections - from You under this License, however, will not have their licenses - terminated provided such individuals or entities remain in full - compliance with those licenses. Sections 1, 2, 5, 6, 7, and 8 will - survive any termination of this License. - b. Subject to the above terms and conditions, the license granted here is - perpetual (for the duration of the applicable copyright in the Work). - Notwithstanding the above, Licensor reserves the right to release the - Work under different license terms or to stop distributing the Work at - any time; provided, however that any such election will not serve to - withdraw this License (or any other license that has been, or is - required to be, granted under the terms of this License), and this - License will continue in full force and effect unless terminated as - stated above. - - 8. Miscellaneous - - a. Each time You Distribute or Publicly Perform the Work or a Collection, - the Licensor offers to the recipient a license to the Work on the same - terms and conditions as the license granted to You under this License. - b. Each time You Distribute or Publicly Perform an Adaptation, Licensor - offers to the recipient a license to the original Work on the same - terms and conditions as the license granted to You under this License. - c. If any provision of this License is invalid or unenforceable under - applicable law, it shall not affect the validity or enforceability of - the remainder of the terms of this License, and without further action - by the parties to this agreement, such provision shall be reformed to - the minimum extent necessary to make such provision valid and - enforceable. - d. No term or provision of this License shall be deemed waived and no - breach consented to unless such waiver or consent shall be in writing - and signed by the party to be charged with such waiver or consent. - e. This License constitutes the entire agreement between the parties with - respect to the Work licensed here. There are no understandings, - agreements or representations with respect to the Work not specified - here. Licensor shall not be bound by any additional provisions that - may appear in any communication from You. This License may not be - modified without the mutual written agreement of the Licensor and You. - f. The rights granted under, and the subject matter referenced, in this - License were drafted utilizing the terminology of the Berne Convention - for the Protection of Literary and Artistic Works (as amended on - September 28, 1979), the Rome Convention of 1961, the WIPO Copyright - Treaty of 1996, the WIPO Performances and Phonograms Treaty of 1996 - and the Universal Copyright Convention (as revised on July 24, 1971). - These rights and subject matter take effect in the relevant - jurisdiction in which the License terms are sought to be enforced - according to the corresponding provisions of the implementation of - those treaty provisions in the applicable national law. If the - standard suite of rights granted under applicable copyright law - includes additional rights not granted under this License, such - additional rights are deemed to be included in the License; this - License is not intended to restrict the license of any rights under - applicable law. - - Creative Commons Notice - - Creative Commons is not a party to this License, and makes no warranty - whatsoever in connection with the Work. Creative Commons will not be - liable to You or any party on any legal theory for any damages - whatsoever, including without limitation any general, special, - incidental or consequential damages arising in connection to this - license. Notwithstanding the foregoing two (2) sentences, if Creative - Commons has expressly identified itself as the Licensor hereunder, it - shall have all rights and obligations of Licensor. - - Except for the limited purpose of indicating to the public that the - Work is licensed under the CCPL, Creative Commons does not authorize - the use by either party of the trademark "Creative Commons" or any - related trademark or logo of Creative Commons without the prior - written consent of Creative Commons. Any permitted use will be in - compliance with Creative Commons' then-current trademark usage - guidelines, as may be published on its website or otherwise made - available upon request from time to time. For the avoidance of doubt, - this trademark restriction does not form part of this License. - - Creative Commons may be contacted at https://creativecommons.org/. --------------------------------------------------------------------------------- - -For: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs -/server/datanode/checker/AbstractFuture.java and -hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs -/server/datanode/checker/TimeoutFuture.java - -Copyright (C) 2007 The Guava Authors - -Licensed under the Apache License, Version 2.0 (the "License"); you may not -use this file except in compliance with the License. You may obtain a copy of -the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations under -the License. - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). - -The binary distribution of this product bundles binaries of -org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the -following notices: -* Copyright 2011 Dain Sundstrom -* Copyright 2011 FuseSource Corp. http://fusesource.com - -The binary distribution of this product bundles binaries of -AWS SDK for Java - Bundle 1.11.134, -AWS Java SDK for AWS KMS 1.11.134, -AWS Java SDK for Amazon S3 1.11.134, -AWS Java SDK for AWS STS 1.11.134, -JMES Path Query library 1.0, -which has the following notices: - * This software includes third party software subject to the following - copyrights: - XML parsing and utility functions from JetS3t - Copyright - 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - - Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility - functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. - -The binary distribution of this product bundles binaries of -Gson 2.2.4, -which has the following notices: - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'ArrayDeque', written by Josh -Bloch of Google, Inc: - - * LICENSE: - * license/LICENSE.deque.txt (Public Domain) - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified version of Roland Kuhn's ASL2 -AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. -It can be obtained at: - - * LICENSE: - * license/LICENSE.abstractnodequeue.txt (Public Domain) - * HOMEPAGE: - * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/libdivsufsort/ - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/protobuf/ - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/snappy/ - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * http://code.google.com/p/caliper/ - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - -This product contains a modified portion of 'JDOM 1.1', which can be obtained at: - - * LICENSE: - * https://github.com/hunterhacker/jdom/blob/jdom-1.1/core/LICENSE.txt - * HOMEPAGE: - * http://www.jdom.org/ - -The binary distribution of this product bundles binaries of -Commons Codec 1.4, -which has the following notices: - * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - =============================================================================== - The content of package org.apache.commons.codec.language.bm has been translated - from the original php source code available at http://stevemorse.org/phoneticinfo.htm - with permission from the original authors. - Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -The binary distribution of this product bundles binaries of -Commons Lang 2.6, -which has the following notices: - * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -The binary distribution of this product bundles binaries of -Apache Log4j 1.2.17, -which has the following notices: - * ResolverUtil.java - Copyright 2005-2006 Tim Fennell - Dumbster SMTP test server - Copyright 2004 Jason Paul Kitchen - TypeUtil.java - Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams - -The binary distribution of this product bundles binaries of -"Java Concurrency in Practice" book annotations 1.0, -which has the following notices: - * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative - Commons Attribution License (http://creativecommons.org/licenses/by/2.5) - Official home: http://www.jcip.net Any republication or derived work - distributed in source code form must include this copyright and license - notice. - -The binary distribution of this product bundles binaries of -Jetty :: Http Utility 9.3.19., -Jetty :: IO Utility 9.3.19., -Jetty :: Security 9.3.19., -Jetty :: Server Core 9.3.19., -Jetty :: Servlet Handling 9.3.19., -Jetty :: Utilities 9.3.19., -Jetty :: Utilities :: Ajax, -Jetty :: Webapp Application Support 9.3.19., -Jetty :: XML utilities 9.3.19., -which has the following notices: - * ============================================================== - Jetty Web Container - Copyright 1995-2016 Mort Bay Consulting Pty Ltd. - ============================================================== - - The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd - unless otherwise noted. - - Jetty is dual licensed under both - - * The Apache 2.0 License - http://www.apache.org/licenses/LICENSE-2.0.html - - and - - * The Eclipse Public 1.0 License - http://www.eclipse.org/legal/epl-v10.html - - Jetty may be distributed under either license. - - ------ - Eclipse - - The following artifacts are EPL. - * org.eclipse.jetty.orbit:org.eclipse.jdt.core - - The following artifacts are EPL and ASL2. - * org.eclipse.jetty.orbit:javax.security.auth.message - - The following artifacts are EPL and CDDL 1.0. - * org.eclipse.jetty.orbit:javax.mail.glassfish - - ------ - Oracle - - The following artifacts are CDDL + GPLv2 with classpath exception. - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - * javax.servlet:javax.servlet-api - * javax.annotation:javax.annotation-api - * javax.transaction:javax.transaction-api - * javax.websocket:javax.websocket-api - - ------ - Oracle OpenJDK - - If ALPN is used to negotiate HTTP/2 connections, then the following - artifacts may be included in the distribution or downloaded when ALPN - module is selected. - - * java.sun.security.ssl - - These artifacts replace/modify OpenJDK classes. The modififications - are hosted at github and both modified and original are under GPL v2 with - classpath exceptions. - http://openjdk.java.net/legal/gplv2+ce.html - - ------ - OW2 - - The following artifacts are licensed by the OW2 Foundation according to the - terms of http://asm.ow2.org/license.html - - org.ow2.asm:asm-commons - org.ow2.asm:asm - - ------ - Apache - - The following artifacts are ASL2 licensed. - - org.apache.taglibs:taglibs-standard-spec - org.apache.taglibs:taglibs-standard-impl - - ------ - MortBay - - The following artifacts are ASL2 licensed. Based on selected classes from - following Apache Tomcat jars, all ASL2 licensed. - - org.mortbay.jasper:apache-jsp - org.apache.tomcat:tomcat-jasper - org.apache.tomcat:tomcat-juli - org.apache.tomcat:tomcat-jsp-api - org.apache.tomcat:tomcat-el-api - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-api - org.apache.tomcat:tomcat-util-scan - org.apache.tomcat:tomcat-util - - org.mortbay.jasper:apache-el - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-el-api - - ------ - Mortbay - - The following artifacts are CDDL + GPLv2 with classpath exception. - - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - org.eclipse.jetty.toolchain:jetty-schemas - - ------ - Assorted - - The UnixCrypt.java code implements the one way cryptography used by - Unix systems for simple password protection. Copyright 1996 Aki Yoshida, - modified April 2001 by Iris Van den Broeke, Daniel Deville. - Permission to use, copy, modify and distribute UnixCrypt - for non-commercial or commercial purposes and without fee is - granted provided that the copyright notice appears in all copies./ - -The binary distribution of this product bundles binaries of -Snappy for Java 1.0.4.1, -which has the following notices: - * This product includes software developed by Google - Snappy: http://code.google.com/p/snappy/ (New BSD License) - - This product includes software developed by Apache - PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ - (Apache 2.0 license) - - This library containd statically linked libstdc++. This inclusion is allowed by - "GCC RUntime Library Exception" - http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html - - == Contributors == - * Tatu Saloranta - * Providing benchmark suite - * Alec Wysoker - * Performance and memory usage improvement - -The binary distribution of this product bundles binaries of -Xerces2 Java Parser 2.9.1, -which has the following notices: - * ========================================================================= - == NOTICE file corresponding to section 4(d) of the Apache License, == - == Version 2.0, in this case for the Apache Xerces Java distribution. == - ========================================================================= - - Apache Xerces Java - Copyright 1999-2007 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were originally based on the following: - - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. - - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. - - voluntary contributions made by Paul Eng on behalf of the - Apache Software Foundation that were originally developed at iClick, Inc., - software copyright (c) 1999. - -The binary distribution of this product bundles binaries of -Logback Classic Module 1.1.2, -Logback Core Module 1.1.2, -which has the following notices: - * Logback: the reliable, generic, fast and flexible logging framework. - Copyright (C) 1999-2012, QOS.ch. All rights reserved. - -The binary distribution of this product bundles binaries of -Apache HBase - Annotations 1.2.6, -Apache HBase - Client 1.2.6, -Apache HBase - Common 1.2.6, -Apache HBase - Hadoop Compatibility 1.2.6, -Apache HBase - Hadoop Two Compatibility 1.2.6, -Apache HBase - Prefix Tree 1.2.6, -Apache HBase - Procedure 1.2.6, -Apache HBase - Protocol 1.2.6, -Apache HBase - Server 1.2.6, -which has the following notices: - * Apache HBase - Copyright 2007-2015 The Apache Software Foundation - - -- - This product incorporates portions of the 'Hadoop' project - - Copyright 2007-2009 The Apache Software Foundation - - Licensed under the Apache License v2.0 - -- - Our Orca logo we got here: http://www.vectorfree.com/jumping-orca - It is licensed Creative Commons Attribution 3.0. - See https://creativecommons.org/licenses/by/3.0/us/ - We changed the logo by stripping the colored background, inverting - it and then rotating it some. - - Later we found that vectorfree.com image is not properly licensed. - The original is owned by vectorportal.com. The original was - relicensed so we could use it as Creative Commons Attribution 3.0. - The license is bundled with the download available here: - http://www.vectorportal.com/subcategory/205/KILLER-WHALE-FREE-VECTOR.eps/ifile/9136/detailtest.asp - -- - This product includes portions of the Bootstrap project v3.0.0 - - Copyright 2013 Twitter, Inc. - - Licensed under the Apache License v2.0 - - This product uses the Glyphicons Halflings icon set. - - http://glyphicons.com/ - - Copyright Jan Kovařík - - Licensed under the Apache License v2.0 as a part of the Bootstrap project. - - -- - This product includes portions of the Guava project v14, specifically - 'hbase-common/src/main/java/org/apache/hadoop/hbase/io/LimitInputStream.java' - - Copyright (C) 2007 The Guava Authors - - Licensed under the Apache License, Version 2.0 - -The binary distribution of this product bundles binaries of -Phoenix Core 4.7.0, -which has the following notices: - Apache Phoenix - Copyright 2013-2016 The Apache Software Foundation - - This product includes software developed by The Apache Software - Foundation (http://www.apache.org/). - - This also includes: - - The phoenix-spark module has been adapted from the phoenix-spark library - distributed under the terms of the Apache 2 license. Original source copyright: - Copyright 2014 Simply Measured, Inc. - Copyright 2015 Interset Software Inc. - - The file bin/daemon.py is based on the file of the same name in python-daemon 2.0.5 - (https://pypi.python.org/pypi/python-daemon/). Original source copyright: - # Copyright © 2008–2015 Ben Finney - # Copyright © 2007–2008 Robert Niederreiter, Jens Klein - # Copyright © 2004–2005 Chad J. Schroeder - # Copyright © 2003 Clark Evans - # Copyright © 2002 Noah Spurrier - # Copyright © 2001 Jürgen Hermann - -The binary distribution of this product bundles binaries of -Plexus Cipher: encryption/decryption Component 1.4, -which has the following notices: - * The code in this component contains a class - Base64 taken from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/Base64.java - which is Apache license: http://www.apache.org/licenses/LICENSE-2.0 - - The PBE key processing routine PBECipher.createCipher() is adopted from http://juliusdavies.ca/svn/not-yet-commons-ssl/tags/commons-ssl-0.3.10/src/java/org/apache/commons/ssl/OpenSSL.java - which is also Apache APL-2.0 license: http://www.apache.org/licenses/LICENSE-2.0 - -The binary distribution of this product bundles binaries of -software.amazon.ion:ion-java 1.0.1, -which has the following notices: - * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. - -The binary distribution of this product bundles binaries of -joda-time:joda-time:2.9.9 -which has the following notices: - * ============================================================================= - = NOTICE file corresponding to section 4d of the Apache License Version 2.0 = - ============================================================================= - This product includes software developed by - Joda.org (http://www.joda.org/). - -The binary distribution of this product bundles binaries of -Ehcache 3.3.1, -which has the following notices: - * Ehcache V3 Copyright 2014-2016 Terracotta, Inc. - -The binary distribution of this product bundles binaries of -snakeyaml (https://bitbucket.org/asomov/snakeyaml), -which has the following notices: - * Copyright (c) 2008, http://www.snakeyaml.org - -The binary distribution of this product bundles binaries of -swagger-annotations (https://github.com/swagger-api/swagger-core), -which has the following notices: - * Copyright 2016 SmartBear Software - -The binary distribution of this product bundles binaries of -metrics-core 3.2.4 -which has the following notices: - * Copyright 2010-2013 Coda Hale and Yammer, Inc. - - This product includes software developed by Coda Hale and Yammer, Inc. - - This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, - LongAdder), which was released with the following comments: - - Written by Doug Lea with assistance from members of JCP JSR-166 - Expert Group and released to the public domain, as explained at - http://creativecommons.org/publicdomain/zero/1.0/ - -Apache Commons IO -Copyright 2002-2012 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -Apache Commons Collections -Copyright 2001-2015 The Apache Software Foundation - -Apache Commons Logging -Copyright 2003-2013 The Apache Software Foundation - -Apache Commons Lang -Copyright 2001-2011 The Apache Software Foundation - -Apache Commons BeanUtils -Copyright 2000-2016 The Apache Software Foundation - -Apache Commons Configuration -Copyright 2001-2017 The Apache Software Foundation - -Apache Commons Lang -Copyright 2001-2014 The Apache Software Foundation - -This product includes software from the Spring Framework, -under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -htrace-core4 -Copyright 2016 The Apache Software Foundation - -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -flink-hadoop-fs -Copyright 2014-2019 The Apache Software Foundation - -Apache HttpClient -Copyright 1999-2017 The Apache Software Foundation - -Apache HttpCore -Copyright 2005-2017 The Apache Software Foundation - -Apache Commons Codec -Copyright 2002-2014 The Apache Software Foundation - -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -============================================================================= -= NOTICE file corresponding to section 4d of the Apache License Version 2.0 = -============================================================================= -This product includes software developed by -Joda.org (http://www.joda.org/). - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink-shaded -// ------------------------------------------------------------------ - -Apache Flink-shaded -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-shaded-netty-openssl-static -Copyright 2014-2019 The Apache Software Foundation - -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- io.netty:netty-tcnative:2.0.25.Final - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-sql-client -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the BSD license. -See bundled license files for details. - -- org.jline:jline-terminal:3.9.0 -- org.jline:jline-reader:3.9.0 - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-state-processor-api -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -flink-swift-fs-hadoop -Copyright 2014-2019 The Apache Software Foundation - -This project includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- org.apache.hadoop:hadoop-annotations:2.8.1 -- org.apache.hadoop:hadoop-auth:2.8.1 -- org.apache.hadoop:hadoop-client:2.8.1 -- org.apache.hadoop:hadoop-common:2.8.1 -- org.apache.hadoop:hadoop-hdfs:2.8.1 -- org.apache.hadoop:hadoop-hdfs-client:2.8.1 -- org.apache.hadoop:hadoop-openstack:2.8.1 -- org.apache.htrace:htrace-core4:4.0.1-incubating -- org.apache.httpcomponents:httpclient:4.5.3 -- org.apache.httpcomponents:httpcore:4.4.6 -- org.apache.commons:commons-compress:1.18 -- org.apache.commons:commons-math3:3.5 -- commons-beanutils:commons-beanutils:1.8.3 -- commons-cli:commons-cli:1.3.1 -- commons-codec:commons-codec:1.10 -- commons-collections:commons-collections:3.2.2 -- commons-configuration:commons-configuration:1.7 -- commons-digester:commons-digester:1.8.1 -- commons-io:commons-io:2.4 -- commons-lang:commons-lang:2.6 -- commons-logging:commons-logging:1.1.3 -- commons-net:commons-net:3.1 -- commons-httpclient:commons-httpclient:3.1 -- com.google.guava:guava:11.0.2 -- com.google.code.gson:gson:2.2.4 -- com.squareup.okhttp:okhttp:2.4.0 -- com.squareup.okio:okio:1.4.0 -- net.minidev:json-smart:1.1.1 -- com.nimbusds:nimbus-jose-jwt:3.9 -- org.mortbay.jetty:jetty-sslengine:6.1.26 -- org.codehaus.jackson:jackson-core-asl:1.9.13 -- org.codehaus.jackson:jackson-mapper-asl:1.9.13 - -This project bundles the following dependencies under the BSD License. -See bundled license files for details. - -- xmlenc:xmlenc:0.52 -- com.google.protobuf:protobuf-java:2.5.0 - -This project bundles "net.jcip:jcip-annotations:1.0". -Written by Brian Goetz and Tim Peierls with assistance from members of JCP JSR-166 Expert Group -and released to the public domain, as explained by the Creative Commons public domain license. -https://creativecommons.org/licenses/publicdomain/ - -This project bundles "org.tukaani:xz:1.5". -This Java implementation of XZ has been put into the public domain, thus you can do -whatever you want with it. All the files in the package have been written by Lasse Collin, -but some files are heavily based on public domain code written by Igor Pavlov. - -This project bundles org.apache.hadoop:*:2.8.1 from which it inherits the following notices: - -The Apache Hadoop project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - -For the org.apache.hadoop.util.bloom.* classes: - -/** - * - * Copyright (c) 2005, European Commission project OneLab under contract - * 034819 (http://www.one-lab.org) - * All rights reserved. - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the distribution. - * - Neither the name of the University Catholique de Louvain - UCL - * nor the names of its contributors may be used to endorse or - * promote products derived from this software without specific prior - * written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN - * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -For portions of the native implementation of slicing-by-8 CRC calculation -in src/main/native/src/org/apache/hadoop/util: - -/** - * Copyright 2008,2009,2010 Massachusetts Institute of Technology. - * All rights reserved. Use of this source code is governed by a - * BSD-style license that can be found in the LICENSE file. - */ - -For src/main/native/src/org/apache/hadoop/io/compress/lz4/{lz4.h,lz4.c,lz4hc.h,lz4hc.c}, - -/* - LZ4 - Fast LZ compression algorithm - Header File - Copyright (C) 2011-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 source repository : http://code.google.com/p/lz4/ - - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c -*/ - -For hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/fuse-dfs/util/tree.h ---------------------------------------------------------------------- -Copyright 2002 Niels Provos -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles binaries of leveldbjni -(https://github.com/fusesource/leveldbjni), which is available under the -following license: - -Copyright (c) 2011 FuseSource Corp. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of FuseSource Corp. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -For hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/compat/{fstatat|openat|unlinkat}.h: - -Copyright (c) 2012 The FreeBSD Foundation -All rights reserved. - -This software was developed by Pawel Jakub Dawidek under sponsorship from -the FreeBSD Foundation. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. - -============= - -The binary distribution of this product bundles binaries of leveldb -(http://code.google.com/p/leveldb/), which is available under the following -license: - -Copyright (c) 2011 The LevelDB Authors. All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -The binary distribution of this product bundles binaries of snappy -(http://code.google.com/p/snappy/), which is available under the following -license: - -Copyright 2011, Google Inc. -All rights reserved. - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/dt-1.9.4/ --------------------------------------------------------------------------------- -Copyright (C) 2008-2016, SpryMedia Ltd. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2010 Aleksander Williams - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js --------------------------------------------------------------------------------- - -Copyright (c) 2011-2016 Tim Wood, Iskren Chernev, Moment.js contributors - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/bootstrap-3.0.2 -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap.min.css -hadoop-tools/hadoop-sls/src/main/html/css/bootstrap-responsive.min.css -And the binary distribution of this product bundles these dependencies under the -following license: -Mockito 1.8.5 -SLF4J 1.7.10 --------------------------------------------------------------------------------- - -The MIT License (MIT) - -For: -hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-1.10.2.min.js -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/jquery.js -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jquery --------------------------------------------------------------------------------- - -Copyright jQuery Foundation and other contributors, https://jquery.org/ - -This software consists of voluntary contributions made by many -individuals. For exact contribution history, see the revision history -available at https://github.com/jquery/jquery - -The following license applies to all parts of this software except as -documented below: - -==== - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -All files located in the node_modules and external directories are -externally maintained libraries used by this software which have their -own licenses; we recommend you read them, as their terms may differ from -the terms above. - -For: -hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/jt/jquery.jstree.js.gz --------------------------------------------------------------------------------- - -Copyright (c) 2014 Ivan Bozhanov - -For: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3.v3.js --------------------------------------------------------------------------------- - -D3 is available under a 3-clause BSD license. For details, see: -hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/d3-LICENSE - -The binary distribution of this product bundles these dependencies under the -following license: -HSQLDB Database 2.0.0 --------------------------------------------------------------------------------- -"COPYRIGHTS AND LICENSES (based on BSD License) - -For work developed by the HSQL Development Group: - -Copyright (c) 2001-2016, The HSQL Development Group -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -Neither the name of the HSQL Development Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -For work originally developed by the Hypersonic SQL Group: - -Copyright (c) 1995-2000 by the Hypersonic SQL Group. -All rights reserved. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -Neither the name of the Hypersonic SQL Group nor the names of its -contributors may be used to endorse or promote products derived from this -software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP, -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -This software consists of voluntary contributions made by many individuals on behalf of the -Hypersonic SQL Group." - -The binary distribution of this product bundles these dependencies under the -following license: -servlet-api 2.5 -jsp-api 2.1 -Streaming API for XML 1.0 --------------------------------------------------------------------------------- -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 -1. Definitions.  - -1.1. Contributor means each individual or entity -that creates or contributes to the creation of -Modifications.  - -1.2. Contributor Version means the combination of the -Original Software, prior Modifications used by a Contributor (if any), and the -Modifications made by that particular Contributor.  - -1.3. Covered -Software means (a) the Original Software, or (b) Modifications, or (c) the -combination of files containing Original Software with files containing -Modifications, in each case including portions -thereof.  - -1.4. Executable means the Covered Software in any form other -than Source Code.  - -1.5. Initial Developer means the individual or entity -that first makes Original Software available under this -License.  - -1.6. Larger Work means a work which combines Covered Software or -portions thereof with code not governed by the terms of this -License.  - -1.7. License means this document.  - -1.8. Licensable means -having the right to grant, to the maximum extent possible, whether at the time -of the initial grant or subsequently acquired, any and all of the rights -conveyed herein.  - -1.9. Modifications means the Source Code and Executable -form of any of the following: -A. Any file that results from an addition to, -deletion from or modification of the contents of a file containing Original -Software or previous Modifications; -B. Any new file that contains any part of the Original Software -or previous Modification; or -C. Any new file that is contributed or otherwise made available -under the terms of this License.  - -1.10. Original Software means the Source Code and Executable form of -computer software code that is originally released under this License.  - -1.11. Patent Claims means any patent claim(s), now owned or -hereafter acquired, including without limitation, method, process, and apparatus -claims, in any patent Licensable by grantor.  - -1.12. Source Code means (a) the common form of computer software code in which -modifications are made and (b) associated documentation included in or -with such code.  - -1.13. You (or Your) means an individual or a legal entity exercising rights -under, and complying with all of the terms of, this License. For legal entities, -You includes any entity which controls, is controlled by, or is under common control -with You. For purposes of this definition, control means (a) the power, direct -or indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (b) ownership of more than fifty percent (50%) of the -outstanding shares or beneficial ownership of such entity.  - -2. License Grants. - -2.1. The Initial Developer Grant. Conditioned upon Your compliance -with Section 3.1 below and subject to third party intellectual property claims, -the Initial Developer hereby grants You a world-wide, royalty-free, -non-exclusive license:  - -(a) under intellectual property rights (other than -patent or trademark) Licensable by Initial Developer, to use, reproduce, modify, -display, perform, sublicense and distribute the Original Software (or portions -thereof), with or without Modifications, and/or as part of a Larger Work; -and  - -(b) under Patent Claims infringed by the making, using or selling of -Original Software, to make, have made, use, practice, sell, and offer for sale, -and/or otherwise dispose of the Original Software (or portions -thereof); - -(c) The licenses granted in Sections 2.1(a) and (b) are -effective on the date Initial Developer first distributes or otherwise makes the -Original Software available to a third party under the terms of this -License; - -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or -(ii) the combination of the Original Software with other software or -devices.  - -2.2. Contributor Grant. Conditioned upon Your compliance with -Section 3.1 below and subject to third party intellectual property claims, each -Contributor hereby grants You a world-wide, royalty-free, non-exclusive -license:  - -(a) under intellectual property rights (other than patent or -trademark) Licensable by Contributor to use, reproduce, modify, display, -perform, sublicense and distribute the Modifications created by such Contributor -(or portions thereof), either on an unmodified basis, with other Modifications, -as Covered Software and/or as part of a Larger Work; and  - -(b) under Patent -Claims infringed by the making, using, or selling of Modifications made by that -Contributor either alone and/or in combination with its Contributor Version (or -portions of such combination), to make, use, sell, offer for sale, have made, -and/or otherwise dispose of: (1) Modifications made by that Contributor (or -portions thereof); and (2) the combination of Modifications made by that -Contributor with its Contributor Version (or portions of such -combination).  - -(c) The licenses granted in Sections 2.2(a) and 2.2(b) are -effective on the date Contributor first distributes or otherwise makes the -Modifications available to a third party. - -(d) Notwithstanding Section 2.2(b) -above, no patent license is granted: (1) for any code that Contributor has -deleted from the Contributor Version; (2) for infringements caused by: -(i) third party modifications of Contributor Version, or (ii) the combination -of Modifications made by that Contributor with other software (except as part of -the Contributor Version) or other devices; or (3) under Patent Claims infringed -by Covered Software in the absence of Modifications made by that -Contributor.  - -3. Distribution Obligations.  - -3.1. Availability of Source -Code. Any Covered Software that You distribute or otherwise make available in -Executable form must also be made available in Source Code form and that Source -Code form must be distributed only under the terms of this License. You must -include a copy of this License with every copy of the Source Code form of the -Covered Software You distribute or otherwise make available. You must inform -recipients of any such Covered Software in Executable form as to how they can -obtain such Covered Software in Source Code form in a reasonable manner on or -through a medium customarily used for software exchange.  - -3.2. -Modifications. The Modifications that You create or to which You contribute are -governed by the terms of this License. You represent that You believe Your -Modifications are Your original creation(s) and/or You have sufficient rights to -grant the rights conveyed by this License.  - -3.3. Required Notices. You must -include a notice in each of Your Modifications that identifies You as the -Contributor of the Modification. You may not remove or alter any copyright, -patent or trademark notices contained within the Covered Software, or any -notices of licensing or any descriptive text giving attribution to any -Contributor or the Initial Developer.  - -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source Code -form that alters or restricts the applicable version of this License or the -recipients rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. - -3.5. Distribution of Executable Versions. You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipients rights in the Source Code form from the rights set forth in this -License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer.  - -3.6. Larger Works. You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software.  - -4. Versions of the License.  - -4.1. -New Versions. Sun Microsystems, Inc. is the initial license steward and may -publish revised and/or new versions of this License from time to time. Each -version will be given a distinguishing version number. Except as provided in -Section 4.3, no one other than the license steward has the right to modify this -License.  - -4.2. Effect of New Versions. You may always continue to use, -distribute or otherwise make the Covered Software available under the terms of -the version of the License under which You originally received the Covered -Software. If the Initial Developer includes a notice in the Original Software -prohibiting it from being distributed or otherwise made available under any -subsequent version of the License, You must distribute and make the Covered -Software available under the terms of the version of the License under which You -originally received the Covered Software. Otherwise, You may also choose to use, -distribute or otherwise make the Covered Software available under the terms of -any subsequent version of the License published by the license -steward.  - -4.3. Modified Versions. When You are an Initial Developer and You -want to create a new license for Your Original Software, You may create and use -a modified version of this License if You: (a) rename the license and remove -any references to the name of the license steward (except to note that the -license differs from this License); and (b) otherwise make it clear that the -license contains terms which differ from this License.  - -5. DISCLAIMER OF WARRANTY. - -COVERED SOFTWARE IS PROVIDED UNDER THIS LICENSE ON AN AS IS BASIS, -WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, WITHOUT -LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE IS FREE OF DEFECTS, -MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE COVERED SOFTWARE IS WITH YOU. SHOULD ANY -COVERED SOFTWARE PROVE DEFECTIVE IN ANY RESPECT, YOU (NOT THE INITIAL DEVELOPER -OR ANY OTHER CONTRIBUTOR) ASSUME THE COST OF ANY NECESSARY SERVICING, REPAIR OR -CORRECTION. THIS DISCLAIMER OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS -LICENSE. NO USE OF ANY COVERED SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER -THIS DISCLAIMER.  - -6. TERMINATION.  - -6.1. This License and the rights -granted hereunder will terminate automatically if You fail to comply with terms -herein and fail to cure such breach within 30 days of becoming aware of the -breach. Provisions which, by their nature, must remain in effect beyond the -termination of this License shall survive.  - -6.2. If You assert a patent -infringement claim (excluding declaratory judgment actions) against Initial -Developer or a Contributor (the Initial Developer or Contributor against whom -You assert such claim is referred to as Participant) alleging that the -Participant Software (meaning the Contributor Version where the Participant is a -Contributor or the Original Software where the Participant is the Initial -Developer) directly or indirectly infringes any patent, then any and all rights -granted directly or indirectly to You by such Participant, the Initial Developer -(if the Initial Developer is not the Participant) and all Contributors under -Sections 2.1 and/or 2.2 of this License shall, upon 60 days notice from -Participant terminate prospectively and automatically at the expiration of such -60 day notice period, unless if within such 60 day period You withdraw Your -claim with respect to the Participant Software against such Participant either -unilaterally or pursuant to a written agreement with Participant.  - -6.3. In -the event of termination under Sections 6.1 or 6.2 above, all end user licenses -that have been validly granted by You or any distributor hereunder prior to -termination (excluding licenses granted to You by any distributor) shall survive -termination.  - -7. LIMITATION OF LIABILITY. -UNDER NO CIRCUMSTANCES AND UNDER -NO LEGAL THEORY, WHETHER TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, -SHALL YOU, THE INITIAL DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF -COVERED SOFTWARE, OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY -PERSON FOR ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY -CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOST PROFITS, LOSS OF -GOODWILL, WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER -COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE -POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF LIABILITY SHALL NOT APPLY TO -LIABILITY FOR DEATH OR PERSONAL INJURY RESULTING FROM SUCH PARTYS NEGLIGENCE TO -THE EXTENT APPLICABLE LAW PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT -ALLOW THE EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO -THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.  - -8. U.S. GOVERNMENT END USERS. - -The Covered Software is a commercial item, as that term is defined in -48 C.F.R. 2.101 (Oct. 1995), consisting of commercial computer software (as -that term is defined at 48 C.F.R.  252.227-7014(a)(1)) and commercial computer -software documentation as such terms are used in 48 C.F.R. 12.212 (Sept. -1995). Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through -227.7202-4 (June 1995), all U.S. Government End Users acquire Covered Software -with only those rights set forth herein. This U.S. Government Rights clause is -in lieu of, and supersedes, any other FAR, DFAR, or other clause or provision -that addresses Government rights in computer software under this -License.  - -9. MISCELLANEOUS. -This License represents the complete agreement -concerning subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdictions conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software.  - -10. RESPONSIBILITY FOR CLAIMS. -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability.  - -The binary distribution of this product bundles these dependencies under the -following license: -Jersey 1.9 -JAXB API bundle for GlassFish V3 2.2.2 -JAXB RI 2.2.3 --------------------------------------------------------------------------------- -COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL)Version 1.1 - -1. Definitions. - -1.1. “Contributor” means each individual or entity that creates or -contributes to the creation of Modifications. -1.2. “Contributor Version” means the combination of the Original Software, -prior Modifications used by a Contributor (if any), and the Modifications made -by that particular Contributor. -1.3. “Covered Software” means (a) the Original Software, or (b) -Modifications, or (c) the combination of files containing Original Software with -files containing Modifications, in each case including portions thereof. -1.4. “Executable” means the Covered Software in any form other than Source -Code. -1.5. “Initial Developer” means the individual or entity that first makes -Original Software available under this License. -1.6. “Larger Work” means a work which combines Covered Software or portions -thereof with code not governed by the terms of this License. -1.7. “License” means this document. -1.8. “Licensable” means having the right to grant, to the maximum extent -possible, whether at the time of the initial grant or subsequently acquired, any -and all of the rights conveyed herein. -1.9. “Modifications” means the Source Code and Executable form of any of the -following: -A. Any file that results from an addition to, deletion from or modification of -the contents of a file containing Original Software or previous Modifications; -B. Any new file that contains any part of the Original Software or previous -Modification; or -C. Any new file that is contributed or otherwise made available under the terms -of this License. -1.10. “Original Software” means the Source Code and Executable form of -computer software code that is originally released under this License. -1.11. “Patent Claims” means any patent claim(s), now owned or hereafter -acquired, including without limitation, method, process, and apparatus claims, -in any patent Licensable by grantor. -1.12. “Source Code” means (a) the common form of computer software code in -which modifications are made and (b) associated documentation included in or -with such code. -1.13. “You” (or “Your”) means an individual or a legal entity exercising -rights under, and complying with all of the terms of, this License. For legal -entities, “You” includes any entity which controls, is controlled by, or is -under common control with You. For purposes of this definition, “control” -means (a) the power, direct or indirect, to cause the direction or management of -such entity, whether by contract or otherwise, or (b) ownership of more than -fifty percent (50%) of the outstanding shares or beneficial ownership of such -entity. - -2.1. The Initial Developer Grant. - -Conditioned upon Your compliance with Section 3.1 below and subject to -third party intellectual property claims, the Initial Developer hereby grants -You a world-wide, royalty-free, non-exclusive license: -(a) under intellectual -property rights (other than patent or trademark) Licensable by Initial -Developer, to use, reproduce, modify, display, perform, sublicense and -distribute the Original Software (or portions thereof), with or without -Modifications, and/or as part of a Larger Work; and -(b) under Patent Claims -infringed by the making, using or selling of Original Software, to make, have -made, use, practice, sell, and offer for sale, and/or otherwise dispose of the -Original Software (or portions thereof). -(c) The licenses granted in Sections -2.1(a) and (b) are effective on the date Initial Developer first distributes or -otherwise makes the Original Software available to a third party under the terms -of this License. -(d) Notwithstanding Section 2.1(b) above, no patent license is -granted: (1) for code that You delete from the Original Software, or (2) for -infringements caused by: (i) the modification of the Original Software, or (ii) -the combination of the Original Software with other software or devices. - -2.2. Contributor Grant. - -Conditioned upon Your compliance with Section 3.1 below and -subject to third party intellectual property claims, each Contributor hereby -grants You a world-wide, royalty-free, non-exclusive license: -(a) under -intellectual property rights (other than patent or trademark) Licensable by -Contributor to use, reproduce, modify, display, perform, sublicense and -distribute the Modifications created by such Contributor (or portions thereof), -either on an unmodified basis, with other Modifications, as Covered Software -and/or as part of a Larger Work; and -(b) under Patent Claims infringed by the -making, using, or selling of Modifications made by that Contributor either alone -and/or in combination with its Contributor Version (or portions of such -combination), to make, use, sell, offer for sale, have made, and/or otherwise -dispose of: (1) Modifications made by that Contributor (or portions thereof); -and (2) the combination of Modifications made by that Contributor with its -Contributor Version (or portions of such combination). -(c) The licenses granted -in Sections 2.2(a) and 2.2(b) are effective on the date Contributor first -distributes or otherwise makes the Modifications available to a third -party. -(d) Notwithstanding Section 2.2(b) above, no patent license is granted: -(1) for any code that Contributor has deleted from the Contributor Version; (2) -for infringements caused by: (i) third party modifications of Contributor -Version, or (ii) the combination of Modifications made by that Contributor with -other software (except as part of the Contributor Version) or other devices; or -(3) under Patent Claims infringed by Covered Software in the absence of -Modifications made by that Contributor. - -3. Distribution Obligations. - -3.1. Availability of Source Code. -Any Covered Software that You distribute or -otherwise make available in Executable form must also be made available in -Source Code form and that Source Code form must be distributed only under the -terms of this License. You must include a copy of this License with every copy -of the Source Code form of the Covered Software You distribute or otherwise make -available. You must inform recipients of any such Covered Software in Executable -form as to how they can obtain such Covered Software in Source Code form in a -reasonable manner on or through a medium customarily used for software -exchange. -3.2. Modifications. -The Modifications that You create or to which -You contribute are governed by the terms of this License. You represent that You -believe Your Modifications are Your original creation(s) and/or You have -sufficient rights to grant the rights conveyed by this License. -3.3. Required Notices. -You must include a notice in each of Your Modifications that -identifies You as the Contributor of the Modification. You may not remove or -alter any copyright, patent or trademark notices contained within the Covered -Software, or any notices of licensing or any descriptive text giving attribution -to any Contributor or the Initial Developer. -3.4. Application of Additional Terms. -You may not offer or impose any terms on any Covered Software in Source -Code form that alters or restricts the applicable version of this License or the -recipients' rights hereunder. You may choose to offer, and to charge a fee for, -warranty, support, indemnity or liability obligations to one or more recipients -of Covered Software. However, you may do so only on Your own behalf, and not on -behalf of the Initial Developer or any Contributor. You must make it absolutely -clear that any such warranty, support, indemnity or liability obligation is -offered by You alone, and You hereby agree to indemnify the Initial Developer -and every Contributor for any liability incurred by the Initial Developer or -such Contributor as a result of warranty, support, indemnity or liability terms -You offer. -3.5. Distribution of Executable Versions. -You may distribute the -Executable form of the Covered Software under the terms of this License or under -the terms of a license of Your choice, which may contain terms different from -this License, provided that You are in compliance with the terms of this License -and that the license for the Executable form does not attempt to limit or alter -the recipient's rights in the Source Code form from the rights set forth in -this License. If You distribute the Covered Software in Executable form under a -different license, You must make it absolutely clear that any terms which differ -from this License are offered by You alone, not by the Initial Developer or -Contributor. You hereby agree to indemnify the Initial Developer and every -Contributor for any liability incurred by the Initial Developer or such -Contributor as a result of any such terms You offer. -3.6. Larger Works. -You -may create a Larger Work by combining Covered Software with other code not -governed by the terms of this License and distribute the Larger Work as a single -product. In such a case, You must make sure the requirements of this License are -fulfilled for the Covered Software. - -4. Versions of the License. - -4.1. New Versions. -Oracle is the initial license steward and may publish revised and/or -new versions of this License from time to time. Each version will be given a -distinguishing version number. Except as provided in Section 4.3, no one other -than the license steward has the right to modify this License. -4.2. Effect of New Versions. -You may always continue to use, distribute or otherwise make the -Covered Software available under the terms of the version of the License under -which You originally received the Covered Software. If the Initial Developer -includes a notice in the Original Software prohibiting it from being distributed -or otherwise made available under any subsequent version of the License, You -must distribute and make the Covered Software available under the terms of the -version of the License under which You originally received the Covered Software. -Otherwise, You may also choose to use, distribute or otherwise make the Covered -Software available under the terms of any subsequent version of the License -published by the license steward. -4.3. Modified Versions. -When You are an -Initial Developer and You want to create a new license for Your Original -Software, You may create and use a modified version of this License if You: (a) -rename the license and remove any references to the name of the license steward -(except to note that the license differs from this License); and (b) otherwise -make it clear that the license contains terms which differ from this -License. - -COVERED SOFTWARE IS PROVIDED UNDER THIS -LICENSE ON AN “AS IS” BASIS, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, WITHOUT LIMITATION, WARRANTIES THAT THE COVERED SOFTWARE -IS FREE OF DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR -NON-INFRINGING. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED -SOFTWARE IS WITH YOU. SHOULD ANY COVERED SOFTWARE PROVE DEFECTIVE IN ANY -RESPECT, YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE -COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER OF -WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF ANY COVERED -SOFTWARE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER. - -6. TERMINATION. - -6.1. This License and the rights granted hereunder will -terminate automatically if You fail to comply with terms herein and fail to cure -such breach within 30 days of becoming aware of the breach. Provisions which, by -their nature, must remain in effect beyond the termination of this License shall -survive. -6.2. If You assert a patent infringement claim (excluding declaratory -judgment actions) against Initial Developer or a Contributor (the Initial -Developer or Contributor against whom You assert such claim is referred to as -“Participant”) alleging that the Participant Software (meaning the -Contributor Version where the Participant is a Contributor or the Original -Software where the Participant is the Initial Developer) directly or indirectly -infringes any patent, then any and all rights granted directly or indirectly to -You by such Participant, the Initial Developer (if the Initial Developer is not -the Participant) and all Contributors under Sections 2.1 and/or 2.2 of this -License shall, upon 60 days notice from Participant terminate prospectively and -automatically at the expiration of such 60 day notice period, unless if within -such 60 day period You withdraw Your claim with respect to the Participant -Software against such Participant either unilaterally or pursuant to a written -agreement with Participant. -6.3. If You assert a patent infringement claim -against Participant alleging that the Participant Software directly or -indirectly infringes any patent where such claim is resolved (such as by license -or settlement) prior to the initiation of patent infringement litigation, then -the reasonable value of the licenses granted by such Participant under Sections -2.1 or 2.2 shall be taken into account in determining the amount or value of any -payment or license. -6.4. In the event of termination under Sections 6.1 or 6.2 -above, all end user licenses that have been validly granted by You or any -distributor hereunder prior to termination (excluding licenses granted to You by -any distributor) shall survive termination. - -7. LIMITATION OF LIABILITY. - -UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT -(INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL -DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED SOFTWARE, OR ANY -SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR ANY INDIRECT, -SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY CHARACTER INCLUDING, -WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL, WORK STOPPAGE, COMPUTER -FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR LOSSES, EVEN -IF SUCH PARTY SHALL HAVE BEEN INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS -LIMITATION OF LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL -INJURY RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW -PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE EXCLUSION OR -LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS EXCLUSION AND -LIMITATION MAY NOT APPLY TO YOU. - -The Covered -Software is a “commercial item,” as that term is defined in 48 C.F.R. 2.101 -(Oct. 1995), consisting of “commercial computer software” (as that term is -defined at 48 C.F.R. § 252.227-7014(a)(1)) and “commercial computer software -documentation” as such terms are used in 48 C.F.R. 12.212 (Sept. 1995). -Consistent with 48 C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 -(June 1995), all U.S. Government End Users acquire Covered Software with only -those rights set forth herein. This U.S. Government Rights clause is in lieu of, -and supersedes, any other FAR, DFAR, or other clause or provision that addresses -Government rights in computer software under this License. - -9. MISCELLANEOUS. - -This License represents the complete agreement concerning -subject matter hereof. If any provision of this License is held to be -unenforceable, such provision shall be reformed only to the extent necessary to -make it enforceable. This License shall be governed by the law of the -jurisdiction specified in a notice contained within the Original Software -(except to the extent applicable law, if any, provides otherwise), excluding -such jurisdiction's conflict-of-law provisions. Any litigation relating to this -License shall be subject to the jurisdiction of the courts located in the -jurisdiction and venue specified in a notice contained within the Original -Software, with the losing party responsible for costs, including, without -limitation, court costs and reasonable attorneys' fees and expenses. The -application of the United Nations Convention on Contracts for the International -Sale of Goods is expressly excluded. Any law or regulation which provides that -the language of a contract shall be construed against the drafter shall not -apply to this License. You agree that You alone are responsible for compliance -with the United States export administration regulations (and the export control -laws and regulation of any other countries) when You use, distribute or -otherwise make available any Covered Software. - -10. RESPONSIBILITY FOR CLAIMS. - -As between Initial Developer and the Contributors, each party is -responsible for claims and damages arising, directly or indirectly, out of its -utilization of rights under this License and You agree to work with Initial -Developer and Contributors to distribute such responsibility on an equitable -basis. Nothing herein is intended or shall be deemed to constitute any admission -of liability. - -The binary distribution of this product bundles these dependencies under the -following license: -Protocol Buffer Java API 2.5.0 --------------------------------------------------------------------------------- -This license applies to all parts of Protocol Buffers except the following: - - - Atomicops support for generic gcc, located in - src/google/protobuf/stubs/atomicops_internals_generic_gcc.h. - This file is copyrighted by Red Hat Inc. - - - Atomicops support for AIX/POWER, located in - src/google/protobuf/stubs/atomicops_internals_power.h. - This file is copyrighted by Bloomberg Finance LP. - -Copyright 2014, Google Inc. All rights reserved. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - -For: -XML Commons External Components XML APIs 1.3.04 --------------------------------------------------------------------------------- -By obtaining, using and/or copying this work, you (the licensee) agree that you -have read, understood, and will comply with the following terms and conditions. - -Permission to copy, modify, and distribute this software and its documentation, -with or without modification, for any purpose and without fee or royalty is -hereby granted, provided that you include the following on ALL copies of the -software and documentation or portions thereof, including modifications: -- The full text of this NOTICE in a location viewable to users of the -redistributed or derivative work. -- Any pre-existing intellectual property disclaimers, notices, or terms and -conditions. If none exist, the W3C Software Short Notice should be included -(hypertext is preferred, text is permitted) within the body of any redistributed -or derivative code. -- Notice of any changes or modifications to the files, including the date changes -were made. (We recommend you provide URIs to the location from which the code is -derived.) - -The binary distribution of this product bundles these dependencies under the -following license: -JUnit 4.11 -ecj-4.3.1.jar --------------------------------------------------------------------------------- -Eclipse Public License - v 1.0 - -THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC -LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM -CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. - -1. DEFINITIONS - -"Contribution" means: - -a) in the case of the initial Contributor, the initial code and documentation -distributed under this Agreement, and -b) in the case of each subsequent Contributor: -i) changes to the Program, and -ii) additions to the Program; -where such changes and/or additions to the Program originate from and are -distributed by that particular Contributor. A Contribution 'originates' from a -Contributor if it was added to the Program by such Contributor itself or anyone -acting on such Contributor's behalf. Contributions do not include additions to -the Program which: (i) are separate modules of software distributed in -conjunction with the Program under their own license agreement, and (ii) are not -derivative works of the Program. -"Contributor" means any person or entity that distributes the Program. - -"Licensed Patents" mean patent claims licensable by a Contributor which are -necessarily infringed by the use or sale of its Contribution alone or when -combined with the Program. - -"Program" means the Contributions distributed in accordance with this Agreement. - -"Recipient" means anyone who receives the Program under this Agreement, -including all Contributors. - -2. GRANT OF RIGHTS - -a) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free copyright license to -reproduce, prepare derivative works of, publicly display, publicly perform, -distribute and sublicense the Contribution of such Contributor, if any, and such -derivative works, in source code and object code form. -b) Subject to the terms of this Agreement, each Contributor hereby grants -Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed -Patents to make, use, sell, offer to sell, import and otherwise transfer the -Contribution of such Contributor, if any, in source code and object code form. -This patent license shall apply to the combination of the Contribution and the -Program if, at the time the Contribution is added by the Contributor, such -addition of the Contribution causes such combination to be covered by the -Licensed Patents. The patent license shall not apply to any other combinations -which include the Contribution. No hardware per se is licensed hereunder. -c) Recipient understands that although each Contributor grants the licenses to -its Contributions set forth herein, no assurances are provided by any -Contributor that the Program does not infringe the patent or other intellectual -property rights of any other entity. Each Contributor disclaims any liability to -Recipient for claims brought by any other entity based on infringement of -intellectual property rights or otherwise. As a condition to exercising the -rights and licenses granted hereunder, each Recipient hereby assumes sole -responsibility to secure any other intellectual property rights needed, if any. -For example, if a third party patent license is required to allow Recipient to -distribute the Program, it is Recipient's responsibility to acquire that license -before distributing the Program. -d) Each Contributor represents that to its knowledge it has sufficient copyright -rights in its Contribution, if any, to grant the copyright license set forth in -this Agreement. -3. REQUIREMENTS - -A Contributor may choose to distribute the Program in object code form under its -own license agreement, provided that: - -a) it complies with the terms and conditions of this Agreement; and -b) its license agreement: -i) effectively disclaims on behalf of all Contributors all warranties and -conditions, express and implied, including warranties or conditions of title and -non-infringement, and implied warranties or conditions of merchantability and -fitness for a particular purpose; -ii) effectively excludes on behalf of all Contributors all liability for -damages, including direct, indirect, special, incidental and consequential -damages, such as lost profits; -iii) states that any provisions which differ from this Agreement are offered by -that Contributor alone and not by any other party; and -iv) states that source code for the Program is available from such Contributor, -and informs licensees how to obtain it in a reasonable manner on or through a -medium customarily used for software exchange. -When the Program is made available in source code form: - -a) it must be made available under this Agreement; and -b) a copy of this Agreement must be included with each copy of the Program. -Contributors may not remove or alter any copyright notices contained within the -Program. - -Each Contributor must identify itself as the originator of its Contribution, if -any, in a manner that reasonably allows subsequent Recipients to identify the -originator of the Contribution. - -4. COMMERCIAL DISTRIBUTION - -Commercial distributors of software may accept certain responsibilities with -respect to end users, business partners and the like. While this license is -intended to facilitate the commercial use of the Program, the Contributor who -includes the Program in a commercial product offering should do so in a manner -which does not create potential liability for other Contributors. Therefore, if -a Contributor includes the Program in a commercial product offering, such -Contributor ("Commercial Contributor") hereby agrees to defend and indemnify -every other Contributor ("Indemnified Contributor") against any losses, damages -and costs (collectively "Losses") arising from claims, lawsuits and other legal -actions brought by a third party against the Indemnified Contributor to the -extent caused by the acts or omissions of such Commercial Contributor in -connection with its distribution of the Program in a commercial product -offering. The obligations in this section do not apply to any claims or Losses -relating to any actual or alleged intellectual property infringement. In order -to qualify, an Indemnified Contributor must: a) promptly notify the Commercial -Contributor in writing of such claim, and b) allow the Commercial Contributor to -control, and cooperate with the Commercial Contributor in, the defense and any -related settlement negotiations. The Indemnified Contributor may participate in -any such claim at its own expense. - -For example, a Contributor might include the Program in a commercial product -offering, Product X. That Contributor is then a Commercial Contributor. If that -Commercial Contributor then makes performance claims, or offers warranties -related to Product X, those performance claims and warranties are such -Commercial Contributor's responsibility alone. Under this section, the -Commercial Contributor would have to defend claims against the other -Contributors related to those performance claims and warranties, and if a court -requires any other Contributor to pay any damages as a result, the Commercial -Contributor must pay those damages. - -5. NO WARRANTY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR -IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, -NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each -Recipient is solely responsible for determining the appropriateness of using and -distributing the Program and assumes all risks associated with its exercise of -rights under this Agreement , including but not limited to the risks and costs -of program errors, compliance with applicable laws, damage to or loss of data, -programs or equipment, and unavailability or interruption of operations. - -6. DISCLAIMER OF LIABILITY - -EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY -CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST -PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, -STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS -GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -7. GENERAL - -If any provision of this Agreement is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this Agreement, and without further action by the parties hereto, such -provision shall be reformed to the minimum extent necessary to make such -provision valid and enforceable. - -If Recipient institutes patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Program itself -(excluding combinations of the Program with other software or hardware) -infringes such Recipient's patent(s), then such Recipient's rights granted under -Section 2(b) shall terminate as of the date such litigation is filed. - -All Recipient's rights under this Agreement shall terminate if it fails to -comply with any of the material terms or conditions of this Agreement and does -not cure such failure in a reasonable period of time after becoming aware of -such noncompliance. If all Recipient's rights under this Agreement terminate, -Recipient agrees to cease use and distribution of the Program as soon as -reasonably practicable. However, Recipient's obligations under this Agreement -and any licenses granted by Recipient relating to the Program shall continue and -survive. - -Everyone is permitted to copy and distribute copies of this Agreement, but in -order to avoid inconsistency the Agreement is copyrighted and may only be -modified in the following manner. The Agreement Steward reserves the right to -publish new versions (including revisions) of this Agreement from time to time. -No one other than the Agreement Steward has the right to modify this Agreement. -The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation -may assign the responsibility to serve as the Agreement Steward to a suitable -separate entity. Each new version of the Agreement will be given a -distinguishing version number. The Program (including Contributions) may always -be distributed subject to the version of the Agreement under which it was -received. In addition, after a new version of the Agreement is published, -Contributor may elect to distribute the Program (including its Contributions) -under the new version. Except as expressly stated in Sections 2(a) and 2(b) -above, Recipient receives no rights or licenses to the intellectual property of -any Contributor under this Agreement, whether expressly, by implication, -estoppel or otherwise. All rights in the Program not expressly granted under -this Agreement are reserved. - -This Agreement is governed by the laws of the State of New York and the -intellectual property laws of the United States of America. No party to this -Agreement will bring a legal action under this Agreement more than one year -after the cause of action arose. Each party waives its rights to a jury trial in -any resulting litigation. - -The binary distribution of this product bundles these dependencies under the -following license: -ASM Core 3.2 -JSch 0.1.51 -ParaNamer Core 2.3 -JLine 0.9.94 -leveldbjni-all 1.8 -Hamcrest Core 1.3 -xmlenc Library 0.52 --------------------------------------------------------------------------------- -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the nor the - names of its contributors may be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY -DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The binary distribution of this product bundles these dependencies under the -following license: -FindBugs-jsr305 3.0.0 --------------------------------------------------------------------------------- -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those -of the authors and should not be interpreted as representing official policies, -either expressed or implied, of the FreeBSD Project. - -The binary distribution of this product bundles these dependencies under the -following license: -Java Concurrency in Practice book annotations 1.0 --------------------------------------------------------------------------------- -THE WORK (AS DEFINED BELOW) IS PROVIDED UNDER THE TERMS OF THIS CREATIVE COMMONS -PUBLIC LICENSE ("CCPL" OR "LICENSE"). THE WORK IS PROTECTED BY COPYRIGHT AND/OR -OTHER APPLICABLE LAW. ANY USE OF THE WORK OTHER THAN AS AUTHORIZED UNDER THIS -LICENSE OR COPYRIGHT LAW IS PROHIBITED. - -BY EXERCISING ANY RIGHTS TO THE WORK PROVIDED HERE, YOU ACCEPT AND AGREE TO BE -BOUND BY THE TERMS OF THIS LICENSE. THE LICENSOR GRANTS YOU THE RIGHTS CONTAINED -HERE IN CONSIDERATION OF YOUR ACCEPTANCE OF SUCH TERMS AND CONDITIONS. - -1. Definitions - -"Collective Work" means a work, such as a periodical issue, anthology or -encyclopedia, in which the Work in its entirety in unmodified form, along with a -number of other contributions, constituting separate and independent works in -themselves, are assembled into a collective whole. A work that constitutes a -Collective Work will not be considered a Derivative Work (as defined below) for -the purposes of this License. -"Derivative Work" means a work based upon the Work or upon the Work and other -pre-existing works, such as a translation, musical arrangement, dramatization, -fictionalization, motion picture version, sound recording, art reproduction, -abridgment, condensation, or any other form in which the Work may be recast, -transformed, or adapted, except that a work that constitutes a Collective Work -will not be considered a Derivative Work for the purpose of this License. For -the avoidance of doubt, where the Work is a musical composition or sound -recording, the synchronization of the Work in timed-relation with a moving image -("synching") will be considered a Derivative Work for the purpose of this -License. -"Licensor" means the individual or entity that offers the Work under the terms -of this License. -"Original Author" means the individual or entity who created the Work. -"Work" means the copyrightable work of authorship offered under the terms of -this License. -"You" means an individual or entity exercising rights under this License who has -not previously violated the terms of this License with respect to the Work, or -who has received express permission from the Licensor to exercise rights under -this License despite a previous violation. -2. Fair Use Rights. Nothing in this license is intended to reduce, limit, or -restrict any rights arising from fair use, first sale or other limitations on -the exclusive rights of the copyright owner under copyright law or other -applicable laws. - -3. License Grant. Subject to the terms and conditions of this License, Licensor -hereby grants You a worldwide, royalty-free, non-exclusive, perpetual (for the -duration of the applicable copyright) license to exercise the rights in the Work -as stated below: - -to reproduce the Work, to incorporate the Work into one or more Collective -Works, and to reproduce the Work as incorporated in the Collective Works; -to create and reproduce Derivative Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission the Work including as -incorporated in Collective Works; -to distribute copies or phonorecords of, display publicly, perform publicly, and -perform publicly by means of a digital audio transmission Derivative Works. -For the avoidance of doubt, where the work is a musical composition: - -Performance Royalties Under Blanket Licenses. Licensor waives the exclusive -right to collect, whether individually or via a performance rights society (e.g. -ASCAP, BMI, SESAC), royalties for the public performance or public digital -performance (e.g. webcast) of the Work. -Mechanical Rights and Statutory Royalties. Licensor waives the exclusive right -to collect, whether individually or via a music rights agency or designated -agent (e.g. Harry Fox Agency), royalties for any phonorecord You create from the -Work ("cover version") and distribute, subject to the compulsory license created -by 17 USC Section 115 of the US Copyright Act (or the equivalent in other -jurisdictions). -Webcasting Rights and Statutory Royalties. For the avoidance of doubt, where the -Work is a sound recording, Licensor waives the exclusive right to collect, -whether individually or via a performance-rights society (e.g. SoundExchange), -royalties for the public digital performance (e.g. webcast) of the Work, subject -to the compulsory license created by 17 USC Section 114 of the US Copyright Act -(or the equivalent in other jurisdictions). -The above rights may be exercised in all media and formats whether now known or -hereafter devised. The above rights include the right to make such modifications -as are technically necessary to exercise the rights in other media and formats. -All rights not expressly granted by Licensor are hereby reserved. - -4. Restrictions.The license granted in Section 3 above is expressly made subject -to and limited by the following restrictions: - -You may distribute, publicly display, publicly perform, or publicly digitally -perform the Work only under the terms of this License, and You must include a -copy of, or the Uniform Resource Identifier for, this License with every copy or -phonorecord of the Work You distribute, publicly display, publicly perform, or -publicly digitally perform. You may not offer or impose any terms on the Work -that alter or restrict the terms of this License or the recipients' exercise of -the rights granted hereunder. You may not sublicense the Work. You must keep -intact all notices that refer to this License and to the disclaimer of -warranties. You may not distribute, publicly display, publicly perform, or -publicly digitally perform the Work with any technological measures that control -access or use of the Work in a manner inconsistent with the terms of this -License Agreement. The above applies to the Work as incorporated in a Collective -Work, but this does not require the Collective Work apart from the Work itself -to be made subject to the terms of this License. If You create a Collective -Work, upon notice from any Licensor You must, to the extent practicable, remove -from the Collective Work any credit as required by clause 4(b), as requested. If -You create a Derivative Work, upon notice from any Licensor You must, to the -extent practicable, remove from the Derivative Work any credit as required by -clause 4(b), as requested. -If you distribute, publicly display, publicly perform, or publicly digitally -perform the Work or any Derivative Works or Collective Works, You must keep -intact all copyright notices for the Work and provide, reasonable to the medium -or means You are utilizing: (i) the name of the Original Author (or pseudonym, -if applicable) if supplied, and/or (ii) if the Original Author and/or Licensor -designate another party or parties (e.g. a sponsor institute, publishing entity, -journal) for attribution in Licensor's copyright notice, terms of service or by -other reasonable means, the name of such party or parties; the title of the Work -if supplied; to the extent reasonably practicable, the Uniform Resource -Identifier, if any, that Licensor specifies to be associated with the Work, -unless such URI does not refer to the copyright notice or licensing information -for the Work; and in the case of a Derivative Work, a credit identifying the use -of the Work in the Derivative Work (e.g., "French translation of the Work by -Original Author," or "Screenplay based on original Work by Original Author"). -Such credit may be implemented in any reasonable manner; provided, however, that -in the case of a Derivative Work or Collective Work, at a minimum such credit -will appear where any other comparable authorship credit appears and in a manner -at least as prominent as such other comparable authorship credit. -5. Representations, Warranties and Disclaimer - -UNLESS OTHERWISE MUTUALLY AGREED TO BY THE PARTIES IN WRITING, LICENSOR OFFERS -THE WORK AS-IS AND MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND CONCERNING -THE WORK, EXPRESS, IMPLIED, STATUTORY OR OTHERWISE, INCLUDING, WITHOUT -LIMITATION, WARRANTIES OF TITLE, MERCHANTIBILITY, FITNESS FOR A PARTICULAR -PURPOSE, NONINFRINGEMENT, OR THE ABSENCE OF LATENT OR OTHER DEFECTS, ACCURACY, -OR THE PRESENCE OF ABSENCE OF ERRORS, WHETHER OR NOT DISCOVERABLE. SOME -JURISDICTIONS DO NOT ALLOW THE EXCLUSION OF IMPLIED WARRANTIES, SO SUCH -EXCLUSION MAY NOT APPLY TO YOU. - -6. Limitation on Liability. EXCEPT TO THE EXTENT REQUIRED BY APPLICABLE LAW, IN -NO EVENT WILL LICENSOR BE LIABLE TO YOU ON ANY LEGAL THEORY FOR ANY SPECIAL, -INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR EXEMPLARY DAMAGES ARISING OUT OF THIS -LICENSE OR THE USE OF THE WORK, EVEN IF LICENSOR HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - -7. Termination - -This License and the rights granted hereunder will terminate automatically upon -any breach by You of the terms of this License. Individuals or entities who have -received Derivative Works or Collective Works from You under this License, -however, will not have their licenses terminated provided such individuals or -entities remain in full compliance with those licenses. Sections 1, 2, 5, 6, 7, -and 8 will survive any termination of this License. -Subject to the above terms and conditions, the license granted here is perpetual -(for the duration of the applicable copyright in the Work). Notwithstanding the -above, Licensor reserves the right to release the Work under different license -terms or to stop distributing the Work at any time; provided, however that any -such election will not serve to withdraw this License (or any other license that -has been, or is required to be, granted under the terms of this License), and -this License will continue in full force and effect unless terminated as stated -above. -8. Miscellaneous - -Each time You distribute or publicly digitally perform the Work or a Collective -Work, the Licensor offers to the recipient a license to the Work on the same -terms and conditions as the license granted to You under this License. -Each time You distribute or publicly digitally perform a Derivative Work, -Licensor offers to the recipient a license to the original Work on the same -terms and conditions as the license granted to You under this License. -If any provision of this License is invalid or unenforceable under applicable -law, it shall not affect the validity or enforceability of the remainder of the -terms of this License, and without further action by the parties to this -agreement, such provision shall be reformed to the minimum extent necessary to -make such provision valid and enforceable. -No term or provision of this License shall be deemed waived and no breach -consented to unless such waiver or consent shall be in writing and signed by the -party to be charged with such waiver or consent. -This License constitutes the entire agreement between the parties with respect -to the Work licensed here. There are no understandings, agreements or -representations with respect to the Work not specified here. Licensor shall not -be bound by any additional provisions that may appear in any communication from -You. This License may not be modified without the mutual written agreement of -the Licensor and You. - -Apache Commons Collections -Copyright 2001-2015 The Apache Software Foundation - -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). - -Apache Commons Compress -Copyright 2002-2018 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). - -The files in the package org.apache.commons.compress.archivers.sevenz -were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), -which has been placed in the public domain: - -"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) - -flink-hadoop-fs -Copyright 2014-2019 The Apache Software Foundation - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). - -The binary distribution of this product bundles binaries of -org.iq80.leveldb:leveldb-api (https://github.com/dain/leveldb), which has the -following notices: -* Copyright 2011 Dain Sundstrom -* Copyright 2011 FuseSource Corp. http://fusesource.com - -The binary distribution of this product bundles binaries of -org.fusesource.hawtjni:hawtjni-runtime (https://github.com/fusesource/hawtjni), -which has the following notices: -* This product includes software developed by FuseSource Corp. - http://fusesource.com -* This product includes software developed at - Progress Software Corporation and/or its subsidiaries or affiliates. -* This product includes software developed by IBM Corporation and others. - -The binary distribution of this product bundles binaries of -AWS Java SDK 1.10.6, -which has the following notices: - * This software includes third party software subject to the following - copyrights: - XML parsing and utility functions from JetS3t - Copyright - 2006-2009 James Murty. - JSON parsing and utility functions from JSON.org - - Copyright 2002 JSON.org. - PKCS#1 PEM encoded private key parsing and utility - functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. - -The binary distribution of this product bundles binaries of -Gson 2.2.4, -which has the following notices: - - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2014 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified portion of 'Webbit', an event based -WebSocket and HTTP server, which can be obtained at: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product contains a modified portion of 'SLF4J', a simple logging -facade for Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product contains a modified portion of 'ArrayDeque', written by Josh -Bloch of Google, Inc: - - * LICENSE: - * license/LICENSE.deque.txt (Public Domain) - -This product contains a modified portion of 'Apache Harmony', an open source -Java SE, which can be obtained at: - - * LICENSE: - * license/LICENSE.harmony.txt (Apache License 2.0) - * HOMEPAGE: - * http://archive.apache.org/dist/harmony/ - -This product contains a modified version of Roland Kuhn's ASL2 -AbstractNodeQueue, which is based on Dmitriy Vyukov's non-intrusive MPSC queue. -It can be obtained at: - - * LICENSE: - * license/LICENSE.abstractnodequeue.txt (Public Domain) - * HOMEPAGE: - * https://github.com/akka/akka/blob/wip-2.2.3-for-scala-2.11/akka-actor/src/main/java/akka/dispatch/AbstractNodeQueue.java - -This product contains a modified portion of 'jbzip2', a Java bzip2 compression -and decompression library written by Matthew J. Francis. It can be obtained at: - - * LICENSE: - * license/LICENSE.jbzip2.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jbzip2/ - -This product contains a modified portion of 'libdivsufsort', a C API library to construct -the suffix array and the Burrows-Wheeler transformed string for any input string of -a constant-size alphabet written by Yuta Mori. It can be obtained at: - - * LICENSE: - * license/LICENSE.libdivsufsort.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/libdivsufsort/ - -This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, - which can be obtained at: - - * LICENSE: - * license/LICENSE.jctools.txt (ASL2 License) - * HOMEPAGE: - * https://github.com/JCTools/JCTools - -This product optionally depends on 'JZlib', a re-implementation of zlib in -pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product optionally depends on 'Compress-LZF', a Java library for encoding and -decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: - - * LICENSE: - * license/LICENSE.compress-lzf.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/ning/compress - -This product optionally depends on 'lz4', a LZ4 Java compression -and decompression library written by Adrien Grand. It can be obtained at: - - * LICENSE: - * license/LICENSE.lz4.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jpountz/lz4-java - -This product optionally depends on 'lzma-java', a LZMA Java compression -and decompression library, which can be obtained at: - - * LICENSE: - * license/LICENSE.lzma-java.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/jponge/lzma-java - -This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression -and decompression library written by William Kinney. It can be obtained at: - - * LICENSE: - * license/LICENSE.jfastlz.txt (MIT License) - * HOMEPAGE: - * https://code.google.com/p/jfastlz/ - -This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/protobuf/ - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'Snappy', a compression library produced -by Google Inc, which can be obtained at: - - * LICENSE: - * license/LICENSE.snappy.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/snappy/ - -This product optionally depends on 'JBoss Marshalling', an alternative Java -serialization API, which can be obtained at: - - * LICENSE: - * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://www.jboss.org/jbossmarshalling - -This product optionally depends on 'Caliper', Google's micro- -benchmarking framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.caliper.txt (Apache License 2.0) - * HOMEPAGE: - * http://code.google.com/p/caliper/ - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ - -This product optionally depends on 'Apache Log4J', a logging framework, which -can be obtained at: - - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ - -This product optionally depends on 'Aalto XML', an ultra-high performance -non-blocking XML processor, which can be obtained at: - - * LICENSE: - * license/LICENSE.aalto-xml.txt (Apache License 2.0) - * HOMEPAGE: - * http://wiki.fasterxml.com/AaltoHome - -This product contains a modified version of 'HPACK', a Java implementation of -the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: - - * LICENSE: - * license/LICENSE.hpack.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/twitter/hpack - -This product contains a modified portion of 'Apache Commons Lang', a Java library -provides utilities for the java.lang API, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-lang.txt (Apache License 2.0) - * HOMEPAGE: - * https://commons.apache.org/proper/commons-lang/ - -The binary distribution of this product bundles binaries of -Commons Codec 1.4, -which has the following notices: - * src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.javacontains test data from http://aspell.net/test/orig/batch0.tab.Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - =============================================================================== - The content of package org.apache.commons.codec.language.bm has been translated - from the original php source code available at http://stevemorse.org/phoneticinfo.htm - with permission from the original authors. - Original source copyright:Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -The binary distribution of this product bundles binaries of -Commons Lang 2.6, -which has the following notices: - * This product includes software from the Spring Framework,under the Apache License 2.0 (see: StringUtils.containsWhitespace()) - -The binary distribution of this product bundles binaries of -Apache Log4j 1.2.17, -which has the following notices: - * ResolverUtil.java - Copyright 2005-2006 Tim Fennell - Dumbster SMTP test server - Copyright 2004 Jason Paul Kitchen - TypeUtil.java - Copyright 2002-2012 Ramnivas Laddad, Juergen Hoeller, Chris Beams - -The binary distribution of this product bundles binaries of -Java Concurrency in Practice book annotations 1.0, -which has the following notices: - * Copyright (c) 2005 Brian Goetz and Tim Peierls Released under the Creative - Commons Attribution License (http://creativecommons.org/licenses/by/2.5) - Official home: http://www.jcip.net Any republication or derived work - distributed in source code form must include this copyright and license - notice. - -The binary distribution of this product bundles binaries of -Jetty 6.1.26, -which has the following notices: - * ============================================================== - Jetty Web Container - Copyright 1995-2016 Mort Bay Consulting Pty Ltd. - ============================================================== - - The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd - unless otherwise noted. - - Jetty is dual licensed under both - - * The Apache 2.0 License - http://www.apache.org/licenses/LICENSE-2.0.html - - and - - * The Eclipse Public 1.0 License - http://www.eclipse.org/legal/epl-v10.html - - Jetty may be distributed under either license. - - ------ - Eclipse - - The following artifacts are EPL. - * org.eclipse.jetty.orbit:org.eclipse.jdt.core - - The following artifacts are EPL and ASL2. - * org.eclipse.jetty.orbit:javax.security.auth.message - - The following artifacts are EPL and CDDL 1.0. - * org.eclipse.jetty.orbit:javax.mail.glassfish - - ------ - Oracle - - The following artifacts are CDDL + GPLv2 with classpath exception. - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - * javax.servlet:javax.servlet-api - * javax.annotation:javax.annotation-api - * javax.transaction:javax.transaction-api - * javax.websocket:javax.websocket-api - - ------ - Oracle OpenJDK - - If ALPN is used to negotiate HTTP/2 connections, then the following - artifacts may be included in the distribution or downloaded when ALPN - module is selected. - - * java.sun.security.ssl - - These artifacts replace/modify OpenJDK classes. The modififications - are hosted at github and both modified and original are under GPL v2 with - classpath exceptions. - http://openjdk.java.net/legal/gplv2+ce.html - - ------ - OW2 - - The following artifacts are licensed by the OW2 Foundation according to the - terms of http://asm.ow2.org/license.html - - org.ow2.asm:asm-commons - org.ow2.asm:asm - - ------ - Apache - - The following artifacts are ASL2 licensed. - - org.apache.taglibs:taglibs-standard-spec - org.apache.taglibs:taglibs-standard-impl - - ------ - MortBay - - The following artifacts are ASL2 licensed. Based on selected classes from - following Apache Tomcat jars, all ASL2 licensed. - - org.mortbay.jasper:apache-jsp - org.apache.tomcat:tomcat-jasper - org.apache.tomcat:tomcat-juli - org.apache.tomcat:tomcat-jsp-api - org.apache.tomcat:tomcat-el-api - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-api - org.apache.tomcat:tomcat-util-scan - org.apache.tomcat:tomcat-util - - org.mortbay.jasper:apache-el - org.apache.tomcat:tomcat-jasper-el - org.apache.tomcat:tomcat-el-api - - ------ - Mortbay - - The following artifacts are CDDL + GPLv2 with classpath exception. - - https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html - - org.eclipse.jetty.toolchain:jetty-schemas - - ------ - Assorted - - The UnixCrypt.java code implements the one way cryptography used by - Unix systems for simple password protection. Copyright 1996 Aki Yoshida, - modified April 2001 by Iris Van den Broeke, Daniel Deville. - Permission to use, copy, modify and distribute UnixCrypt - for non-commercial or commercial purposes and without fee is - granted provided that the copyright notice appears in all copies./ - -The binary distribution of this product bundles binaries of -Snappy for Java 1.0.4.1, -which has the following notices: - * This product includes software developed by Google - Snappy: http://code.google.com/p/snappy/ (New BSD License) - - This product includes software developed by Apache - PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ - (Apache 2.0 license) - - This library containd statically linked libstdc++. This inclusion is allowed by - "GCC RUntime Library Exception" - http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html - - == Contributors == - * Tatu Saloranta - * Providing benchmark suite - * Alec Wysoker - * Performance and memory usage improvement - -The binary distribution of this product bundles binaries of -Xerces2 Java Parser 2.9.1, -which has the following notices: - * ========================================================================= - == NOTICE file corresponding to section 4(d) of the Apache License, == - == Version 2.0, in this case for the Apache Xerces Java distribution. == - ========================================================================= - - Apache Xerces Java - Copyright 1999-2007 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were originally based on the following: - - software copyright (c) 1999, IBM Corporation., http://www.ibm.com. - - software copyright (c) 1999, Sun Microsystems., http://www.sun.com. - - voluntary contributions made by Paul Eng on behalf of the - Apache Software Foundation that were originally developed at iClick, Inc., - software copyright (c) 1999. - -Apache Commons CLI -Copyright 2001-2015 The Apache Software Foundation - -Apache Commons Math -Copyright 2001-2015 The Apache Software Foundation - -This product includes software developed for Orekit by -CS Systèmes d'Information (http://www.c-s.fr/) -Copyright 2010-2012 CS Systèmes d'Information - -Apache HttpClient -Copyright 1999-2017 The Apache Software Foundation - -Apache HttpCore -Copyright 2005-2017 The Apache Software Foundation - -Apache Commons Codec -Copyright 2002-2014 The Apache Software Foundation - -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -Apache Commons IO -Copyright 2002-2012 The Apache Software Foundation - -Apache Commons Net -Copyright 2001-2012 The Apache Software Foundation - -Apache Commons Lang -Copyright 2001-2011 The Apache Software Foundation - -Apache Commons Configuration -Copyright 2001-2011 The Apache Software Foundation - -Apache Commons Digester -Copyright 2001-2008 The Apache Software Foundation - -Apache Commons BeanUtils -Copyright 2000-2010 The Apache Software Foundation - -htrace-core4 -Copyright 2015 The Apache Software Foundation - -This product currently only contains code developed by authors -of specific components, as identified by the source code files; -if such notes are missing files have been created by -Tatu Saloranta. - -For additional credits (generally to people who reported problems) -see CREDITS file. - -Apache Jakarta HttpClient -Copyright 1999-2007 The Apache Software Foundation - -Apache Commons Logging -Copyright 2003-2013 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-table-uber-blink -Copyright 2014-2019 The Apache Software Foundation - -flink-table-common -Copyright 2014-2019 The Apache Software Foundation - -flink-table-api-java -Copyright 2014-2019 The Apache Software Foundation - -flink-table-api-scala -Copyright 2014-2019 The Apache Software Foundation - -flink-table-api-java-bridge -Copyright 2014-2019 The Apache Software Foundation - -flink-table-api-scala-bridge -Copyright 2014-2019 The Apache Software Foundation - -flink-table-planner-blink -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.google.guava:guava:19.0 -- com.fasterxml.jackson.core:jackson-annotations:2.9.6 -- com.fasterxml.jackson.core:jackson-core:2.9.6 -- com.fasterxml.jackson.core:jackson-databind:2.9.6 -- com.jayway.jsonpath:json-path:2.4.0 -- joda-time:joda-time:2.5 -- org.apache.calcite:calcite-core:1.20.0 -- org.apache.calcite:calcite-linq4j:1.20.0 -- org.apache.calcite.avatica:avatica-core:1.15.0 -- commons-codec:commons-codec:1.10 - -This project bundles the following dependencies under the BSD license. -See bundled license files for details - -- org.codehaus.janino:janino:3.0.9 -- org.codehaus.janino:commons-compiler:3.0.9 - -flink-sql-parser -Copyright 2014-2019 The Apache Software Foundation - -Apache Calcite Avatica -Copyright 2012-2019 The Apache Software Foundation - -Calcite Core -Copyright 2012-2019 The Apache Software Foundation - -Apache Commons Codec -Copyright 2002-2014 The Apache Software Foundation - -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -Calcite Linq4j -Copyright 2012-2019 The Apache Software Foundation - -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -flink-table-runtime-blink -Copyright 2014-2019 The Apache Software Foundation - -- org.lz4:lz4-java:1.5.0 -- org.apache.calcite.avatica:avatica-core:1.13.0 - -flink-cep -Copyright 2014-2019 The Apache Software Foundation - -// ------------------------------------------------------------------ -// NOTICE file corresponding to the section 4d of The Apache License, -// Version 2.0, in this case for Apache Flink -// ------------------------------------------------------------------ - -Apache Flink -Copyright 2006-2019 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -flink-table-uber -Copyright 2014-2019 The Apache Software Foundation - -flink-table-common -Copyright 2014-2019 The Apache Software Foundation - -flink-table-api-java -Copyright 2014-2019 The Apache Software Foundation - -flink-table-api-scala -Copyright 2014-2019 The Apache Software Foundation - -flink-table-api-java-bridge -Copyright 2014-2019 The Apache Software Foundation - -flink-table-api-scala-bridge -Copyright 2014-2019 The Apache Software Foundation - -flink-table-planner -Copyright 2014-2019 The Apache Software Foundation - -This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) - -- com.google.guava:guava:19.0 -- com.fasterxml.jackson.core:jackson-annotations:2.9.6 -- com.fasterxml.jackson.core:jackson-core:2.9.6 -- com.fasterxml.jackson.core:jackson-databind:2.9.6 -- com.jayway.jsonpath:json-path:2.4.0 -- joda-time:joda-time:2.5 -- org.apache.calcite:calcite-core:1.20.0 -- org.apache.calcite:calcite-linq4j:1.20.0 -- org.apache.calcite.avatica:avatica-core:1.15.0 -- commons-codec:commons-codec:1.10 - -This project bundles the following dependencies under the BSD license. -See bundled license files for details - -- org.codehaus.janino:janino:3.0.9 -- org.codehaus.janino:commons-compiler:3.0.9 - -flink-sql-parser -Copyright 2014-2019 The Apache Software Foundation - -Calcite Core -Copyright 2012-2019 The Apache Software Foundation - -Apache Commons Codec -Copyright 2002-2014 The Apache Software Foundation - -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. - -Apache Calcite Avatica -Copyright 2012-2019 The Apache Software Foundation - -Calcite Linq4j -Copyright 2012-2019 The Apache Software Foundation - -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. - -## Licensing - -Jackson core and extension components may licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -## Credits - -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. - -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). - -============================================================================= -= NOTICE file corresponding to section 4d of the Apache License Version 2.0 = -============================================================================= -This product includes software developed by -Joda.org (http://www.joda.org/). - -flink-cep -Copyright 2014-2019 The Apache Software Foundation - -Apache log4j -Copyright 2007 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/README.md b/README.md index c67526cbc67c5d3b15b7f70f151ab5dd1dc53c20..9733b3df759058adf1da5d50982e2f2a220c63f1 100644 --- a/README.md +++ b/README.md @@ -67,10 +67,10 @@ counts.writeAsCsv(outputPath) Prerequisites for building Flink: -* Unix-like environment (we use Linux, Mac OS X, Cygwin) +* Unix-like environment (we use Linux, Mac OS X, Cygwin, WSL) * Git * Maven (we recommend version 3.2.5 and require at least 3.1.1) -* Java 8 (Java 9 and 10 are not yet supported) +* Java 8 or 11 (Java 9 or 10 may work) ``` git clone https://github.com/apache/flink.git @@ -78,7 +78,7 @@ cd flink mvn clean package -DskipTests # this will take up to 10 minutes ``` -Flink is now installed in `build-target` +Flink is now installed in `build-target`. *NOTE: Maven 3.3.x can build Flink, but will not properly shade away certain dependencies. Maven 3.1.1 creates the libraries properly. To build unit tests with Java 8, use Java 8u51 or above to prevent failures in unit tests that use the PowerMock runner.* @@ -98,7 +98,7 @@ Minimal requirements for an IDE are: The IntelliJ IDE supports Maven out of the box and offers a plugin for Scala development. * IntelliJ download: [https://www.jetbrains.com/idea/](https://www.jetbrains.com/idea/) -* IntelliJ Scala Plugin: [http://plugins.jetbrains.com/plugin/?id=1347](http://plugins.jetbrains.com/plugin/?id=1347) +* IntelliJ Scala Plugin: [https://plugins.jetbrains.com/plugin/?id=1347](https://plugins.jetbrains.com/plugin/?id=1347) Check out our [Setting up IntelliJ](https://ci.apache.org/projects/flink/flink-docs-master/flinkDev/ide_setup.html#intellij-idea) guide for details. diff --git a/docs/_includes/generated/common_section.html b/docs/_includes/generated/common_section.html index 1c6685b2f4c6570d28e66a3e8f8d365808f48e4b..8d7a232446181bc1870292f23c6543fcd0ec8c87 100644 --- a/docs/_includes/generated/common_section.html +++ b/docs/_includes/generated/common_section.html @@ -13,9 +13,9 @@ JVM heap size for the JobManager. -
taskmanager.heap.size
- "1024m" - JVM heap size for the TaskManagers, which are the parallel workers of the system. On YARN setups, this value is automatically configured to the size of the TaskManager's YARN container, minus a certain tolerance value. +
taskmanager.memory.total-flink.size
+ (none) + Total Flink Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, except for JVM Metaspace and JVM Overhead. It consists of Framework Heap Memory, Task Heap Memory, Task Off-Heap Memory, Managed Memory, and Shuffle Memory.
parallelism.default
diff --git a/docs/_includes/generated/failure_rate_restart_strategy_configuration.html b/docs/_includes/generated/failure_rate_restart_strategy_configuration.html index 4c41f4a941ab8a535860a5bfac794ac2c0583028..589dce9855dc62cb943418910d1e0ad65395fa34 100644 --- a/docs/_includes/generated/failure_rate_restart_strategy_configuration.html +++ b/docs/_includes/generated/failure_rate_restart_strategy_configuration.html @@ -10,12 +10,12 @@
restart-strategy.failure-rate.delay
"1 s" - Delay between two consecutive restart attempts if `restart-strategy` has been set to `failure-rate`. It can be specified using Scala's `FiniteDuration` notation: "1 min", "20 s" + Delay between two consecutive restart attempts if `restart-strategy` has been set to `failure-rate`. It can be specified using notation: "1 min", "20 s"
restart-strategy.failure-rate.failure-rate-interval
"1 min" - Time interval for measuring failure rate if `restart-strategy` has been set to `failure-rate`. It can be specified using Scala's `FiniteDuration` notation: "1 min", "20 s" + Time interval for measuring failure rate if `restart-strategy` has been set to `failure-rate`. It can be specified using notation: "1 min", "20 s"
restart-strategy.failure-rate.max-failures-per-interval
diff --git a/docs/_includes/generated/fixed_delay_restart_strategy_configuration.html b/docs/_includes/generated/fixed_delay_restart_strategy_configuration.html index f9c470999a6cd76c793c7798d60a8ef1d451188a..9e11b7fa20f6387d672c65eb9932f6a8e5e2308b 100644 --- a/docs/_includes/generated/fixed_delay_restart_strategy_configuration.html +++ b/docs/_includes/generated/fixed_delay_restart_strategy_configuration.html @@ -15,7 +15,7 @@
restart-strategy.fixed-delay.delay
"1 s" - Delay between two consecutive restart attempts if `restart-strategy` has been set to `fixed-delay`. Delaying the retries can be helpful when the program interacts with external systems where for example connections or pending transactions should reach a timeout before re-execution is attempted. It can be specified using Scala's `FiniteDuration` notation: "1 min", "20 s" + Delay between two consecutive restart attempts if `restart-strategy` has been set to `fixed-delay`. Delaying the retries can be helpful when the program interacts with external systems where for example connections or pending transactions should reach a timeout before re-execution is attempted. It can be specified using notation: "1 min", "20 s" diff --git a/docs/_includes/generated/job_manager_configuration.html b/docs/_includes/generated/job_manager_configuration.html index b4ae08e050e071d1a0aea0bbe81ec11c068aea12..0a60f282adb0968a1818544f71bccab9e7d7251c 100644 --- a/docs/_includes/generated/job_manager_configuration.html +++ b/docs/_includes/generated/job_manager_configuration.html @@ -47,6 +47,11 @@ 3600 The time in seconds after which a completed job expires and is purged from the job store. + +
jobstore.max-capacity
+ 2147483647 + The max number of completed jobs that can be kept in the job store. +
slot.idle.timeout
50000 diff --git a/docs/_includes/generated/mesos_configuration.html b/docs/_includes/generated/mesos_configuration.html index 9d5be9f358026905e1174376ef81745ba2800b1f..9f951c52aab11161ffc52b978c5fe781c476cad0 100644 --- a/docs/_includes/generated/mesos_configuration.html +++ b/docs/_includes/generated/mesos_configuration.html @@ -27,6 +27,11 @@ true Enables SSL for the Flink artifact server. Note that security.ssl.enabled also needs to be set to true encryption to enable encryption. + +
mesos.resourcemanager.declined-offer-refuse-duration
+ 5000 + Amount of time to ask the Mesos master to not resend a declined resource offer again. This ensures a declined resource offer isn't resent immediately after being declined +
mesos.resourcemanager.framework.name
"Flink" @@ -57,5 +62,10 @@ (none) Comma-separated list of configuration keys which represent a configurable port. All port keys will dynamically get a port assigned through Mesos. + +
mesos.resourcemanager.unused-offer-expiration
+ 120000 + Amount of time to wait for unused expired offers before declining them. This ensures your scheduler will not hoard unuseful offers. + diff --git a/docs/_includes/generated/netty_shuffle_environment_configuration.html b/docs/_includes/generated/netty_shuffle_environment_configuration.html index 7787ed0d93114bdfd9e98c7511b07a6133fe4225..5a4f3a84621e2a65c851d5cd7ad43a637d4371db 100644 --- a/docs/_includes/generated/netty_shuffle_environment_configuration.html +++ b/docs/_includes/generated/netty_shuffle_environment_configuration.html @@ -32,21 +32,6 @@ 8 Number of extra network buffers to use for each outgoing/incoming gate (result partition/input gate). In credit-based flow control mode, this indicates how many floating credits are shared among all the input channels. The floating buffers are distributed based on backlog (real-time output buffers in the subpartition) feedback, and can help relieve back-pressure caused by unbalanced data distribution among the subpartitions. This value should be increased in case of higher round trip times between nodes and/or larger number of machines in the cluster. - -
taskmanager.network.memory.fraction
- 0.1 - Fraction of JVM memory to use for network buffers. This determines how many streaming data exchange channels a TaskManager can have at the same time and how well buffered the channels are. If a job is rejected or you get a warning that the system has not enough buffers available, increase this value or the min/max values below. Also note, that "taskmanager.network.memory.min"` and "taskmanager.network.memory.max" may override this fraction. - - -
taskmanager.network.memory.max
- "1gb" - Maximum memory size for network buffers. - - -
taskmanager.network.memory.min
- "64mb" - Minimum memory size for network buffers. -
taskmanager.network.request-backoff.initial
100 diff --git a/docs/_includes/generated/rest_v1_dispatcher.html b/docs/_includes/generated/rest_v1_dispatcher.html index 29ff267e1b3c51414e167accabf0532cecd43e72..a3b91ee2e83d9ba6cf53096344e3fbfbcff41486 100644 --- a/docs/_includes/generated/rest_v1_dispatcher.html +++ b/docs/_includes/generated/rest_v1_dispatcher.html @@ -3509,6 +3509,9 @@ Using 'curl' you can upload a jar via 'curl -X POST -H "Expect:" -F "jarfile=@pa "additionalProperties" : { "type" : "integer" } + }, + "taskmanager-id" : { + "type" : "string" } } } diff --git a/docs/_includes/generated/restart_backoff_time_strategy_configuration.html b/docs/_includes/generated/restart_backoff_time_strategy_configuration.html deleted file mode 100644 index 33785e0c7f2281a578d3f49dba1d6a542d0cd1bd..0000000000000000000000000000000000000000 --- a/docs/_includes/generated/restart_backoff_time_strategy_configuration.html +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
KeyDefaultDescription
restart-backoff-time-strategy.failure-rate.backoff-time
0Backoff time in milliseconds between two consecutive restart attempts.
restart-backoff-time-strategy.failure-rate.failure-rate-interval
60000Time interval in milliseconds for measuring failure rate.
restart-backoff-time-strategy.failure-rate.max-failures-per-interval
1Maximum number of failures in given time interval before failing a job.
restart-backoff-time-strategy.fixed-delay.attempts
2147483647Maximum number of attempts the fixed delay restart strategy will try before failing a job.
restart-backoff-time-strategy.fixed-delay.backoff-time
0Backoff time in milliseconds between two consecutive restart attempts.
diff --git a/docs/_includes/generated/restart_strategy_configuration.html b/docs/_includes/generated/restart_strategy_configuration.html index d6c8d72bcebd73f1ea4af1159443297803f0d78d..e17bbed899c47cbd922ce8f6753b1b04aadc839f 100644 --- a/docs/_includes/generated/restart_strategy_configuration.html +++ b/docs/_includes/generated/restart_strategy_configuration.html @@ -10,7 +10,7 @@
restart-strategy
(none) - Defines the restart strategy to use in case of job failures.
Accepted values are:
  • `none`, `off`, `disable`: No restart strategy.
  • `fixeddelay`, `fixed-delay`: Fixed delay restart strategy. More details can be found here.
  • `failurerate`, `failure-rate`: Failure rate restart strategy. More details can be found here.
  • `org.foobar.MyRestartStrategyFactoryFactory`: Fully qualified name of `RestartStrategyFactory` factory which has has a method `RestartStrategyFactory createFactory(Configuration configuration)`.
If checkpointing is disabled, the default value is `none`. If checkpointing is enabled, the default value is `fixed-delay` with `Integer.MAX_VALUE` restart attempts and '`1 s`' delay. + Defines the restart strategy to use in case of job failures.
Accepted values are:
  • `none`, `off`, `disable`: No restart strategy.
  • `fixeddelay`, `fixed-delay`: Fixed delay restart strategy. More details can be found here.
  • `failurerate`, `failure-rate`: Failure rate restart strategy. More details can be found here.
If checkpointing is disabled, the default value is `none`. If checkpointing is enabled, the default value is `fixed-delay` with `Integer.MAX_VALUE` restart attempts and '`1 s`' delay. diff --git a/docs/_includes/generated/task_manager_configuration.html b/docs/_includes/generated/task_manager_configuration.html index 16da6d52cd0bb4ee274c877c92d6bee5ade09fd2..ca649b526ae77e8ded9922db038119a812f70482 100644 --- a/docs/_includes/generated/task_manager_configuration.html +++ b/docs/_includes/generated/task_manager_configuration.html @@ -42,11 +42,6 @@ false Whether the quarantine monitor for task managers shall be started. The quarantine monitor shuts down the actor system if it detects that it has quarantined another actor system or if it has been quarantined by another actor system. - -
taskmanager.heap.size
- "1024m" - JVM heap size for the TaskManagers, which are the parallel workers of the system. On YARN setups, this value is automatically configured to the size of the TaskManager's YARN container, minus a certain tolerance value. -
taskmanager.host
(none) diff --git a/docs/_includes/generated/task_manager_memory_configuration.html b/docs/_includes/generated/task_manager_memory_configuration.html index c8433ed9e6e6d70cafb7249cee9d7765be381a52..e05115d4b4675fcd7571eb08148306cf4116b4bb 100644 --- a/docs/_includes/generated/task_manager_memory_configuration.html +++ b/docs/_includes/generated/task_manager_memory_configuration.html @@ -8,14 +8,49 @@ -
taskmanager.memory.fraction
- 0.7 - The relative amount of memory (after subtracting the amount of memory used by network buffers) that the task manager reserves for sorting, hash tables, and caching of intermediate results. For example, a value of `0.8` means that a task manager reserves 80% of its memory (on-heap or off-heap depending on taskmanager.memory.off-heap) for internal data buffers, leaving 20% of free memory for the task manager's heap for objects created by user-defined functions. This parameter is only evaluated, if taskmanager.memory.size is not set. +
taskmanager.memory.framework.heap.size
+ "128m" + Framework Heap Memory size for TaskExecutors. This is the size of JVM heap memory reserved for TaskExecutor framework, which will not be allocated to task slots. -
taskmanager.memory.off-heap
- false - Memory allocation method (JVM heap or off-heap), used for managed memory of the TaskManager. For setups with larger quantities of memory, this can improve the efficiency of the operations performed on the memory.
When set to true, then it is advised that `taskmanager.memory.preallocate` is also set to true. +
taskmanager.memory.jvm-metaspace.size
+ "192m" + JVM Metaspace Size for the TaskExecutors. + + +
taskmanager.memory.jvm-overhead.fraction
+ 0.1 + Fraction of Total Process Memory to be reserved for JVM Overhead. This is off-heap memory reserved for JVM overhead, such as thread stack space, I/O direct memory, compile cache, etc. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value. + + +
taskmanager.memory.jvm-overhead.max
+ "1g" + Max JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, I/O direct memory, compile cache, etc. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value. + + +
taskmanager.memory.jvm-overhead.min
+ "128m" + Min JVM Overhead size for the TaskExecutors. This is off-heap memory reserved for JVM overhead, such as thread stack space, I/O direct memory, compile cache, etc. The size of JVM Overhead is derived to make up the configured fraction of the Total Process Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of JVM Overhead can be explicitly specified by setting the min/max size to the same value. + + +
taskmanager.memory.managed.fraction
+ 0.5 + Fraction of Total Flink Memory to be used as Managed Memory, if Managed Memory size is not explicitly specified. + + +
taskmanager.memory.managed.off-heap.fraction
+ -1.0 + Fraction of Managed Memory that Off-Heap Managed Memory takes, if Off-Heap Managed Memory size is not explicitly specified. If the fraction is not explicitly specified (or configured with negative values), it will be derived from the legacy config option 'taskmanager.memory.off-heap', to use either all on-heap memory or all off-heap memory for Managed Memory. + + +
taskmanager.memory.managed.off-heap.size
+ (none) + Off-Heap Managed Memory size for TaskExecutors. This is the part of Managed Memory that is off-heap, while the remaining is on-heap. If unspecified, it will be derived to make up the configured fraction of the Managed Memory size. + + +
taskmanager.memory.managed.size
+ (none) + Managed Memory size for TaskExecutors. This is the size of memory managed by the memory manager, including both On-Heap Managed Memory and Off-Heap Managed Memory, reserved for sorting, hash tables, caching of intermediate results and state backends. Memory consumers can either allocate memory from the memory manager in the form of MemorySegments, or reserve bytes from the memory manager and keep their memory usage within that boundary. If unspecified, it will be derived to make up the configured fraction of the Total Flink Memory.
taskmanager.memory.preallocate
@@ -28,9 +63,39 @@ Size of memory buffers used by the network stack and the memory manager. -
taskmanager.memory.size
- "0" - The amount of memory (in megabytes) that the task manager reserves on-heap or off-heap (depending on taskmanager.memory.off-heap) for sorting, hash tables, and caching of intermediate results. If unspecified, the memory manager will take a fixed ratio with respect to the size of the task manager JVM as specified by taskmanager.memory.fraction. +
taskmanager.memory.shuffle.fraction
+ 0.1 + Fraction of Total Flink Memory to be used as Shuffle Memory. Shuffle Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Shuffle Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Shuffle Memory can be explicitly specified by setting the min/max size to the same value. + + +
taskmanager.memory.shuffle.max
+ "1g" + Max Shuffle Memory size for TaskExecutors. Shuffle Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Shuffle Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Shuffle Memory can be explicitly specified by setting the min/max to the same value. + + +
taskmanager.memory.shuffle.min
+ "64m" + Min Shuffle Memory size for TaskExecutors. Shuffle Memory is off-heap memory reserved for ShuffleEnvironment (e.g., network buffers). Shuffle Memory size is derived to make up the configured fraction of the Total Flink Memory. If the derived size is less/greater than the configured min/max size, the min/max size will be used. The exact size of Shuffle Memory can be explicitly specified by setting the min/max to the same value. + + +
taskmanager.memory.task.heap.size
+ (none) + Task Heap Memory size for TaskExecutors. This is the size of JVM heap memory reserved for user code. If not specified, it will be derived as Total Flink Memory minus Framework Heap Memory, Task Off-Heap Memory, (On-Heap and Off-Heap) Managed Memory and Shuffle Memory. + + +
taskmanager.memory.task.off-heap.size
+ "0b" + Task Heap Memory size for TaskExecutors. This is the size of off heap memory (JVM direct memory or native memory) reserved for user code. + + +
taskmanager.memory.total-flink.size
+ (none) + Total Flink Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, except for JVM Metaspace and JVM Overhead. It consists of Framework Heap Memory, Task Heap Memory, Task Off-Heap Memory, Managed Memory, and Shuffle Memory. + + +
taskmanager.memory.total-process.size
+ (none) + Total Process Memory size for the TaskExecutors. This includes all the memory that a TaskExecutor consumes, consisting of Total Flink Memory, JVM Metaspace, and JVM Overhead. On containerized setups, this should be set to the container memory. diff --git a/docs/concepts/glossary.zh.md b/docs/concepts/glossary.zh.md index 22f2c99ef2ef177bdfb2e83874383a5a7a763be3..5c5bd1d4cbfe6a0ccc3db4850143ac490ddb24b1 100644 --- a/docs/concepts/glossary.zh.md +++ b/docs/concepts/glossary.zh.md @@ -1,7 +1,7 @@ --- -title: Glossary +title: 词汇表 nav-pos: 3 -nav-title: Glossary +nav-title: 词汇表 nav-parent_id: concepts --- - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${project.version} - provided - true - diff --git a/flink-connectors/flink-connector-elasticsearch7/pom.xml b/flink-connectors/flink-connector-elasticsearch7/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..3c68eac4fe477e55bd938731fbea054151d7cbac --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch7/pom.xml @@ -0,0 +1,190 @@ + + + + + 4.0.0 + + + flink-connectors + org.apache.flink + 1.10-SNAPSHOT + .. + + + flink-connector-elasticsearch7_${scala.binary.version} + flink-connector-elasticsearch7 + + jar + + + + 7.3.2 + + + + + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + provided + + + + org.apache.flink + flink-connector-elasticsearch-base_${scala.binary.version} + ${project.version} + + + + org.elasticsearch + elasticsearch + + + + + + + org.elasticsearch.client + elasticsearch-rest-high-level-client + ${elasticsearch.version} + + + + + + org.apache.flink + flink-table-api-java-bridge_${scala.binary.version} + ${project.version} + provided + true + + + + + + org.apache.flink + flink-test-utils_${scala.binary.version} + ${project.version} + test + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + test + test-jar + + + + org.apache.flink + flink-connector-elasticsearch-base_${scala.binary.version} + ${project.version} + + + org.elasticsearch + elasticsearch + + + test-jar + test + + + + + + org.elasticsearch.client + transport + ${elasticsearch.version} + test + + + + org.elasticsearch.plugin + transport-netty4-client + ${elasticsearch.version} + test + + + + + + org.apache.logging.log4j + log4j-core + 2.9.1 + test + + + + + org.apache.flink + flink-table-planner_${scala.binary.version} + ${project.version} + test-jar + test + + + + + org.apache.flink + flink-json + ${project.version} + test + + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.12.2 + + + true + + + org.apache.logging.log4j:log4j-to-slf4j + + + + + + + diff --git a/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/Elasticsearch7ApiCallBridge.java b/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/Elasticsearch7ApiCallBridge.java new file mode 100644 index 0000000000000000000000000000000000000000..c569f3b8109b41393c2a29961ee99176657bc5af --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/Elasticsearch7ApiCallBridge.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.elasticsearch7; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchApiCallBridge; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; +import org.apache.flink.util.Preconditions; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.bulk.BackoffPolicy; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.common.unit.TimeValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Implementation of {@link ElasticsearchApiCallBridge} for Elasticsearch 7 and later versions. + */ +@Internal +public class Elasticsearch7ApiCallBridge implements ElasticsearchApiCallBridge { + + private static final long serialVersionUID = -5222683870097809633L; + + private static final Logger LOG = LoggerFactory.getLogger(Elasticsearch7ApiCallBridge.class); + + /** + * User-provided HTTP Host. + */ + private final List httpHosts; + + /** + * The factory to configure the rest client. + */ + private final RestClientFactory restClientFactory; + + Elasticsearch7ApiCallBridge(List httpHosts, RestClientFactory restClientFactory) { + Preconditions.checkArgument(httpHosts != null && !httpHosts.isEmpty()); + this.httpHosts = httpHosts; + this.restClientFactory = Preconditions.checkNotNull(restClientFactory); + } + + @Override + public RestHighLevelClient createClient(Map clientConfig) throws IOException { + RestClientBuilder builder = RestClient.builder(httpHosts.toArray(new HttpHost[httpHosts.size()])); + restClientFactory.configureRestClientBuilder(builder); + + RestHighLevelClient rhlClient = new RestHighLevelClient(builder); + + if (LOG.isInfoEnabled()) { + LOG.info("Pinging Elasticsearch cluster via hosts {} ...", httpHosts); + } + + if (!rhlClient.ping(RequestOptions.DEFAULT)) { + throw new RuntimeException("There are no reachable Elasticsearch nodes!"); + } + + if (LOG.isInfoEnabled()) { + LOG.info("Created Elasticsearch RestHighLevelClient connected to {}", httpHosts.toString()); + } + + return rhlClient; + } + + @Override + public BulkProcessor.Builder createBulkProcessorBuilder(RestHighLevelClient client, BulkProcessor.Listener listener) { + return BulkProcessor.builder((request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener); + } + + @Override + public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) { + if (!bulkItemResponse.isFailed()) { + return null; + } else { + return bulkItemResponse.getFailure().getCause(); + } + } + + @Override + public void configureBulkProcessorBackoff( + BulkProcessor.Builder builder, + @Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) { + + BackoffPolicy backoffPolicy; + if (flushBackoffPolicy != null) { + switch (flushBackoffPolicy.getBackoffType()) { + case CONSTANT: + backoffPolicy = BackoffPolicy.constantBackoff( + new TimeValue(flushBackoffPolicy.getDelayMillis()), + flushBackoffPolicy.getMaxRetryCount()); + break; + case EXPONENTIAL: + default: + backoffPolicy = BackoffPolicy.exponentialBackoff( + new TimeValue(flushBackoffPolicy.getDelayMillis()), + flushBackoffPolicy.getMaxRetryCount()); + } + } else { + backoffPolicy = BackoffPolicy.noBackoff(); + } + + builder.setBackoffPolicy(backoffPolicy); + } + + @Override + public RequestIndexer createBulkProcessorIndexer( + BulkProcessor bulkProcessor, + boolean flushOnCheckpoint, + AtomicLong numPendingRequestsRef) { + return new Elasticsearch7BulkProcessorIndexer( + bulkProcessor, + flushOnCheckpoint, + numPendingRequestsRef); + } +} diff --git a/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/Elasticsearch7BulkProcessorIndexer.java b/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/Elasticsearch7BulkProcessorIndexer.java new file mode 100644 index 0000000000000000000000000000000000000000..0b7a43741db5665ea6d344190ee6a1e5d843d5fa --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/Elasticsearch7BulkProcessorIndexer.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.elasticsearch7; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; + +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Implementation of a {@link RequestIndexer}, using a {@link BulkProcessor}. + * {@link ActionRequest ActionRequests} will be buffered before sending a bulk request to the Elasticsearch cluster. + * + *

Note: This class is binary compatible to Elasticsearch 7. + */ +@Internal +class Elasticsearch7BulkProcessorIndexer implements RequestIndexer { + + private final BulkProcessor bulkProcessor; + private final boolean flushOnCheckpoint; + private final AtomicLong numPendingRequestsRef; + + Elasticsearch7BulkProcessorIndexer( + BulkProcessor bulkProcessor, + boolean flushOnCheckpoint, + AtomicLong numPendingRequestsRef) { + this.bulkProcessor = checkNotNull(bulkProcessor); + this.flushOnCheckpoint = flushOnCheckpoint; + this.numPendingRequestsRef = checkNotNull(numPendingRequestsRef); + } + + @Override + public void add(DeleteRequest... deleteRequests) { + for (DeleteRequest deleteRequest : deleteRequests) { + if (flushOnCheckpoint) { + numPendingRequestsRef.getAndIncrement(); + } + this.bulkProcessor.add(deleteRequest); + } + } + + @Override + public void add(IndexRequest... indexRequests) { + for (IndexRequest indexRequest : indexRequests) { + if (flushOnCheckpoint) { + numPendingRequestsRef.getAndIncrement(); + } + this.bulkProcessor.add(indexRequest); + } + } + + @Override + public void add(UpdateRequest... updateRequests) { + for (UpdateRequest updateRequest : updateRequests) { + if (flushOnCheckpoint) { + numPendingRequestsRef.getAndIncrement(); + } + this.bulkProcessor.add(updateRequest); + } + } +} diff --git a/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/ElasticsearchSink.java b/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/ElasticsearchSink.java new file mode 100644 index 0000000000000000000000000000000000000000..5b874e29491eb7125bec18bf7fdac68df2bf252c --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/ElasticsearchSink.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.elasticsearch7; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; +import org.apache.flink.streaming.connectors.elasticsearch.util.NoOpFailureHandler; +import org.apache.flink.util.Preconditions; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.bulk.BulkProcessor; +import org.elasticsearch.client.RestHighLevelClient; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Elasticsearch 7.x sink that requests multiple {@link ActionRequest ActionRequests} + * against a cluster for each incoming element. + * + *

The sink internally uses a {@link RestHighLevelClient} to communicate with an Elasticsearch cluster. + * The sink will fail if no cluster can be connected to using the provided transport addresses passed to the constructor. + * + *

Internally, the sink will use a {@link BulkProcessor} to send {@link ActionRequest ActionRequests}. + * This will buffer elements before sending a request to the cluster. The behaviour of the + * {@code BulkProcessor} can be configured using these config keys: + *

    + *
  • {@code bulk.flush.max.actions}: Maximum amount of elements to buffer + *
  • {@code bulk.flush.max.size.mb}: Maximum amount of data (in megabytes) to buffer + *
  • {@code bulk.flush.interval.ms}: Interval at which to flush data regardless of the other two + * settings in milliseconds + *
+ * + *

You also have to provide an {@link ElasticsearchSinkFunction}. This is used to create multiple + * {@link ActionRequest ActionRequests} for each incoming element. See the class level documentation of + * {@link ElasticsearchSinkFunction} for an example. + * + * @param Type of the elements handled by this sink + */ +@PublicEvolving +public class ElasticsearchSink extends ElasticsearchSinkBase { + + private static final long serialVersionUID = 1L; + + private ElasticsearchSink( + Map bulkRequestsConfig, + List httpHosts, + ElasticsearchSinkFunction elasticsearchSinkFunction, + ActionRequestFailureHandler failureHandler, + RestClientFactory restClientFactory) { + + super(new Elasticsearch7ApiCallBridge(httpHosts, restClientFactory), bulkRequestsConfig, elasticsearchSinkFunction, failureHandler); + } + + /** + * A builder for creating an {@link ElasticsearchSink}. + * + * @param Type of the elements handled by the sink this builder creates. + */ + @PublicEvolving + public static class Builder { + + private final List httpHosts; + private final ElasticsearchSinkFunction elasticsearchSinkFunction; + + private Map bulkRequestsConfig = new HashMap<>(); + private ActionRequestFailureHandler failureHandler = new NoOpFailureHandler(); + private RestClientFactory restClientFactory = restClientBuilder -> {}; + + /** + * Creates a new {@code ElasticsearchSink} that connects to the cluster using a {@link RestHighLevelClient}. + * + * @param httpHosts The list of {@link HttpHost} to which the {@link RestHighLevelClient} connects to. + * @param elasticsearchSinkFunction This is used to generate multiple {@link ActionRequest} from the incoming element. + */ + public Builder(List httpHosts, ElasticsearchSinkFunction elasticsearchSinkFunction) { + this.httpHosts = Preconditions.checkNotNull(httpHosts); + this.elasticsearchSinkFunction = Preconditions.checkNotNull(elasticsearchSinkFunction); + } + + /** + * Sets the maximum number of actions to buffer for each bulk request. + * + * @param numMaxActions the maxinum number of actions to buffer per bulk request. + */ + public void setBulkFlushMaxActions(int numMaxActions) { + Preconditions.checkArgument( + numMaxActions > 0, + "Max number of buffered actions must be larger than 0."); + + this.bulkRequestsConfig.put(CONFIG_KEY_BULK_FLUSH_MAX_ACTIONS, String.valueOf(numMaxActions)); + } + + /** + * Sets the maximum size of buffered actions, in mb, per bulk request. + * + * @param maxSizeMb the maximum size of buffered actions, in mb. + */ + public void setBulkFlushMaxSizeMb(int maxSizeMb) { + Preconditions.checkArgument( + maxSizeMb > 0, + "Max size of buffered actions must be larger than 0."); + + this.bulkRequestsConfig.put(CONFIG_KEY_BULK_FLUSH_MAX_SIZE_MB, String.valueOf(maxSizeMb)); + } + + /** + * Sets the bulk flush interval, in milliseconds. + * + * @param intervalMillis the bulk flush interval, in milliseconds. + */ + public void setBulkFlushInterval(long intervalMillis) { + Preconditions.checkArgument( + intervalMillis >= 0, + "Interval (in milliseconds) between each flush must be larger than or equal to 0."); + + this.bulkRequestsConfig.put(CONFIG_KEY_BULK_FLUSH_INTERVAL_MS, String.valueOf(intervalMillis)); + } + + /** + * Sets whether or not to enable bulk flush backoff behaviour. + * + * @param enabled whether or not to enable backoffs. + */ + public void setBulkFlushBackoff(boolean enabled) { + this.bulkRequestsConfig.put(CONFIG_KEY_BULK_FLUSH_BACKOFF_ENABLE, String.valueOf(enabled)); + } + + /** + * Sets the type of back of to use when flushing bulk requests. + * + * @param flushBackoffType the backoff type to use. + */ + public void setBulkFlushBackoffType(FlushBackoffType flushBackoffType) { + this.bulkRequestsConfig.put( + CONFIG_KEY_BULK_FLUSH_BACKOFF_TYPE, + Preconditions.checkNotNull(flushBackoffType).toString()); + } + + /** + * Sets the maximum number of retries for a backoff attempt when flushing bulk requests. + * + * @param maxRetries the maximum number of retries for a backoff attempt when flushing bulk requests + */ + public void setBulkFlushBackoffRetries(int maxRetries) { + Preconditions.checkArgument( + maxRetries > 0, + "Max number of backoff attempts must be larger than 0."); + + this.bulkRequestsConfig.put(CONFIG_KEY_BULK_FLUSH_BACKOFF_RETRIES, String.valueOf(maxRetries)); + } + + /** + * Sets the amount of delay between each backoff attempt when flushing bulk requests, in milliseconds. + * + * @param delayMillis the amount of delay between each backoff attempt when flushing bulk requests, in milliseconds. + */ + public void setBulkFlushBackoffDelay(long delayMillis) { + Preconditions.checkArgument( + delayMillis >= 0, + "Delay (in milliseconds) between each backoff attempt must be larger than or equal to 0."); + this.bulkRequestsConfig.put(CONFIG_KEY_BULK_FLUSH_BACKOFF_DELAY, String.valueOf(delayMillis)); + } + + /** + * Sets a failure handler for action requests. + * + * @param failureHandler This is used to handle failed {@link ActionRequest}. + */ + public void setFailureHandler(ActionRequestFailureHandler failureHandler) { + this.failureHandler = Preconditions.checkNotNull(failureHandler); + } + + /** + * Sets a REST client factory for custom client configuration. + * + * @param restClientFactory the factory that configures the rest client. + */ + public void setRestClientFactory(RestClientFactory restClientFactory) { + this.restClientFactory = Preconditions.checkNotNull(restClientFactory); + } + + /** + * Creates the Elasticsearch sink. + * + * @return the created Elasticsearch sink. + */ + public ElasticsearchSink build() { + return new ElasticsearchSink<>(bulkRequestsConfig, httpHosts, elasticsearchSinkFunction, failureHandler, restClientFactory); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Builder builder = (Builder) o; + return Objects.equals(httpHosts, builder.httpHosts) && + Objects.equals(elasticsearchSinkFunction, builder.elasticsearchSinkFunction) && + Objects.equals(bulkRequestsConfig, builder.bulkRequestsConfig) && + Objects.equals(failureHandler, builder.failureHandler) && + Objects.equals(restClientFactory, builder.restClientFactory); + } + + @Override + public int hashCode() { + return Objects.hash( + httpHosts, + elasticsearchSinkFunction, + bulkRequestsConfig, + failureHandler, + restClientFactory); + } + } +} diff --git a/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/RestClientFactory.java b/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/RestClientFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..6001f43c8bf4eeb92c940f637907fa21db84d350 --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/connectors/elasticsearch7/RestClientFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.elasticsearch7; + +import org.apache.flink.annotation.PublicEvolving; + +import org.elasticsearch.client.RestClientBuilder; + +import java.io.Serializable; + +/** + * A factory that is used to configure the {@link org.elasticsearch.client.RestHighLevelClient} internally + * used in the {@link ElasticsearchSink}. + */ +@PublicEvolving +public interface RestClientFactory extends Serializable { + + /** + * Configures the rest client builder. + * + * @param restClientBuilder the configured rest client builder. + */ + void configureRestClientBuilder(RestClientBuilder restClientBuilder); + +} diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlTimeType.java b/flink-connectors/flink-connector-elasticsearch7/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/EmbeddedElasticsearchNodeEnvironmentImpl.java similarity index 32% rename from flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlTimeType.java rename to flink-connectors/flink-connector-elasticsearch7/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/EmbeddedElasticsearchNodeEnvironmentImpl.java index 23855ce7af4b10475376243dc09cc48db914251a..6f051a383bbca2180c771a2f08dd3b76646072ab 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlTimeType.java +++ b/flink-connectors/flink-connector-elasticsearch7/src/test/java/org/apache/flink/streaming/connectors/elasticsearch/EmbeddedElasticsearchNodeEnvironmentImpl.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,58 +16,64 @@ * limitations under the License. */ -package org.apache.flink.sql.parser.type; +package org.apache.flink.streaming.connectors.elasticsearch; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlWriter; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSinkITCase; + +import org.elasticsearch.client.Client; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.node.InternalSettingsPreparer; +import org.elasticsearch.node.Node; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.transport.Netty4Plugin; + +import java.io.File; +import java.util.Collections; /** - * Parse type "TIME WITHOUT TIME ZONE", "TIME(3) WITHOUT TIME ZONE", "TIME WITH LOCAL TIME ZONE", - * or "TIME(3) WITH LOCAL TIME ZONE". + * Implementation of {@link EmbeddedElasticsearchNodeEnvironment} for Elasticsearch 7. + * Will be dynamically loaded in {@link ElasticsearchSinkITCase} for integration tests. */ -public class SqlTimeType extends SqlIdentifier implements ExtendedSqlType { - private final int precision; - private final boolean withLocalTimeZone; +public class EmbeddedElasticsearchNodeEnvironmentImpl implements EmbeddedElasticsearchNodeEnvironment { - public SqlTimeType(SqlParserPos pos, int precision, boolean withLocalTimeZone) { - super(getTypeName(withLocalTimeZone), pos); - this.precision = precision; - this.withLocalTimeZone = withLocalTimeZone; - } + private Node node; - private static String getTypeName(boolean withLocalTimeZone) { - if (withLocalTimeZone) { - return SqlTypeName.TIME_WITH_LOCAL_TIME_ZONE.name(); - } else { - return SqlTypeName.TIME.name(); - } - } + @Override + public void start(File tmpDataFolder, String clusterName) throws Exception { + if (node == null) { + Settings settings = Settings.builder() + .put("cluster.name", clusterName) + .put("http.cors.enabled", true) + .put("path.home", tmpDataFolder.getParent()) + .put("path.data", tmpDataFolder.getAbsolutePath()) + .build(); - public SqlTypeName getSqlTypeName() { - if (withLocalTimeZone) { - return SqlTypeName.TIME_WITH_LOCAL_TIME_ZONE; - } else { - return SqlTypeName.TIME; + node = new PluginNode(settings); + node.start(); } } - public int getPrecision() { - return this.precision; + @Override + public void close() throws Exception { + if (node != null && !node.isClosed()) { + node.close(); + node = null; + } } @Override - public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - writer.keyword(SqlTypeName.TIME.name()); - if (this.precision >= 0) { - final SqlWriter.Frame frame = - writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "(", ")"); - writer.print(precision); - writer.endList(frame); + public Client getClient() { + if (node != null && !node.isClosed()) { + return node.client(); + } else { + return null; } - if (this.withLocalTimeZone) { - writer.keyword("WITH LOCAL TIME ZONE"); + } + + private static class PluginNode extends Node { + public PluginNode(Settings settings) { + super(InternalSettingsPreparer.prepareEnvironment(settings, Collections.emptyMap(), null, () -> "node1"), Collections.>singletonList(Netty4Plugin.class), true); } } + } diff --git a/flink-connectors/flink-connector-elasticsearch7/src/test/java/org/apache/flink/streaming/connectors/elasticsearch7/ElasticsearchSinkITCase.java b/flink-connectors/flink-connector-elasticsearch7/src/test/java/org/apache/flink/streaming/connectors/elasticsearch7/ElasticsearchSinkITCase.java new file mode 100644 index 0000000000000000000000000000000000000000..c8de4473a705ba3580ec7e6711becc96781be81e --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch7/src/test/java/org/apache/flink/streaming/connectors/elasticsearch7/ElasticsearchSinkITCase.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.connectors.elasticsearch7; + +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkTestBase; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.RestHighLevelClient; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +/** + * IT cases for the {@link ElasticsearchSink}. + * + *

The Elasticsearch ITCases for 7.x CANNOT be executed in the IDE directly, since it is required that the + * Log4J-to-SLF4J adapter dependency must be excluded from the test classpath for the Elasticsearch embedded + * node used in the tests to work properly. + */ +public class ElasticsearchSinkITCase extends ElasticsearchSinkTestBase { + + @Test + public void testElasticsearchSink() throws Exception { + runElasticsearchSinkTest(); + } + + @Test + public void testNullAddresses() throws Exception { + runNullAddressesTest(); + } + + @Test + public void testEmptyAddresses() throws Exception { + runEmptyAddressesTest(); + } + + @Test + public void testInvalidElasticsearchCluster() throws Exception{ + runInvalidElasticsearchClusterTest(); + } + + @Override + protected ElasticsearchSinkBase, RestHighLevelClient> createElasticsearchSink( + int bulkFlushMaxActions, + String clusterName, + List httpHosts, + ElasticsearchSinkFunction> elasticsearchSinkFunction) { + + ElasticsearchSink.Builder> builder = new ElasticsearchSink.Builder<>(httpHosts, elasticsearchSinkFunction); + builder.setBulkFlushMaxActions(bulkFlushMaxActions); + + return builder.build(); + } + + @Override + protected ElasticsearchSinkBase, RestHighLevelClient> createElasticsearchSinkForEmbeddedNode( + int bulkFlushMaxActions, + String clusterName, + ElasticsearchSinkFunction> elasticsearchSinkFunction) throws Exception { + + return createElasticsearchSinkForNode( + bulkFlushMaxActions, clusterName, elasticsearchSinkFunction, "127.0.0.1"); + } + + @Override + protected ElasticsearchSinkBase, RestHighLevelClient> createElasticsearchSinkForNode( + int bulkFlushMaxActions, + String clusterName, + ElasticsearchSinkFunction> elasticsearchSinkFunction, + String ipAddress) throws Exception { + + ArrayList httpHosts = new ArrayList<>(); + httpHosts.add(new HttpHost(ipAddress, 9200, "http")); + + ElasticsearchSink.Builder> builder = new ElasticsearchSink.Builder<>(httpHosts, elasticsearchSinkFunction); + builder.setBulkFlushMaxActions(bulkFlushMaxActions); + + return builder.build(); + } +} diff --git a/flink-connectors/flink-connector-elasticsearch7/src/test/resources/log4j-test.properties b/flink-connectors/flink-connector-elasticsearch7/src/test/resources/log4j-test.properties new file mode 100644 index 0000000000000000000000000000000000000000..fcd86546668477e443ec309ab172625f66f17aae --- /dev/null +++ b/flink-connectors/flink-connector-elasticsearch7/src/test/resources/log4j-test.properties @@ -0,0 +1,24 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +log4j.rootLogger=INFO, testlogger + +log4j.appender.testlogger=org.apache.log4j.ConsoleAppender +log4j.appender.testlogger.target=System.err +log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout +log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableInputFormat.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableInputFormat.java index 8a38fb35556c9373fe39a3f426dfbc4b587baf95..f00288f900ea92efb0efa86072a31e870b8cb9dc 100644 --- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableInputFormat.java +++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableInputFormat.java @@ -52,6 +52,7 @@ import java.io.ObjectOutputStream; import java.util.ArrayList; import java.util.List; import java.util.Properties; +import java.util.stream.IntStream; import static org.apache.flink.util.Preconditions.checkNotNull; import static org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR; @@ -73,9 +74,6 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase partitionColNames; //For non-partition hive table, partitions only contains one partition which partitionValues is empty. @@ -88,17 +86,25 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase partitions) { + List partitions, + int[] projectedFields) { super(jobConf.getCredentials()); checkNotNull(catalogTable, "catalogTable can not be null."); this.partitions = checkNotNull(partitions, "partitions can not be null."); this.jobConf = new JobConf(jobConf); this.partitionColNames = catalogTable.getPartitionKeys(); - rowArity = catalogTable.getSchema().getFieldCount(); + int rowArity = catalogTable.getSchema().getFieldCount(); + fields = projectedFields != null ? projectedFields : IntStream.range(0, rowArity).toArray(); } @Override @@ -137,6 +143,7 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase= structFields.size()) { + String partition = partitionColNames.get(fields[i] - structFields.size()); + reuse.setField(i, hiveTablePartition.getPartitionSpec().get(partition)); + } + } + } + rowReused = true; + } this.fetched = false; - return row; + return reuse; } // -------------------------------------------------------------------------------------------- @@ -236,9 +253,9 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase) in.readObject(); partitions = (List) in.readObject(); + fields = (int[]) in.readObject(); } } diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java index 451bd938a4d7240cd5211e44b528638210958ef2..cfa1e629fec69262398e11d980d5191d91b34892 100644 --- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java +++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java @@ -29,6 +29,7 @@ import org.apache.flink.table.catalog.hive.client.HiveMetastoreClientWrapper; import org.apache.flink.table.catalog.hive.descriptors.HiveCatalogValidator; import org.apache.flink.table.sources.InputFormatTableSource; import org.apache.flink.table.sources.PartitionableTableSource; +import org.apache.flink.table.sources.ProjectableTableSource; import org.apache.flink.table.sources.TableSource; import org.apache.flink.table.types.DataType; import org.apache.flink.table.types.logical.LogicalTypeRoot; @@ -45,6 +46,7 @@ import org.slf4j.LoggerFactory; import java.sql.Date; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -52,7 +54,7 @@ import java.util.Map; /** * A TableSource implementation to read data from Hive tables. */ -public class HiveTableSource extends InputFormatTableSource implements PartitionableTableSource { +public class HiveTableSource extends InputFormatTableSource implements PartitionableTableSource, ProjectableTableSource { private static Logger logger = LoggerFactory.getLogger(HiveTableSource.class); @@ -66,6 +68,7 @@ public class HiveTableSource extends InputFormatTableSource implements Part private Map, HiveTablePartition> partitionSpec2HiveTablePartition = new HashMap<>(); private boolean initAllPartitions; private boolean partitionPruned; + private int[] projectedFields; public HiveTableSource(JobConf jobConf, ObjectPath tablePath, CatalogTable catalogTable) { this.jobConf = Preconditions.checkNotNull(jobConf); @@ -77,18 +80,23 @@ public class HiveTableSource extends InputFormatTableSource implements Part partitionPruned = false; } + // A constructor mainly used to create copies during optimizations like partition pruning and projection push down. private HiveTableSource(JobConf jobConf, ObjectPath tablePath, CatalogTable catalogTable, List allHivePartitions, String hiveVersion, - List> partitionList) { + List> partitionList, + boolean initAllPartitions, + boolean partitionPruned, + int[] projectedFields) { this.jobConf = Preconditions.checkNotNull(jobConf); this.tablePath = Preconditions.checkNotNull(tablePath); this.catalogTable = Preconditions.checkNotNull(catalogTable); this.allHivePartitions = allHivePartitions; this.hiveVersion = hiveVersion; this.partitionList = partitionList; - this.initAllPartitions = true; - partitionPruned = true; + this.initAllPartitions = initAllPartitions; + this.partitionPruned = partitionPruned; + this.projectedFields = projectedFields; } @Override @@ -96,7 +104,7 @@ public class HiveTableSource extends InputFormatTableSource implements Part if (!initAllPartitions) { initAllPartitions(); } - return new HiveTableInputFormat(jobConf, catalogTable, allHivePartitions); + return new HiveTableInputFormat(jobConf, catalogTable, allHivePartitions, projectedFields); } @Override @@ -106,7 +114,17 @@ public class HiveTableSource extends InputFormatTableSource implements Part @Override public DataType getProducedDataType() { - return getTableSchema().toRowDataType(); + TableSchema originSchema = getTableSchema(); + if (projectedFields == null) { + return originSchema.toRowDataType(); + } + String[] names = new String[projectedFields.length]; + DataType[] types = new DataType[projectedFields.length]; + for (int i = 0; i < projectedFields.length; i++) { + names[i] = originSchema.getFieldName(projectedFields[i]).get(); + types[i] = originSchema.getFieldDataType(projectedFields[i]).get(); + } + return TableSchema.builder().fields(names, types).build().toRowDataType(); } @Override @@ -140,7 +158,8 @@ public class HiveTableSource extends InputFormatTableSource implements Part "partition spec %s", partitionSpec)); remainingHivePartitions.add(hiveTablePartition); } - return new HiveTableSource(jobConf, tablePath, catalogTable, remainingHivePartitions, hiveVersion, partitionList); + return new HiveTableSource(jobConf, tablePath, catalogTable, remainingHivePartitions, + hiveVersion, partitionList, true, true, projectedFields); } } @@ -223,7 +242,17 @@ public class HiveTableSource extends InputFormatTableSource implements Part @Override public String explainSource() { - return super.explainSource() + String.format(" TablePath: %s, PartitionPruned: %s, PartitionNums: %d", - tablePath.getFullName(), partitionPruned, null == allHivePartitions ? 0 : allHivePartitions.size()); + String explain = String.format(" TablePath: %s, PartitionPruned: %s, PartitionNums: %d", + tablePath.getFullName(), partitionPruned, null == allHivePartitions ? 0 : allHivePartitions.size()); + if (projectedFields != null) { + explain += ", ProjectedFields: " + Arrays.toString(projectedFields); + } + return super.explainSource() + explain; + } + + @Override + public TableSource projectFields(int[] fields) { + return new HiveTableSource(jobConf, tablePath, catalogTable, allHivePartitions, hiveVersion, + partitionList, initAllPartitions, partitionPruned, fields); } } diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java index 3920d0f99513ca3e1d7c4482560264a0c28bf58e..878cd686f79d46208a5a9a0cb0b13513c3c977f9 100644 --- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java +++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java @@ -181,9 +181,9 @@ public class HiveTableSourceTest { String abstractSyntaxTree = explain[1]; String optimizedLogicalPlan = explain[2]; String physicalExecutionPlan = explain[3]; - assertTrue(abstractSyntaxTree.contains("HiveTableSource(year, value, pt) TablePath: source_db.test_table_pt_1, PartitionPruned: false, PartitionNums: 2]")); - assertTrue(optimizedLogicalPlan.contains("HiveTableSource(year, value, pt) TablePath: source_db.test_table_pt_1, PartitionPruned: true, PartitionNums: 1]")); - assertTrue(physicalExecutionPlan.contains("HiveTableSource(year, value, pt) TablePath: source_db.test_table_pt_1, PartitionPruned: true, PartitionNums: 1]")); + assertTrue(abstractSyntaxTree.contains("HiveTableSource(year, value, pt) TablePath: source_db.test_table_pt_1, PartitionPruned: false, PartitionNums: 2")); + assertTrue(optimizedLogicalPlan.contains("HiveTableSource(year, value, pt) TablePath: source_db.test_table_pt_1, PartitionPruned: true, PartitionNums: 1")); + assertTrue(physicalExecutionPlan.contains("HiveTableSource(year, value, pt) TablePath: source_db.test_table_pt_1, PartitionPruned: true, PartitionNums: 1")); // second check execute results List rows = JavaConverters.seqAsJavaListConverter(TableUtil.collect((TableImpl) src)).asJava(); assertEquals(2, rows.size()); @@ -191,4 +191,35 @@ public class HiveTableSourceTest { assertArrayEquals(new String[]{"2014,3,0", "2014,4,0"}, rowStrings); } + @Test + public void testProjectionPushDown() throws Exception { + hiveShell.execute("create table src(x int,y string) partitioned by (p1 bigint, p2 string)"); + final String catalogName = "hive"; + try { + hiveShell.insertInto("default", "src") + .addRow(1, "a", 2013, "2013") + .addRow(2, "b", 2013, "2013") + .addRow(3, "c", 2014, "2014") + .commit(); + TableEnvironment tableEnv = HiveTestUtils.createTableEnv(); + tableEnv.registerCatalog(catalogName, hiveCatalog); + Table table = tableEnv.sqlQuery("select p1, count(y) from hive.`default`.src group by p1"); + String[] explain = tableEnv.explain(table).split("==.*==\n"); + assertEquals(4, explain.length); + String logicalPlan = explain[2]; + String physicalPlan = explain[3]; + String expectedExplain = + "HiveTableSource(x, y, p1, p2) TablePath: default.src, PartitionPruned: false, PartitionNums: 2, ProjectedFields: [2, 1]"; + assertTrue(logicalPlan.contains(expectedExplain)); + assertTrue(physicalPlan.contains(expectedExplain)); + + List rows = JavaConverters.seqAsJavaListConverter(TableUtil.collect((TableImpl) table)).asJava(); + assertEquals(2, rows.size()); + Object[] rowStrings = rows.stream().map(Row::toString).sorted().toArray(); + assertArrayEquals(new String[]{"2013,2", "2014,1"}, rowStrings); + } finally { + hiveShell.execute("drop table src"); + } + } + } diff --git a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java index 0932d4260e11cd1f5028be950a56420e938308ee..b2d0a96c3c53fd26bcc8310039637a02e85d434e 100644 --- a/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java +++ b/flink-connectors/flink-connector-kafka-0.11/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducer011ITCase.java @@ -49,7 +49,7 @@ import java.util.stream.IntStream; import static org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011.Semantic; import static org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011.Semantic.AT_LEAST_ONCE; import static org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer011.Semantic.EXACTLY_ONCE; -import static org.apache.flink.util.ExceptionUtils.findThrowable; +import static org.apache.flink.util.ExceptionUtils.findSerializedThrowable; import static org.apache.flink.util.Preconditions.checkState; import static org.hamcrest.Matchers.lessThan; import static org.junit.Assert.assertThat; @@ -160,7 +160,7 @@ public class FlinkKafkaProducer011ITCase extends KafkaTestBaseWithFlink { } catch (Exception ex) { // testHarness1 will be fenced off after creating and closing testHarness2 - if (!findThrowable(ex, ProducerFencedException.class).isPresent()) { + if (!findSerializedThrowable(ex, ProducerFencedException.class, ClassLoader.getSystemClassLoader()).isPresent()) { throw ex; } } @@ -664,7 +664,8 @@ public class FlinkKafkaProducer011ITCase extends KafkaTestBaseWithFlink { } private boolean isCausedBy(FlinkKafka011ErrorCode expectedErrorCode, Throwable ex) { - Optional cause = findThrowable(ex, FlinkKafka011Exception.class); + // Extract the root cause kafka exception (if any) from the serialized throwable. + Optional cause = findSerializedThrowable(ex, FlinkKafka011Exception.class, ClassLoader.getSystemClassLoader()); if (cause.isPresent()) { return cause.get().getErrorCode().equals(expectedErrorCode); } diff --git a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java index 125ece05bcc82c63ae4eb223babfafc9a4597742..82f3e5fbf9161fc77c4da5611af53baed2ae1a4b 100644 --- a/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java +++ b/flink-connectors/flink-connector-kafka-base/src/main/java/org/apache/flink/table/descriptors/Kafka.java @@ -18,6 +18,7 @@ package org.apache.flink.table.descriptors; +import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase; import org.apache.flink.streaming.connectors.kafka.config.StartupMode; import org.apache.flink.streaming.connectors.kafka.partitioner.FlinkKafkaPartitioner; @@ -50,6 +51,7 @@ import static org.apache.flink.table.descriptors.KafkaValidator.CONNECTOR_TYPE_V /** * Connector descriptor for the Apache Kafka message queue. */ +@PublicEvolving public class Kafka extends ConnectorDescriptor { private String version; diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java index 9616e9138b66e1f2fe270789dda10d568afd28a3..a971386c44f08e021741a09c265f0ac4f7b23f3b 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaProducerTestBase.java @@ -247,6 +247,9 @@ public abstract class KafkaProducerTestBase extends KafkaTestBaseWithFlink { // increase batch.size and linger.ms - this tells KafkaProducer to batch produced events instead of flushing them immediately properties.setProperty("batch.size", "10240000"); properties.setProperty("linger.ms", "10000"); + // kafka producer messages guarantee + properties.setProperty("retries", "3"); + properties.setProperty("acks", "all"); BrokerRestartingMapper.resetState(kafkaServer::blockProxyTraffic); diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java index e6fd77af64069b995520c5bacdb79cbfbe7afa30..ea940e0fc634ba98460af54306a06d0055958b64 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaShortRetentionTestBase.java @@ -82,7 +82,7 @@ public class KafkaShortRetentionTestBase implements Serializable { private static Configuration getConfiguration() { Configuration flinkConfig = new Configuration(); - flinkConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m"); + flinkConfig.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "16m"); return flinkConfig; } diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java index 5639c8270bb40da9ae6d2d43df7a3c8f86552b90..d46e21c2b0186a7f886faee4034ae923949ba4bf 100644 --- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java +++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTestBase.java @@ -117,7 +117,7 @@ public abstract class KafkaTestBase extends TestLogger { protected static Configuration getFlinkConfiguration() { Configuration flinkConfig = new Configuration(); - flinkConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m"); + flinkConfig.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "16m"); flinkConfig.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "my_reporter." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName()); return flinkConfig; } diff --git a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducerITCase.java b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducerITCase.java index 2242e811fae0acc26098acb93363ec992d40bf34..3146af476310b74621a1952ca1543733c3253653 100644 --- a/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducerITCase.java +++ b/flink-connectors/flink-connector-kafka/src/test/java/org/apache/flink/streaming/connectors/kafka/FlinkKafkaProducerITCase.java @@ -45,7 +45,7 @@ import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.apache.flink.util.ExceptionUtils.findThrowable; +import static org.apache.flink.util.ExceptionUtils.findSerializedThrowable; import static org.apache.flink.util.Preconditions.checkState; import static org.hamcrest.Matchers.lessThan; import static org.junit.Assert.assertThat; @@ -156,7 +156,7 @@ public class FlinkKafkaProducerITCase extends KafkaTestBase { } catch (Exception ex) { // testHarness1 will be fenced off after creating and closing testHarness2 - if (!findThrowable(ex, ProducerFencedException.class).isPresent()) { + if (!findSerializedThrowable(ex, ProducerFencedException.class, ClassLoader.getSystemClassLoader()).isPresent()) { throw ex; } } @@ -662,7 +662,7 @@ public class FlinkKafkaProducerITCase extends KafkaTestBase { } private boolean isCausedBy(FlinkKafkaErrorCode expectedErrorCode, Throwable ex) { - Optional cause = findThrowable(ex, FlinkKafkaException.class); + Optional cause = findSerializedThrowable(ex, FlinkKafkaException.class, ClassLoader.getSystemClassLoader()); if (cause.isPresent()) { return cause.get().getErrorCode().equals(expectedErrorCode); } diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/manualtests/ManualExactlyOnceTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/manualtests/ManualExactlyOnceTest.java index 033ead6c4cc7dd08103d1b2e8da160a26bfd9d3d..890ac305350967fc8bc1cc8cb950a7402e47a6f5 100644 --- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/manualtests/ManualExactlyOnceTest.java +++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/manualtests/ManualExactlyOnceTest.java @@ -78,7 +78,7 @@ public class ManualExactlyOnceTest { } final Configuration flinkConfig = new Configuration(); - flinkConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m"); + flinkConfig.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "16m"); flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s"); MiniClusterResource flink = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder() diff --git a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/manualtests/ManualExactlyOnceWithStreamReshardingTest.java b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/manualtests/ManualExactlyOnceWithStreamReshardingTest.java index 26a422f7f261ebf539f65cf197dad30081e4941b..8f6880a517c84d0d266c25213fd7fcb248fbc84b 100644 --- a/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/manualtests/ManualExactlyOnceWithStreamReshardingTest.java +++ b/flink-connectors/flink-connector-kinesis/src/test/java/org/apache/flink/streaming/connectors/kinesis/manualtests/ManualExactlyOnceWithStreamReshardingTest.java @@ -90,7 +90,7 @@ public class ManualExactlyOnceWithStreamReshardingTest { } final Configuration flinkConfig = new Configuration(); - flinkConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m"); + flinkConfig.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "16m"); flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s"); MiniClusterResource flink = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder() diff --git a/flink-connectors/flink-hbase/pom.xml b/flink-connectors/flink-hbase/pom.xml index 10bffabf36ecfbb2f55461a26d5a0f120b75fb17..d5237fc5041bbd4233053846620257a920e69e59 100644 --- a/flink-connectors/flink-hbase/pom.xml +++ b/flink-connectors/flink-hbase/pom.xml @@ -287,6 +287,13 @@ under the License. ${project.version} test + + org.apache.flink + flink-table-common + ${project.version} + test-jar + test + org.apache.flink flink-table-planner_${scala.binary.version} diff --git a/flink-connectors/flink-hbase/src/main/java/org/apache/flink/addons/hbase/HBaseTableSource.java b/flink-connectors/flink-hbase/src/main/java/org/apache/flink/addons/hbase/HBaseTableSource.java index b1e716110e9848ce4b8b341a611c696d043c6f13..98dfc62b34f5fc7a689c36925c6073a2e2e1c59e 100644 --- a/flink-connectors/flink-hbase/src/main/java/org/apache/flink/addons/hbase/HBaseTableSource.java +++ b/flink-connectors/flink-hbase/src/main/java/org/apache/flink/addons/hbase/HBaseTableSource.java @@ -31,12 +31,13 @@ import org.apache.flink.table.sources.BatchTableSource; import org.apache.flink.table.sources.LookupableTableSource; import org.apache.flink.table.sources.ProjectableTableSource; import org.apache.flink.table.sources.StreamTableSource; -import org.apache.flink.table.utils.TableConnectorUtils; import org.apache.flink.types.Row; import org.apache.flink.util.Preconditions; import org.apache.hadoop.conf.Configuration; +import java.util.Arrays; + /** * Creates a TableSource to scan an HBase table. * @@ -141,7 +142,8 @@ public class HBaseTableSource implements BatchTableSource, ProjectableTable @Override public String explainSource() { - return TableConnectorUtils.generateRuntimeName(this.getClass(), getTableSchema().getFieldNames()); + return "HBaseTableSource[schema=" + Arrays.toString(getTableSchema().getFieldNames()) + + ", projectFields=" + Arrays.toString(projectFields) + "]"; } @Override diff --git a/flink-connectors/flink-hbase/src/main/java/org/apache/flink/table/descriptors/HBase.java b/flink-connectors/flink-hbase/src/main/java/org/apache/flink/table/descriptors/HBase.java new file mode 100644 index 0000000000000000000000000000000000000000..9dcdff4d4345e769a749c2e321991ee1ff01345b --- /dev/null +++ b/flink-connectors/flink-hbase/src/main/java/org/apache/flink/table/descriptors/HBase.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.descriptors; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.MemorySize; +import org.apache.flink.util.TimeUtils; + +import java.util.Map; + +import static org.apache.flink.table.descriptors.ConnectorDescriptorValidator.CONNECTOR_VERSION; +import static org.apache.flink.table.descriptors.HBaseValidator.CONNECTOR_TABLE_NAME; +import static org.apache.flink.table.descriptors.HBaseValidator.CONNECTOR_TYPE_VALUE_HBASE; +import static org.apache.flink.table.descriptors.HBaseValidator.CONNECTOR_WRITE_BUFFER_FLUSH_INTERVAL; +import static org.apache.flink.table.descriptors.HBaseValidator.CONNECTOR_WRITE_BUFFER_FLUSH_MAX_ROWS; +import static org.apache.flink.table.descriptors.HBaseValidator.CONNECTOR_WRITE_BUFFER_FLUSH_MAX_SIZE; +import static org.apache.flink.table.descriptors.HBaseValidator.CONNECTOR_ZK_NODE_PARENT; +import static org.apache.flink.table.descriptors.HBaseValidator.CONNECTOR_ZK_QUORUM; + +/** + * Connector descriptor for Apache HBase. + */ +@PublicEvolving +public class HBase extends ConnectorDescriptor { + private DescriptorProperties properties = new DescriptorProperties(); + + public HBase() { + super(CONNECTOR_TYPE_VALUE_HBASE, 1, true); + } + + /** + * Set the Apache HBase version to be used. Required. + * + * @param version HBase version. E.g., "1.4.3". + */ + public HBase version(String version) { + properties.putString(CONNECTOR_VERSION, version); + return this; + } + + /** + * Set the HBase table name, Required. + * + * @param tableName Name of HBase table. E.g., "testNamespace:testTable", "testDefaultTable" + */ + public HBase tableName(String tableName) { + properties.putString(CONNECTOR_TABLE_NAME, tableName); + return this; + } + + /** + * Set the zookeeper quorum address to connect the HBase cluster. Required. + * + * @param zookeeperQuorum zookeeper quorum address to connect the HBase cluster. + * E.g., "localhost:2181,localhost:2182,localhost:2183". + */ + public HBase zookeeperQuorum(String zookeeperQuorum) { + properties.putString(CONNECTOR_ZK_QUORUM, zookeeperQuorum); + return this; + } + + /** + * Set the zookeeper node parent path of HBase cluster. Default to use "/hbase", Optional. + * + * @param zookeeperNodeParent zookeeper node path of hbase cluster. E.g, "/hbase/example-root-znode". + */ + public HBase zookeeperNodeParent(String zookeeperNodeParent) { + properties.putString(CONNECTOR_ZK_NODE_PARENT, zookeeperNodeParent); + return this; + } + + /** + * Set threshold when to flush buffered request based on the memory byte size of rows currently added. + * Default to 2mb. Optional. + * + * @param maxSize the maximum size (using the syntax of {@link MemorySize}). + */ + public HBase writeBufferFlushMaxSize(String maxSize) { + properties.putMemorySize(CONNECTOR_WRITE_BUFFER_FLUSH_MAX_SIZE, MemorySize.parse(maxSize, MemorySize.MemoryUnit.BYTES)); + return this; + } + + /** + * Set threshold when to flush buffered request based on the number of rows currently added. + * Defaults to not set, i.e. won't flush based on the number of buffered rows. Optional. + * + * @param writeBufferFlushMaxRows number of added rows when begin the request flushing. + */ + public HBase writeBufferFlushMaxRows(int writeBufferFlushMaxRows) { + properties.putInt(CONNECTOR_WRITE_BUFFER_FLUSH_MAX_ROWS, writeBufferFlushMaxRows); + return this; + } + + /** + * Set an interval when to flushing buffered requesting if the interval passes, in milliseconds. + * Defaults to not set, i.e. won't flush based on flush interval. Optional. + * + * @param interval flush interval. The string should be in format "{length value}{time unit label}" + * E.g, "123ms", "1 s", If no time unit label is specified, it will be considered as + * milliseconds. For more details about the format, please see + * {@link TimeUtils#parseDuration(String)}}. + */ + public HBase writeBufferFlushInterval(String interval) { + properties.putString(CONNECTOR_WRITE_BUFFER_FLUSH_INTERVAL, interval); + return this; + } + + @Override + protected Map toConnectorProperties() { + return properties.asMap(); + } +} diff --git a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..b9a2f5dc2a4e6ae8f531ff82c5a52d5313c47bda --- /dev/null +++ b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/HBaseDescriptorTest.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.addons.hbase; + +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.descriptors.Descriptor; +import org.apache.flink.table.descriptors.DescriptorProperties; +import org.apache.flink.table.descriptors.DescriptorTestBase; +import org.apache.flink.table.descriptors.DescriptorValidator; +import org.apache.flink.table.descriptors.HBase; +import org.apache.flink.table.descriptors.HBaseValidator; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Test case for {@link HBase} descriptor. + */ +public class HBaseDescriptorTest extends DescriptorTestBase { + + @Override + protected List descriptors() { + HBase hbaseDesc0 = new HBase() + .version("1.4.3") + .tableName("testNs:table0") + .zookeeperQuorum("localhost:2181,localhost:2182,localhost:2183") + .zookeeperNodeParent("/hbase/root-dir"); + + HBase hbaseDesc1 = new HBase() + .version("1.4.3") + .tableName("testNs:table1") + .zookeeperQuorum("localhost:2181") + .zookeeperNodeParent("/hbase/root") + .writeBufferFlushInterval("2s") + .writeBufferFlushMaxRows(100) + .writeBufferFlushMaxSize("1mb"); + + return Arrays.asList(hbaseDesc0, hbaseDesc1); + } + + @Override + protected List> properties() { + Map prop0 = new HashMap<>(); + prop0.put("connector.version", "1.4.3"); + prop0.put("connector.type", "hbase"); + prop0.put("connector.table-name", "testNs:table0"); + prop0.put("connector.zookeeper.quorum", "localhost:2181,localhost:2182,localhost:2183"); + prop0.put("connector.zookeeper.znode.parent", "/hbase/root-dir"); + prop0.put("connector.property-version", "1"); + + Map prop1 = new HashMap<>(); + prop1.put("connector.version", "1.4.3"); + prop1.put("connector.type", "hbase"); + prop1.put("connector.table-name", "testNs:table1"); + prop1.put("connector.zookeeper.quorum", "localhost:2181"); + prop1.put("connector.zookeeper.znode.parent", "/hbase/root"); + prop1.put("connector.property-version", "1"); + prop1.put("connector.write.buffer-flush.interval", "2s"); + prop1.put("connector.write.buffer-flush.max-rows", "100"); + prop1.put("connector.write.buffer-flush.max-size", "1048576 bytes"); + + return Arrays.asList(prop0, prop1); + } + + @Override + protected DescriptorValidator validator() { + return new HBaseValidator(); + } + + @Test + public void testRequiredFields() { + HBase hbaseDesc0 = new HBase(); + HBase hbaseDesc1 = new HBase() + .version("1.4.3") + .zookeeperQuorum("localhost:2181") + .zookeeperNodeParent("/hbase/root"); // no table name + HBase hbaseDesc2 = new HBase() + .version("1.4.3") + .tableName("ns:table") + .zookeeperNodeParent("/hbase/root"); // no zookeeper quorum + HBase hbaseDesc3 = new HBase() + .tableName("ns:table") + .zookeeperQuorum("localhost:2181"); // no version + + HBase[] testCases = new HBase[]{hbaseDesc0, hbaseDesc1, hbaseDesc2, hbaseDesc3}; + for (int i = 0; i < testCases.length; i++) { + HBase hbaseDesc = testCases[i]; + DescriptorProperties properties = new DescriptorProperties(); + properties.putProperties(hbaseDesc.toProperties()); + boolean caughtExpectedException = false; + try { + validator().validate(properties); + } catch (ValidationException e) { + caughtExpectedException = true; + } + Assert.assertTrue("The case#" + i + " didn't get the expected error", caughtExpectedException); + } + } +} diff --git a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseFlinkTestConstants.java b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseFlinkTestConstants.java index 57224c21a8010836c471aceafa45f3eec0e579c5..896dbbe1c73cb2372be32a5fd597b31c3e18b0da 100644 --- a/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseFlinkTestConstants.java +++ b/flink-connectors/flink-hbase/src/test/java/org/apache/flink/addons/hbase/example/HBaseFlinkTestConstants.java @@ -18,12 +18,12 @@ package org.apache.flink.addons.hbase.example; -import org.apache.flink.configuration.ConfigConstants; +import org.apache.hadoop.hbase.util.Bytes; class HBaseFlinkTestConstants { - static final byte[] CF_SOME = "someCf".getBytes(ConfigConstants.DEFAULT_CHARSET); - static final byte[] Q_SOME = "someQual".getBytes(ConfigConstants.DEFAULT_CHARSET); + static final byte[] CF_SOME = Bytes.toBytes("someCf"); + static final byte[] Q_SOME = Bytes.toBytes("someQual"); static final String TEST_TABLE_NAME = "test-table"; static final String TMP_DIR = "/tmp/test"; diff --git a/flink-connectors/flink-orc/src/main/java/org/apache/flink/orc/OrcTableSource.java b/flink-connectors/flink-orc/src/main/java/org/apache/flink/orc/OrcTableSource.java index 6e3ada4c493dc14eb3515b4482d1b617946b31c2..011e93da028f84cfde594871610d928aaabf4d18 100644 --- a/flink-connectors/flink-orc/src/main/java/org/apache/flink/orc/OrcTableSource.java +++ b/flink-connectors/flink-orc/src/main/java/org/apache/flink/orc/OrcTableSource.java @@ -213,14 +213,15 @@ public class OrcTableSource @Override public String explainSource() { - return "OrcFile[path=" + path + ", schema=" + orcSchema + ", filter=" + predicateString() + "]"; + return "OrcFile[path=" + path + ", schema=" + orcSchema + ", filter=" + predicateString() + + ", selectedFields=" + Arrays.toString(selectedFields) + "]"; } private String predicateString() { - if (predicates != null) { - return "AND(" + Arrays.toString(predicates) + ")"; - } else { + if (predicates == null || predicates.length == 0) { return "TRUE"; + } else { + return "AND(" + Arrays.toString(predicates) + ")"; } } diff --git a/flink-connectors/flink-sql-connector-elasticsearch7/pom.xml b/flink-connectors/flink-sql-connector-elasticsearch7/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..63aa212dcc0ab2dc15aedec9cde4631b0339816f --- /dev/null +++ b/flink-connectors/flink-sql-connector-elasticsearch7/pom.xml @@ -0,0 +1,208 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-connectors + 1.10-SNAPSHOT + .. + + + flink-sql-connector-elasticsearch7_${scala.binary.version} + flink-sql-connector-elasticsearch7 + + jar + + + + org.apache.flink + flink-connector-elasticsearch-base_${scala.binary.version} + ${project.version} + + + + org.elasticsearch + elasticsearch + + + + + + org.apache.flink + flink-connector-elasticsearch7_${scala.binary.version} + ${project.version} + + + + + + org.apache.flink + flink-table-api-java-bridge_${scala.binary.version} + ${project.version} + provided + true + + + + org.apache.flink + flink-connector-elasticsearch-base_${scala.binary.version} + ${project.version} + + + org.elasticsearch + elasticsearch + + + test-jar + test + + + + + org.apache.flink + flink-json + ${project.version} + test + + + + + org.apache.flink + flink-table-planner_${scala.binary.version} + ${project.version} + test-jar + test + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + false + + + *:* + + + + com.carrotsearch:hppc + com.tdunning:t-digest + joda-time:joda-time + net.sf.jopt-simple:jopt-simple + org.elasticsearch:jna + org.elasticsearch:elasticsearch-geo + org.elasticsearch.plugin:lang-mustache-client + com.github.spullara.mustache.java:compiler + org.hdrhistogram:HdrHistogram + org.yaml:snakeyaml + + + + + + org.elasticsearch:elasticsearch + + config/** + modules.txt + plugins.txt + org/joda/** + + + + org.elasticsearch.client:elasticsearch-rest-high-level-client + + forbidden/** + + + + org.apache.httpcomponents:httpclient + + mozilla/** + + + + org.apache.lucene:lucene-analyzers-common + + org/tartarus/** + + + + *:* + + + META-INF/versions/** + META-INF/services/com.fasterxml.** + META-INF/services/org.apache.lucene.** + META-INF/services/org.elasticsearch.** + META-INF/LICENSE.txt + + + + + + + org.apache.commons + org.apache.flink.elasticsearch7.shaded.org.apache.commons + + + org.apache.http + org.apache.flink.elasticsearch7.shaded.org.apache.http + + + org.apache.lucene + org.apache.flink.elasticsearch7.shaded.org.apache.lucene + + + org.elasticsearch + org.apache.flink.elasticsearch7.shaded.org.elasticsearch + + + org.apache.logging + org.apache.flink.elasticsearch7.shaded.org.apache.logging + + + com.fasterxml.jackson + org.apache.flink.elasticsearch7.shaded.com.fasterxml.jackson + + + + + + + + + + diff --git a/flink-connectors/flink-sql-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/sql/connectors/elasticsearch7/Elasticsearch7UpsertTableSink.java b/flink-connectors/flink-sql-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/sql/connectors/elasticsearch7/Elasticsearch7UpsertTableSink.java new file mode 100644 index 0000000000000000000000000000000000000000..b6cf75c9ed800c94f433b2de2d74857db85a5ddf --- /dev/null +++ b/flink-connectors/flink-sql-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/sql/connectors/elasticsearch7/Elasticsearch7UpsertTableSink.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.sql.connectors.elasticsearch7; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.configuration.MemorySize; +import org.apache.flink.streaming.api.functions.sink.SinkFunction; +import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase; +import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink; +import org.apache.flink.streaming.connectors.elasticsearch7.RestClientFactory; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.types.Row; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.delete.DeleteRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.common.xcontent.XContentType; + +import javax.annotation.Nullable; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption.BULK_FLUSH_BACKOFF_DELAY; +import static org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption.BULK_FLUSH_BACKOFF_ENABLED; +import static org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption.BULK_FLUSH_BACKOFF_RETRIES; +import static org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption.BULK_FLUSH_BACKOFF_TYPE; +import static org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption.BULK_FLUSH_INTERVAL; +import static org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption.BULK_FLUSH_MAX_ACTIONS; +import static org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption.BULK_FLUSH_MAX_SIZE; +import static org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption.DISABLE_FLUSH_ON_CHECKPOINT; +import static org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption.REST_PATH_PREFIX; + +/** + * Version-specific upsert table sink for Elasticsearch 7. + */ +@Internal +public class Elasticsearch7UpsertTableSink extends ElasticsearchUpsertTableSinkBase { + + @VisibleForTesting + static final RequestFactory UPDATE_REQUEST_FACTORY = + new Elasticsearch7RequestFactory(); + + public Elasticsearch7UpsertTableSink( + boolean isAppendOnly, + TableSchema schema, + List hosts, + String index, + String keyDelimiter, + String keyNullLiteral, + SerializationSchema serializationSchema, + XContentType contentType, + ActionRequestFailureHandler failureHandler, + Map sinkOptions) { + + super( + isAppendOnly, + schema, + hosts, + index, + "", + keyDelimiter, + keyNullLiteral, + serializationSchema, + contentType, + failureHandler, + sinkOptions, + UPDATE_REQUEST_FACTORY); + } + + @VisibleForTesting + Elasticsearch7UpsertTableSink( + boolean isAppendOnly, + TableSchema schema, + List hosts, + String index, + String docType, + String keyDelimiter, + String keyNullLiteral, + SerializationSchema serializationSchema, + XContentType contentType, + ActionRequestFailureHandler failureHandler, + Map sinkOptions) { + + super( + isAppendOnly, + schema, + hosts, + index, + docType, + keyDelimiter, + keyNullLiteral, + serializationSchema, + contentType, + failureHandler, + sinkOptions, + UPDATE_REQUEST_FACTORY); + } + + @Override + protected ElasticsearchUpsertTableSinkBase copy( + boolean isAppendOnly, + TableSchema schema, + List hosts, + String index, + String docType, + String keyDelimiter, + String keyNullLiteral, + SerializationSchema serializationSchema, + XContentType contentType, + ActionRequestFailureHandler failureHandler, + Map sinkOptions, + RequestFactory requestFactory) { + + return new Elasticsearch7UpsertTableSink( + isAppendOnly, + schema, + hosts, + index, + keyDelimiter, + keyNullLiteral, + serializationSchema, + contentType, + failureHandler, + sinkOptions); + } + + @Override + protected SinkFunction> createSinkFunction( + List hosts, + ActionRequestFailureHandler failureHandler, + Map sinkOptions, + ElasticsearchUpsertSinkFunction upsertSinkFunction) { + + final List httpHosts = hosts.stream() + .map((host) -> new HttpHost(host.hostname, host.port, host.protocol)) + .collect(Collectors.toList()); + + final ElasticsearchSink.Builder> builder = createBuilder(upsertSinkFunction, httpHosts); + + builder.setFailureHandler(failureHandler); + + Optional.ofNullable(sinkOptions.get(BULK_FLUSH_MAX_ACTIONS)) + .ifPresent(v -> builder.setBulkFlushMaxActions(Integer.valueOf(v))); + + Optional.ofNullable(sinkOptions.get(BULK_FLUSH_MAX_SIZE)) + .ifPresent(v -> builder.setBulkFlushMaxSizeMb(MemorySize.parse(v).getMebiBytes())); + + Optional.ofNullable(sinkOptions.get(BULK_FLUSH_INTERVAL)) + .ifPresent(v -> builder.setBulkFlushInterval(Long.valueOf(v))); + + Optional.ofNullable(sinkOptions.get(BULK_FLUSH_BACKOFF_ENABLED)) + .ifPresent(v -> builder.setBulkFlushBackoff(Boolean.valueOf(v))); + + Optional.ofNullable(sinkOptions.get(BULK_FLUSH_BACKOFF_TYPE)) + .ifPresent(v -> builder.setBulkFlushBackoffType(ElasticsearchSinkBase.FlushBackoffType.valueOf(v))); + + Optional.ofNullable(sinkOptions.get(BULK_FLUSH_BACKOFF_RETRIES)) + .ifPresent(v -> builder.setBulkFlushBackoffRetries(Integer.valueOf(v))); + + Optional.ofNullable(sinkOptions.get(BULK_FLUSH_BACKOFF_DELAY)) + .ifPresent(v -> builder.setBulkFlushBackoffDelay(Long.valueOf(v))); + + builder.setRestClientFactory( + new DefaultRestClientFactory(sinkOptions.get(REST_PATH_PREFIX))); + + final ElasticsearchSink> sink = builder.build(); + + Optional.ofNullable(sinkOptions.get(DISABLE_FLUSH_ON_CHECKPOINT)) + .ifPresent(v -> { + if (Boolean.valueOf(v)) { + sink.disableFlushOnCheckpoint(); + } + }); + + return sink; + } + + @VisibleForTesting + ElasticsearchSink.Builder> createBuilder( + ElasticsearchUpsertSinkFunction upsertSinkFunction, + List httpHosts) { + return new ElasticsearchSink.Builder<>(httpHosts, upsertSinkFunction); + } + + // -------------------------------------------------------------------------------------------- + // Helper classes + // -------------------------------------------------------------------------------------------- + + /** + * Serializable {@link RestClientFactory} used by the sink. + */ + @VisibleForTesting + static class DefaultRestClientFactory implements RestClientFactory { + + private String pathPrefix; + + public DefaultRestClientFactory(@Nullable String pathPrefix) { + this.pathPrefix = pathPrefix; + } + + @Override + public void configureRestClientBuilder(RestClientBuilder restClientBuilder) { + if (pathPrefix != null) { + restClientBuilder.setPathPrefix(pathPrefix); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DefaultRestClientFactory that = (DefaultRestClientFactory) o; + return Objects.equals(pathPrefix, that.pathPrefix); + } + + @Override + public int hashCode() { + return Objects.hash(pathPrefix); + } + } + + /** + * Version-specific creation of {@link org.elasticsearch.action.ActionRequest}s used by the sink. + */ + private static class Elasticsearch7RequestFactory implements RequestFactory { + + @Override + public UpdateRequest createUpdateRequest( + String index, + String docType, + String key, + XContentType contentType, + byte[] document) { + return new UpdateRequest(index, key) + .doc(document, contentType) + .upsert(document, contentType); + } + + @Override + public IndexRequest createIndexRequest( + String index, + String docType, + XContentType contentType, + byte[] document) { + return new IndexRequest(index) + .source(document, contentType); + } + + @Override + public DeleteRequest createDeleteRequest(String index, String docType, String key) { + return new DeleteRequest(index, key); + } + } +} diff --git a/flink-connectors/flink-sql-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/sql/connectors/elasticsearch7/Elasticsearch7UpsertTableSinkFactory.java b/flink-connectors/flink-sql-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/sql/connectors/elasticsearch7/Elasticsearch7UpsertTableSinkFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..ca2f5449916986ef516e117f9a3f2a01e090eb64 --- /dev/null +++ b/flink-connectors/flink-sql-connector-elasticsearch7/src/main/java/org/apache/flink/streaming/sql/connectors/elasticsearch7/Elasticsearch7UpsertTableSinkFactory.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.sql.connectors.elasticsearch7; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.Host; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkFactoryBase; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.sinks.UpsertStreamTableSink; +import org.apache.flink.types.Row; + +import org.elasticsearch.common.xcontent.XContentType; + +import java.util.List; +import java.util.Map; + +import static org.apache.flink.table.descriptors.ElasticsearchValidator.CONNECTOR_VERSION_VALUE_7; + +/** + * Table factory for creating an {@link UpsertStreamTableSink} for Elasticsearch 7. + */ +@Internal +public class Elasticsearch7UpsertTableSinkFactory extends ElasticsearchUpsertTableSinkFactoryBase { + + @Override + protected String elasticsearchVersion() { + return CONNECTOR_VERSION_VALUE_7; + } + + @Override + protected ElasticsearchUpsertTableSinkBase createElasticsearchUpsertTableSink( + boolean isAppendOnly, + TableSchema schema, + List hosts, + String index, + String docType, + String keyDelimiter, + String keyNullLiteral, + SerializationSchema serializationSchema, + XContentType contentType, + ActionRequestFailureHandler failureHandler, + Map sinkOptions) { + + return new Elasticsearch7UpsertTableSink( + isAppendOnly, + schema, + hosts, + index, + keyDelimiter, + keyNullLiteral, + serializationSchema, + contentType, + failureHandler, + sinkOptions); + } +} diff --git a/flink-connectors/flink-sql-connector-elasticsearch7/src/main/resources/META-INF/NOTICE b/flink-connectors/flink-sql-connector-elasticsearch7/src/main/resources/META-INF/NOTICE new file mode 100644 index 0000000000000000000000000000000000000000..80866c7c2e0b1f9744292b9005b27189709bfe58 --- /dev/null +++ b/flink-connectors/flink-sql-connector-elasticsearch7/src/main/resources/META-INF/NOTICE @@ -0,0 +1,44 @@ +flink-sql-connector-elasticsearch7 +Copyright 2014-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +This project bundles the following dependencies under the Apache Software License 2.0. (http://www.apache.org/licenses/LICENSE-2.0.txt) + +- com.fasterxml.jackson.core:jackson-core:2.8.11 +- com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.11 +- com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.8.11 +- com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.11 +- commons-codec:commons-codec:1.10 +- commons-logging:commons-logging:1.1.3 +- org.apache.httpcomponents:httpasyncclient:4.1.4 +- org.apache.httpcomponents:httpclient:4.5.3 +- org.apache.httpcomponents:httpcore:4.4.6 +- org.apache.httpcomponents:httpcore-nio:4.4.11 +- org.apache.logging.log4j:log4j-api:2.11.1 +- org.apache.lucene:lucene-analyzers-common:8.1.0 +- org.apache.lucene:lucene-backward-codecs:8.1.0 +- org.apache.lucene:lucene-core:8.1.0 +- org.apache.lucene:lucene-grouping:8.1.0 +- org.apache.lucene:lucene-highlighter:8.1.0 +- org.apache.lucene:lucene-join:8.1.0 +- org.apache.lucene:lucene-memory:8.1.0 +- org.apache.lucene:lucene-misc:8.1.0 +- org.apache.lucene:lucene-queries:8.1.0 +- org.apache.lucene:lucene-queryparser:8.1.0 +- org.apache.lucene:lucene-sandbox:8.1.0 +- org.apache.lucene:lucene-spatial:8.1.0 +- org.apache.lucene:lucene-spatial-extras:8.1.0 +- org.apache.lucene:lucene-spatial3d:8.1.0 +- org.apache.lucene:lucene-suggest:8.1.0 +- org.elasticsearch:elasticsearch:7.3.2 +- org.elasticsearch:elasticsearch-cli:7.3.2 +- org.elasticsearch:elasticsearch-core:7.3.2 +- org.elasticsearch:elasticsearch-secure-sm:7.3.2 +- org.elasticsearch:elasticsearch-x-content:7.3.2 +- org.elasticsearch.client:elasticsearch-rest-client:7.3.2 +- org.elasticsearch.client:elasticsearch-rest-high-level-client:7.3.2 +- org.elasticsearch.plugin:aggs-matrix-stats-client:7.3.2 +- org.elasticsearch.plugin:parent-join-client:7.3.2 +- org.elasticsearch.plugin:rank-eval-client:7.3.2 diff --git a/flink-connectors/flink-sql-connector-elasticsearch7/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory b/flink-connectors/flink-sql-connector-elasticsearch7/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory new file mode 100644 index 0000000000000000000000000000000000000000..fe10b08d2bbce33f41b900168b2a4c5c9a44de92 --- /dev/null +++ b/flink-connectors/flink-sql-connector-elasticsearch7/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.streaming.sql.connectors.elasticsearch7.Elasticsearch7UpsertTableSinkFactory diff --git a/flink-connectors/flink-sql-connector-elasticsearch7/src/test/java/org/apache/flink/streaming/sql/connectors/elasticsearch7/Elasticsearch7UpsertTableSinkFactoryTest.java b/flink-connectors/flink-sql-connector-elasticsearch7/src/test/java/org/apache/flink/streaming/sql/connectors/elasticsearch7/Elasticsearch7UpsertTableSinkFactoryTest.java new file mode 100644 index 0000000000000000000000000000000000000000..a047b02f9bdbd1eba8cfe9ab1a5e70891023c38c --- /dev/null +++ b/flink-connectors/flink-sql-connector-elasticsearch7/src/test/java/org/apache/flink/streaming/sql/connectors/elasticsearch7/Elasticsearch7UpsertTableSinkFactoryTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.sql.connectors.elasticsearch7; + +import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.dag.Transformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.formats.json.JsonRowSerializationSchema; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.datastream.DataStreamSink; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.sink.SinkFunction; +import org.apache.flink.streaming.api.graph.StreamGraph; +import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkBase; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.ElasticsearchUpsertSinkFunction; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.Host; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkBase.SinkOption; +import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchUpsertTableSinkFactoryTestBase; +import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.types.Row; + +import org.apache.http.HttpHost; +import org.elasticsearch.common.xcontent.XContentType; +import org.junit.Test; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.apache.flink.table.descriptors.ElasticsearchValidator.CONNECTOR_VERSION_VALUE_7; +import static org.junit.Assert.assertEquals; + +/** + * Test for {@link Elasticsearch7UpsertTableSink} created by {@link Elasticsearch7UpsertTableSinkFactory}. + */ +public class Elasticsearch7UpsertTableSinkFactoryTest extends ElasticsearchUpsertTableSinkFactoryTestBase { + + @Test + public void testBuilder() { + final TableSchema schema = createTestSchema(); + + final TestElasticsearch7UpsertTableSink testSink = new TestElasticsearch7UpsertTableSink( + false, + schema, + Collections.singletonList(new Host(ElasticsearchUpsertTableSinkFactoryTestBase.HOSTNAME, ElasticsearchUpsertTableSinkFactoryTestBase.PORT, ElasticsearchUpsertTableSinkFactoryTestBase.SCHEMA)), + ElasticsearchUpsertTableSinkFactoryTestBase.INDEX, + ElasticsearchUpsertTableSinkFactoryTestBase.DOC_TYPE, + ElasticsearchUpsertTableSinkFactoryTestBase.KEY_DELIMITER, + ElasticsearchUpsertTableSinkFactoryTestBase.KEY_NULL_LITERAL, + new JsonRowSerializationSchema(schema.toRowType()), + XContentType.JSON, + new DummyFailureHandler(), + createTestSinkOptions()); + + final DataStreamMock dataStreamMock = new DataStreamMock( + new StreamExecutionEnvironmentMock(), + Types.TUPLE(Types.BOOLEAN, schema.toRowType())); + + testSink.emitDataStream(dataStreamMock); + + final ElasticsearchSink.Builder> expectedBuilder = new ElasticsearchSink.Builder<>( + Collections.singletonList(new HttpHost(ElasticsearchUpsertTableSinkFactoryTestBase.HOSTNAME, ElasticsearchUpsertTableSinkFactoryTestBase.PORT, ElasticsearchUpsertTableSinkFactoryTestBase.SCHEMA)), + new ElasticsearchUpsertSinkFunction( + ElasticsearchUpsertTableSinkFactoryTestBase.INDEX, + ElasticsearchUpsertTableSinkFactoryTestBase.DOC_TYPE, + ElasticsearchUpsertTableSinkFactoryTestBase.KEY_DELIMITER, + ElasticsearchUpsertTableSinkFactoryTestBase.KEY_NULL_LITERAL, + new JsonRowSerializationSchema(schema.toRowType()), + XContentType.JSON, + Elasticsearch7UpsertTableSink.UPDATE_REQUEST_FACTORY, + new int[0])); + expectedBuilder.setFailureHandler(new DummyFailureHandler()); + expectedBuilder.setBulkFlushBackoff(true); + expectedBuilder.setBulkFlushBackoffType(ElasticsearchSinkBase.FlushBackoffType.EXPONENTIAL); + expectedBuilder.setBulkFlushBackoffDelay(123); + expectedBuilder.setBulkFlushBackoffRetries(3); + expectedBuilder.setBulkFlushInterval(100); + expectedBuilder.setBulkFlushMaxActions(1000); + expectedBuilder.setBulkFlushMaxSizeMb(1); + expectedBuilder.setRestClientFactory(new Elasticsearch7UpsertTableSink.DefaultRestClientFactory("/myapp")); + + assertEquals(expectedBuilder, testSink.builder); + } + + @Override + protected String getElasticsearchVersion() { + return CONNECTOR_VERSION_VALUE_7; + } + + @Override + protected ElasticsearchUpsertTableSinkBase getExpectedTableSink( + boolean isAppendOnly, + TableSchema schema, + List hosts, + String index, + String docType, + String keyDelimiter, + String keyNullLiteral, + SerializationSchema serializationSchema, + XContentType contentType, + ActionRequestFailureHandler failureHandler, + Map sinkOptions) { + return new Elasticsearch7UpsertTableSink( + isAppendOnly, + schema, + hosts, + index, + keyDelimiter, + keyNullLiteral, + serializationSchema, + contentType, + failureHandler, + sinkOptions); + } + + // -------------------------------------------------------------------------------------------- + // Helper classes + // -------------------------------------------------------------------------------------------- + + private static class TestElasticsearch7UpsertTableSink extends Elasticsearch7UpsertTableSink { + + public ElasticsearchSink.Builder> builder; + + public TestElasticsearch7UpsertTableSink( + boolean isAppendOnly, + TableSchema schema, + List hosts, + String index, + String docType, + String keyDelimiter, + String keyNullLiteral, + SerializationSchema serializationSchema, + XContentType contentType, + ActionRequestFailureHandler failureHandler, + Map sinkOptions) { + + super( + isAppendOnly, + schema, + hosts, + index, + docType, + keyDelimiter, + keyNullLiteral, + serializationSchema, + contentType, + failureHandler, + sinkOptions); + } + + @Override + protected ElasticsearchSink.Builder> createBuilder( + ElasticsearchUpsertSinkFunction upsertSinkFunction, + List httpHosts) { + builder = super.createBuilder(upsertSinkFunction, httpHosts); + return builder; + } + } + + private static class StreamExecutionEnvironmentMock extends StreamExecutionEnvironment { + + @Override + public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { + throw new UnsupportedOperationException(); + } + } + + private static class DataStreamMock extends DataStream> { + + public SinkFunction sinkFunction; + + public DataStreamMock(StreamExecutionEnvironment environment, TypeInformation> outType) { + super(environment, new TransformationMock("name", outType, 1)); + } + + @Override + public DataStreamSink> addSink(SinkFunction> sinkFunction) { + this.sinkFunction = sinkFunction; + return super.addSink(sinkFunction); + } + } + + private static class TransformationMock extends Transformation> { + + public TransformationMock(String name, TypeInformation> outputType, int parallelism) { + super(name, outputType, parallelism); + } + + @Override + public Collection> getTransitivePredecessors() { + return null; + } + } +} diff --git a/flink-connectors/flink-sql-connector-elasticsearch7/src/test/resources/log4j-test.properties b/flink-connectors/flink-sql-connector-elasticsearch7/src/test/resources/log4j-test.properties new file mode 100644 index 0000000000000000000000000000000000000000..fcd86546668477e443ec309ab172625f66f17aae --- /dev/null +++ b/flink-connectors/flink-sql-connector-elasticsearch7/src/test/resources/log4j-test.properties @@ -0,0 +1,24 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +log4j.rootLogger=INFO, testlogger + +log4j.appender.testlogger=org.apache.log4j.ConsoleAppender +log4j.appender.testlogger.target=System.err +log4j.appender.testlogger.layout=org.apache.log4j.PatternLayout +log4j.appender.testlogger.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/flink-connectors/pom.xml b/flink-connectors/pom.xml index 41a5eaa912f61f82f272f32727fbc6ce3c5b7b60..2c2b053b5a95f4b7432042443fe04c535aaca378 100644 --- a/flink-connectors/pom.xml +++ b/flink-connectors/pom.xml @@ -49,6 +49,7 @@ under the License. flink-connector-elasticsearch2 flink-connector-elasticsearch5 flink-connector-elasticsearch6 + flink-connector-elasticsearch7 flink-connector-hive flink-connector-rabbitmq flink-connector-twitter @@ -58,6 +59,7 @@ under the License. flink-connector-kafka flink-connector-gcp-pubsub flink-connector-kinesis + flink-sql-connector-elasticsearch7 src/main/flink-bin/bin @@ -115,6 +103,13 @@ under the License. 755 + + + ../flink-python/bin/ + bin + 0755 + + ../flink-table/flink-sql-client/bin/ diff --git a/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java b/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java index b965355fb5bf7b27d92db54427e6ccfee41626b8..0b5f5be2ae0c8a364348c5c63a1e678df1b38fa6 100755 --- a/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java +++ b/flink-dist/src/test/java/org/apache/flink/dist/TaskManagerHeapSizeCalculationJavaBashTest.java @@ -80,8 +80,8 @@ public class TaskManagerHeapSizeCalculationJavaBashTest extends TestLogger { final long networkBufMin = 64L << 20; // 64MB final long networkBufMax = 1L << 30; // 1GB - int managedMemSize = Integer.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()); - float managedMemFrac = TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(); + int managedMemSize = Integer.valueOf(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.defaultValue()); + float managedMemFrac = TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.defaultValue(); // manual tests from org.apache.flink.runtime.taskexecutor.TaskManagerServices.calculateHeapSizeMB() @@ -119,8 +119,8 @@ public class TaskManagerHeapSizeCalculationJavaBashTest extends TestLogger { final long networkBufMin = 64L << 20; // 64MB final long networkBufMax = 1L << 30; // 1GB - int managedMemSize = Integer.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()); - float managedMemFrac = TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(); + int managedMemSize = Integer.valueOf(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.defaultValue()); + float managedMemFrac = TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.defaultValue(); // manual tests from org.apache.flink.runtime.taskexecutor.TaskManagerServices.calculateHeapSizeMB() @@ -184,11 +184,11 @@ public class TaskManagerHeapSizeCalculationJavaBashTest extends TestLogger { config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax)); if (managedMemSizeMB == 0) { - config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE); + config.removeConfig(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE); } else { - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, managedMemSizeMB + "m"); } - config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac); + config.setFloat(TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION, managedMemFrac); return config; } @@ -214,8 +214,8 @@ public class TaskManagerHeapSizeCalculationJavaBashTest extends TestLogger { int javaMemMB = Math.max((int) (max >> 20), ran.nextInt(Integer.MAX_VALUE)) + 1; boolean useOffHeap = ran.nextBoolean(); - int managedMemSize = Integer.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()); - float managedMemFrac = TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(); + int managedMemSize = Integer.valueOf(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.defaultValue()); + float managedMemFrac = TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.defaultValue(); if (ran.nextBoolean()) { // use fixed-size managed memory @@ -295,8 +295,8 @@ public class TaskManagerHeapSizeCalculationJavaBashTest extends TestLogger { String.valueOf(config.getFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION)), config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN), config.getString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX), - config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), - String.valueOf(config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION))}; + config.getString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE), + String.valueOf(config.getFloat(TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION))}; String scriptOutput = executeScript(command); // we need a tolerance of at least one, to compensate for MB/byte conversion rounding errors diff --git a/flink-end-to-end-tests/flink-elasticsearch7-test/pom.xml b/flink-end-to-end-tests/flink-elasticsearch7-test/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..76ddc0e2aebf6c75642bd389906b26dfea2e3104 --- /dev/null +++ b/flink-end-to-end-tests/flink-elasticsearch7-test/pom.xml @@ -0,0 +1,92 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-end-to-end-tests + 1.10-SNAPSHOT + .. + + + flink-elasticsearch7-test + flink-elasticsearch7-test + jar + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${project.version} + provided + + + org.apache.flink + flink-connector-elasticsearch7_${scala.binary.version} + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + Elasticsearch7SinkExample + package + + shade + + + Elasticsearch7SinkExample + + + com.google.code.findbugs:jsr305 + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.apache.flink.streaming.tests.Elasticsearch7SinkExample + + + + + + + + + + diff --git a/flink-end-to-end-tests/flink-elasticsearch7-test/src/main/java/org/apache/flink/streaming/tests/Elasticsearch7SinkExample.java b/flink-end-to-end-tests/flink-elasticsearch7-test/src/main/java/org/apache/flink/streaming/tests/Elasticsearch7SinkExample.java new file mode 100644 index 0000000000000000000000000000000000000000..e7036b5d390d5463408217d093e8d8ab5279f59e --- /dev/null +++ b/flink-end-to-end-tests/flink-elasticsearch7-test/src/main/java/org/apache/flink/streaming/tests/Elasticsearch7SinkExample.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.tests; + +import org.apache.flink.api.common.functions.FlatMapFunction; +import org.apache.flink.api.common.functions.RuntimeContext; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.connectors.elasticsearch.ActionRequestFailureHandler; +import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; +import org.apache.flink.streaming.connectors.elasticsearch7.ElasticsearchSink; +import org.apache.flink.util.Collector; + +import org.apache.http.HttpHost; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.Requests; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * End to end test for Elasticsearch6Sink. + */ +public class Elasticsearch7SinkExample { + + public static void main(String[] args) throws Exception { + + final ParameterTool parameterTool = ParameterTool.fromArgs(args); + + if (parameterTool.getNumberOfParameters() < 2) { + System.out.println("Missing parameters!\n" + + "Usage: --numRecords --index "); + return; + } + + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.enableCheckpointing(5000); + + DataStream> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1) + .flatMap(new FlatMapFunction>() { + @Override + public void flatMap(Long value, Collector> out) { + final String key = String.valueOf(value); + final String message = "message #" + value; + out.collect(Tuple2.of(key, message + "update #1")); + out.collect(Tuple2.of(key, message + "update #2")); + } + }); + + List httpHosts = new ArrayList<>(); + httpHosts.add(new HttpHost("127.0.0.1", 9200, "http")); + + ElasticsearchSink.Builder> esSinkBuilder = new ElasticsearchSink.Builder<>( + httpHosts, + (Tuple2 element, RuntimeContext ctx, RequestIndexer indexer) -> { + indexer.add(createIndexRequest(element.f1, parameterTool)); + indexer.add(createUpdateRequest(element, parameterTool)); + }); + + esSinkBuilder.setFailureHandler( + new CustomFailureHandler(parameterTool.getRequired("index"))); + + // this instructs the sink to emit after every element, otherwise they would be buffered + esSinkBuilder.setBulkFlushMaxActions(1); + + source.addSink(esSinkBuilder.build()); + + env.execute("Elasticsearch 7.x end to end sink test example"); + } + + private static class CustomFailureHandler implements ActionRequestFailureHandler { + + private static final long serialVersionUID = 942269087742453482L; + + private final String index; + + CustomFailureHandler(String index) { + this.index = index; + } + + @Override + public void onFailure(ActionRequest action, Throwable failure, int restStatusCode, RequestIndexer indexer) throws Throwable { + if (action instanceof IndexRequest) { + Map json = new HashMap<>(); + json.put("data", ((IndexRequest) action).source()); + + indexer.add( + Requests.indexRequest() + .index(index) + .id(((IndexRequest) action).id()) + .source(json)); + } else { + throw new IllegalStateException("unexpected"); + } + } + } + + private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { + Map json = new HashMap<>(); + json.put("data", element); + + String index; + String type; + + if (element.startsWith("message #15")) { + index = ":intentional invalid index:"; + type = ":intentional invalid type:"; + } else { + index = parameterTool.getRequired("index"); + } + + return Requests.indexRequest() + .index(index) + .id(element) + .source(json); + } + + private static UpdateRequest createUpdateRequest(Tuple2 element, ParameterTool parameterTool) { + Map json = new HashMap<>(); + json.put("data", element.f1); + + return new UpdateRequest( + parameterTool.getRequired("index"), + parameterTool.getRequired("type"), + element.f0) + .doc(json) + .upsert(json); + } +} diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml index b7ac5519081b975dbbe43eae99654239978ad9be..bdf04b3139c3eb0a16461fe82b01d103645ef44d 100644 --- a/flink-end-to-end-tests/pom.xml +++ b/flink-end-to-end-tests/pom.xml @@ -71,6 +71,7 @@ under the License. flink-plugins-test flink-tpch-test flink-streaming-kinesis-test + flink-elasticsearch7-test diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh index bdecb328dcfac74b502bb6310a1672386c5c1b62..41e0eec92b12848a59b48da8167cedb9eb76354b 100644 --- a/flink-end-to-end-tests/test-scripts/common.sh +++ b/flink-end-to-end-tests/test-scripts/common.sh @@ -329,6 +329,7 @@ function check_logs_for_errors { | grep -v "org.apache.flink.fs.shaded.hadoop3.org.apache.commons.beanutils.FluentPropertyBeanIntrospector - Error when creating PropertyDescriptor for public final void org.apache.flink.fs.shaded.hadoop3.org.apache.commons.configuration2.AbstractConfiguration.setProperty(java.lang.String,java.lang.Object)! Ignoring this property." \ | grep -v "Error while loading kafka-version.properties :null" \ | grep -v "Failed Elasticsearch item request" \ + | grep -v "[Terror] modules" \ | grep -ic "error" || true) if [[ ${error_count} -gt 0 ]]; then echo "Found error in log files:" diff --git a/flink-examples/flink-examples-table/pom.xml b/flink-examples/flink-examples-table/pom.xml index 7b45bdd986ed58bb130cca24bcb871c907462f8b..cd58a57bb278b1f6f10f64aa835b009ea615b292 100644 --- a/flink-examples/flink-examples-table/pom.xml +++ b/flink-examples/flink-examples-table/pom.xml @@ -51,6 +51,11 @@ under the License. flink-table-planner_${scala.binary.version} ${project.version} + + org.apache.flink + flink-table-planner-blink_${scala.binary.version} + ${project.version} + org.apache.flink diff --git a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamSQLExample.java b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamSQLExample.java index d3f9e0862809ce6408bf165053208129d613db40..e0af76398203d36f16ef3d3489a5f90136e6319d 100644 --- a/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamSQLExample.java +++ b/flink-examples/flink-examples-table/src/main/java/org/apache/flink/table/examples/java/StreamSQLExample.java @@ -18,16 +18,21 @@ package org.apache.flink.table.examples.java; +import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.java.StreamTableEnvironment; import java.util.Arrays; +import java.util.Objects; /** * Simple example for demonstrating the use of SQL on a Stream Table in Java. * + *

Usage: StreamSQLExample --planner <blink|flink>
+ * *

This example shows how to: * - Convert DataStreams to Tables * - Register a Table under a name @@ -42,9 +47,26 @@ public class StreamSQLExample { public static void main(String[] args) throws Exception { + final ParameterTool params = ParameterTool.fromArgs(args); + String planner = params.has("planner") ? params.get("planner") : "flink"; + // set up execution environment StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - StreamTableEnvironment tEnv = StreamTableEnvironment.create(env); + StreamTableEnvironment tEnv; + if (Objects.equals(planner, "blink")) { // use blink planner in streaming mode + EnvironmentSettings settings = EnvironmentSettings.newInstance() + .useBlinkPlanner() + .inStreamingMode() + .build(); + tEnv = StreamTableEnvironment.create(env, settings); + } else if (Objects.equals(planner, "flink")) { // use flink planner in streaming mode + tEnv = StreamTableEnvironment.create(env); + } else { + System.err.println("The planner is incorrect. Please run 'StreamSQLExample --planner ', " + + "where planner (it is either flink or blink, and the default is flink) indicates whether the " + + "example uses flink planner or blink planner."); + return; + } DataStream orderA = env.fromCollection(Arrays.asList( new Order(1L, "beer", 3), diff --git a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala index 2e922e02e36861482004652830717df25e840819..79fa5e8179b857790f6436cb5cf8cc66b491f146 100644 --- a/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala +++ b/flink-examples/flink-examples-table/src/main/scala/org/apache/flink/table/examples/scala/StreamSQLExample.scala @@ -17,14 +17,18 @@ */ package org.apache.flink.table.examples.scala +import org.apache.flink.api.java.utils.ParameterTool import org.apache.flink.api.scala._ import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment} +import org.apache.flink.table.api.EnvironmentSettings import org.apache.flink.table.api.scala._ /** * Simple example for demonstrating the use of SQL on a Stream Table in Scala. * - * This example shows how to: + *

Usage: StreamSQLExample --planner <blink|flink>
+ * + *

This example shows how to: * - Convert DataStreams to Tables * - Register a Table under a name * - Run a StreamSQL query on the registered Table @@ -38,9 +42,25 @@ object StreamSQLExample { def main(args: Array[String]): Unit = { + val params = ParameterTool.fromArgs(args) + val planner = if (params.has("planner")) params.get("planner") else "flink" + // set up execution environment val env = StreamExecutionEnvironment.getExecutionEnvironment - val tEnv = StreamTableEnvironment.create(env) + val tEnv = if (planner == "blink") { // use blink planner in streaming mode + val settings = EnvironmentSettings.newInstance() + .useBlinkPlanner() + .inStreamingMode() + .build() + StreamTableEnvironment.create(env, settings) + } else if (planner == "flink") { // use flink planner in streaming mode + StreamTableEnvironment.create(env) + } else { + System.err.println("The planner is incorrect. Please run 'StreamSQLExample --planner ', " + + "where planner (it is either flink or blink, and the default is flink) indicates whether the " + + "example uses flink planner or blink planner.") + return + } val orderA: DataStream[Order] = env.fromCollection(Seq( Order(1L, "beer", 3), @@ -53,14 +73,17 @@ object StreamSQLExample { Order(4L, "beer", 1))) // convert DataStream to Table - var tableA = tEnv.fromDataStream(orderA, 'user, 'product, 'amount) + val tableA = tEnv.fromDataStream(orderA, 'user, 'product, 'amount) // register DataStream as Table tEnv.registerDataStream("OrderB", orderB, 'user, 'product, 'amount) // union the two tables val result = tEnv.sqlQuery( - s"SELECT * FROM $tableA WHERE amount > 2 UNION ALL " + - "SELECT * FROM OrderB WHERE amount < 2") + s""" + |SELECT * FROM $tableA WHERE amount > 2 + |UNION ALL + |SELECT * FROM OrderB WHERE amount < 2 + """.stripMargin) result.toAppendStream[Order].print() diff --git a/flink-filesystems/flink-s3-fs-hadoop/pom.xml b/flink-filesystems/flink-s3-fs-hadoop/pom.xml index 4bc57a45bf55a2d89aa5bdf040057fa49e6c09be..d04def170fe479030066c440bf4ad351d153eb41 100644 --- a/flink-filesystems/flink-s3-fs-hadoop/pom.xml +++ b/flink-filesystems/flink-s3-fs-hadoop/pom.xml @@ -106,11 +106,6 @@ under the License. com.amazon org.apache.flink.fs.s3base.shaded.com.amazon - - - javax.xml.bind - org.apache.flink.fs.s3hadoop.shaded.javax.xml.bind - org.apache.flink.runtime.util @@ -146,6 +141,31 @@ under the License. 2.3.0 + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + + + javax.xml.bind + org.apache.flink.fs.s3hadoop.shaded.javax.xml.bind + + + + + + + + diff --git a/flink-filesystems/flink-s3-fs-presto/pom.xml b/flink-filesystems/flink-s3-fs-presto/pom.xml index b0a93c1e0c55b777719047bb63af2d6d1a8d8ef9..b56c77ed4cd34b837ea3831afa22d1acef814de9 100644 --- a/flink-filesystems/flink-s3-fs-presto/pom.xml +++ b/flink-filesystems/flink-s3-fs-presto/pom.xml @@ -270,10 +270,6 @@ under the License. io.airlift org.apache.flink.fs.s3presto.shaded.io.airlift - - javax.xml.bind - org.apache.flink.fs.s3presto.shaded.javax.xml.bind - org.HdrHistogram org.apache.flink.fs.s3presto.shaded.org.HdrHistogram @@ -342,6 +338,31 @@ under the License. 2.3.0 + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + + + javax.xml.bind + org.apache.flink.fs.s3presto.shaded.javax.xml.bind + + + + + + + + diff --git a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroExternalJarProgramITCase.java b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroExternalJarProgramITCase.java index 0ad5d329a75992b7834d109c0523e1c8e83517eb..fa79a26e23fbb206134e4de09c821bafa5704682 100644 --- a/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroExternalJarProgramITCase.java +++ b/flink-formats/flink-avro/src/test/java/org/apache/flink/formats/avro/AvroExternalJarProgramITCase.java @@ -18,7 +18,7 @@ package org.apache.flink.formats.avro; -import org.apache.flink.client.program.JobWithJars; +import org.apache.flink.client.ClientUtils; import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.core.fs.Path; import org.apache.flink.formats.avro.testjar.AvroExternalJarProgram; @@ -68,7 +68,7 @@ public class AvroExternalJarProgramITCase extends TestLogger { String jarFile = JAR_FILE; try { - JobWithJars.checkJarFile(new File(jarFile).getAbsoluteFile().toURI().toURL()); + ClientUtils.checkJarFile(new File(jarFile).getAbsoluteFile().toURI().toURL()); } catch (IOException e) { jarFile = "target/".concat(jarFile); } diff --git a/flink-formats/flink-csv/src/main/java/org/apache/flink/formats/csv/CsvRowSerializationSchema.java b/flink-formats/flink-csv/src/main/java/org/apache/flink/formats/csv/CsvRowSerializationSchema.java index a65554f8cab21762f430b2f1cd127cba769998c7..b3c2de4934d92c574e62050162e12f5620be5761 100644 --- a/flink-formats/flink-csv/src/main/java/org/apache/flink/formats/csv/CsvRowSerializationSchema.java +++ b/flink-formats/flink-csv/src/main/java/org/apache/flink/formats/csv/CsvRowSerializationSchema.java @@ -116,9 +116,9 @@ public final class CsvRowSerializationSchema implements SerializationSchema public Builder setLineDelimiter(String delimiter) { Preconditions.checkNotNull(delimiter, "Delimiter must not be null."); - if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n")) { + if (!delimiter.equals("\n") && !delimiter.equals("\r") && !delimiter.equals("\r\n") && !delimiter.equals("")) { throw new IllegalArgumentException( - "Unsupported new line delimiter. Only \\n, \\r, or \\r\\n are supported."); + "Unsupported new line delimiter. Only \\n, \\r, \\r\\n, or empty string are supported."); } this.csvSchema = this.csvSchema.rebuild().setLineSeparator(delimiter).build(); return this; diff --git a/flink-formats/flink-csv/src/main/java/org/apache/flink/table/descriptors/Csv.java b/flink-formats/flink-csv/src/main/java/org/apache/flink/table/descriptors/Csv.java index c467f1f00239e8d4f0ec217eb34bab9d32c10c61..5fb32050bb8f9786219b039f8d75a26b7f7a2a37 100644 --- a/flink-formats/flink-csv/src/main/java/org/apache/flink/table/descriptors/Csv.java +++ b/flink-formats/flink-csv/src/main/java/org/apache/flink/table/descriptors/Csv.java @@ -75,7 +75,7 @@ public class Csv extends FormatDescriptor { } /** - * Sets the line delimiter ("\n" by default; otherwise "\r" or "\r\n" are allowed). + * Sets the line delimiter ("\n" by default; otherwise "\r", "\r\n", or "" are allowed). * * @param delimiter the line delimiter */ diff --git a/flink-formats/flink-csv/src/main/java/org/apache/flink/table/descriptors/CsvValidator.java b/flink-formats/flink-csv/src/main/java/org/apache/flink/table/descriptors/CsvValidator.java index bc5397168138758820f9f4505bb38e539bbd6a04..2f7ebabc198f513797dfc11ff59b77ee33f181e6 100644 --- a/flink-formats/flink-csv/src/main/java/org/apache/flink/table/descriptors/CsvValidator.java +++ b/flink-formats/flink-csv/src/main/java/org/apache/flink/table/descriptors/CsvValidator.java @@ -44,7 +44,7 @@ public class CsvValidator extends FormatDescriptorValidator { public void validate(DescriptorProperties properties) { super.validate(properties); properties.validateString(FORMAT_FIELD_DELIMITER, true, 1, 1); - properties.validateEnumValues(FORMAT_LINE_DELIMITER, true, Arrays.asList("\r", "\n", "\r\n")); + properties.validateEnumValues(FORMAT_LINE_DELIMITER, true, Arrays.asList("\r", "\n", "\r\n", "")); properties.validateString(FORMAT_QUOTE_CHARACTER, true, 1, 1); properties.validateBoolean(FORMAT_ALLOW_COMMENTS, true); properties.validateBoolean(FORMAT_IGNORE_PARSE_ERRORS, true); diff --git a/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvRowDeSerializationSchemaTest.java b/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvRowDeSerializationSchemaTest.java index ece2e45e1871092c0798706a030b08086b834232..02b3e2782f3637eb368f4ed930bbcd3d24e32a6b 100644 --- a/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvRowDeSerializationSchemaTest.java +++ b/flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/CsvRowDeSerializationSchemaTest.java @@ -168,6 +168,17 @@ public class CsvRowDeSerializationSchemaTest { serialize(serSchemaBuilder, Row.of("Test", 12, "Hello"))); } + @Test + public void testEmptyLineDelimiter() throws Exception { + final TypeInformation rowInfo = Types.ROW(Types.STRING, Types.INT, Types.STRING); + final CsvRowSerializationSchema.Builder serSchemaBuilder = new CsvRowSerializationSchema.Builder(rowInfo) + .setLineDelimiter(""); + + assertArrayEquals( + "Test,12,Hello".getBytes(), + serialize(serSchemaBuilder, Row.of("Test", 12, "Hello"))); + } + @Test(expected = IllegalArgumentException.class) public void testInvalidNesting() throws Exception { testNullableField(Types.ROW(Types.ROW(Types.STRING)), "FAIL", Row.of(Row.of("FAIL"))); diff --git a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/ParquetTableSource.java b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/ParquetTableSource.java index 0b5d168bd2801e69aa40b17185d6348df77de24a..91447279730b8c9ea4107a98887f2b059470e19a 100644 --- a/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/ParquetTableSource.java +++ b/flink-formats/flink-parquet/src/main/java/org/apache/flink/formats/parquet/ParquetTableSource.java @@ -66,6 +66,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -223,7 +224,8 @@ public class ParquetTableSource @Override public String explainSource() { return "ParquetFile[path=" + path + ", schema=" + parquetSchema + ", filter=" + predicateString() - + ", typeInfo=" + typeInfo + "]"; + + ", typeInfo=" + typeInfo + ", selectedFields=" + Arrays.toString(selectedFields) + + ", pushDownStatus=" + isFilterPushedDown + "]"; } private String predicateString() { diff --git a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FsNegativeRunningJobsRegistryTest.java b/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FsNegativeRunningJobsRegistryTest.java deleted file mode 100644 index 1273a4ee9d2213459ceaf87bf6939951c353b11d..0000000000000000000000000000000000000000 --- a/flink-fs-tests/src/test/java/org/apache/flink/hdfstests/FsNegativeRunningJobsRegistryTest.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.hdfstests; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.core.fs.Path; -import org.apache.flink.runtime.highavailability.FsNegativeRunningJobsRegistry; -import org.apache.flink.runtime.highavailability.RunningJobsRegistry.JobSchedulingStatus; -import org.apache.flink.util.OperatingSystem; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; - -import static org.junit.Assert.assertEquals; - -/** - * Tests for the {@link FsNegativeRunningJobsRegistry} on HDFS. - */ -public class FsNegativeRunningJobsRegistryTest { - - @ClassRule - public static final TemporaryFolder TEMP_DIR = new TemporaryFolder(); - - private static MiniDFSCluster hdfsCluster; - - private static Path hdfsRootPath; - - // ------------------------------------------------------------------------ - // startup / shutdown - // ------------------------------------------------------------------------ - - @BeforeClass - public static void createHDFS() throws Exception { - Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows()); - - final File tempDir = TEMP_DIR.newFolder(); - - Configuration hdConf = new Configuration(); - hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath()); - - MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); - hdfsCluster = builder.build(); - - hdfsRootPath = new Path("hdfs://" + hdfsCluster.getURI().getHost() + ":" - + hdfsCluster.getNameNodePort() + "/"); - } - - @AfterClass - public static void destroyHDFS() { - if (hdfsCluster != null) { - hdfsCluster.shutdown(); - } - hdfsCluster = null; - hdfsRootPath = null; - } - - // ------------------------------------------------------------------------ - // Tests - // ------------------------------------------------------------------------ - - @Test - public void testCreateAndSetFinished() throws Exception { - final Path workDir = new Path(hdfsRootPath, "test-work-dir"); - final JobID jid = new JobID(); - - FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(workDir); - - // another registry should pick this up - FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(workDir); - - // initially, without any call, the job is pending - assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid)); - assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid)); - - // after set running, the job is running - registry.setJobRunning(jid); - assertEquals(JobSchedulingStatus.RUNNING, registry.getJobSchedulingStatus(jid)); - assertEquals(JobSchedulingStatus.RUNNING, otherRegistry.getJobSchedulingStatus(jid)); - - // set the job to finished and validate - registry.setJobFinished(jid); - assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid)); - assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid)); - } - - @Test - public void testSetFinishedAndRunning() throws Exception { - final Path workDir = new Path(hdfsRootPath, "änother_wörk_directörü"); - final JobID jid = new JobID(); - - FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(workDir); - - // set the job to finished and validate - registry.setJobFinished(jid); - assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid)); - - // set the job to running does not overwrite the finished status - registry.setJobRunning(jid); - assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid)); - - // another registry should pick this up - FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(workDir); - assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid)); - - // clear the running and finished marker, it will be pending - otherRegistry.clearJob(jid); - assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid)); - assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid)); - } -} diff --git a/flink-java/src/main/java/org/apache/flink/api/java/LocalEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/LocalEnvironment.java index 9ce09467b1cb6336882cb7bd3ccca477faa7f926..dc886507a6ccb97b63c607f317ae4445919504b0 100644 --- a/flink-java/src/main/java/org/apache/flink/api/java/LocalEnvironment.java +++ b/flink-java/src/main/java/org/apache/flink/api/java/LocalEnvironment.java @@ -26,6 +26,8 @@ import org.apache.flink.api.common.Plan; import org.apache.flink.api.common.PlanExecutor; import org.apache.flink.configuration.Configuration; +import java.util.Collections; + import static org.apache.flink.util.Preconditions.checkNotNull; /** @@ -73,7 +75,10 @@ public class LocalEnvironment extends ExecutionEnvironment { final Plan p = createProgramPlan(jobName); final PlanExecutor executor = PlanExecutor.createLocalExecutor(configuration); - lastJobExecutionResult = executor.executePlan(p); + lastJobExecutionResult = executor.executePlan( + p, + Collections.emptyList(), + Collections.emptyList()); return lastJobExecutionResult; } diff --git a/flink-java/src/main/java/org/apache/flink/api/java/RemoteEnvironment.java b/flink-java/src/main/java/org/apache/flink/api/java/RemoteEnvironment.java index 786add1400e29acccedff3d54b7e38ce9221d182..9b6add7666392635c7888ed1617e9c8a279825b7 100644 --- a/flink-java/src/main/java/org/apache/flink/api/java/RemoteEnvironment.java +++ b/flink-java/src/main/java/org/apache/flink/api/java/RemoteEnvironment.java @@ -155,8 +155,8 @@ public class RemoteEnvironment extends ExecutionEnvironment { public JobExecutionResult execute(String jobName) throws Exception { final Plan p = createProgramPlan(jobName); - final PlanExecutor executor = PlanExecutor.createRemoteExecutor(host, port, clientConfiguration, jarFiles, globalClasspaths); - lastJobExecutionResult = executor.executePlan(p); + final PlanExecutor executor = PlanExecutor.createRemoteExecutor(host, port, clientConfiguration); + lastJobExecutionResult = executor.executePlan(p, jarFiles, globalClasspaths); return lastJobExecutionResult; } diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java b/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java index 51484c1d853b168f8aed930670ecfde090d73b75..d957977ea8e9b93f8f46e55d8511281020c347ed 100644 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java +++ b/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java @@ -118,4 +118,30 @@ public class MesosOptions { "All port keys will dynamically get a port assigned through Mesos.") .build()); + /** + * Config parameter to configure the amount of time to wait for unused expired Mesos + * offers before they are declined. + */ + public static final ConfigOption UNUSED_OFFER_EXPIRATION = + key("mesos.resourcemanager.unused-offer-expiration") + .defaultValue(120000L) + .withDescription( + Description.builder() + .text("Amount of time to wait for unused expired offers before declining them. " + + "This ensures your scheduler will not hoard unuseful offers.") + .build()); + + /** + * Config parameter to configure the amount of time refuse a particular offer for. + * This ensures the same resource offer isn't resent immediately after declining. + */ + public static final ConfigOption DECLINED_OFFER_REFUSE_DURATION = + key("mesos.resourcemanager.declined-offer-refuse-duration") + .defaultValue(5000L) + .withDescription( + Description.builder() + .text("Amount of time to ask the Mesos master to not resend a " + + "declined resource offer again. This ensures a declined resource offer " + + "isn't resent immediately after being declined") + .build()); } diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java index 377b5b5971ab19cb3b961c2fa0e8854f38006eb1..6c1f486ba9b4c57beb444ac2b14740cf8ea77720 100755 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java +++ b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java @@ -30,9 +30,8 @@ import org.apache.flink.runtime.clusterframework.ContainerSpecification; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; import org.apache.flink.runtime.entrypoint.JobClusterEntrypoint; -import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponentFactory; +import org.apache.flink.runtime.entrypoint.component.DefaultDispatcherResourceManagerComponentFactory; import org.apache.flink.runtime.entrypoint.component.FileJobGraphRetriever; -import org.apache.flink.runtime.entrypoint.component.JobDispatcherResourceManagerComponentFactory; import org.apache.flink.runtime.util.EnvironmentInformation; import org.apache.flink.runtime.util.JvmShutdownSafeguard; import org.apache.flink.runtime.util.SignalHandler; @@ -109,8 +108,8 @@ public class MesosJobClusterEntrypoint extends JobClusterEntrypoint { } @Override - protected DispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration) { - return new JobDispatcherResourceManagerComponentFactory( + protected DefaultDispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration) { + return DefaultDispatcherResourceManagerComponentFactory.createJobComponentFactory( new MesosResourceManagerFactory( mesosServices, schedulerConfiguration, diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java index 98796282cf2066f15aa3f1a11065315c93d08d56..4033d7d83d5eb504e827eba6e9268698aa99ad77 100755 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java +++ b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java @@ -30,8 +30,7 @@ import org.apache.flink.runtime.clusterframework.ContainerSpecification; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; import org.apache.flink.runtime.entrypoint.SessionClusterEntrypoint; -import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponentFactory; -import org.apache.flink.runtime.entrypoint.component.SessionDispatcherResourceManagerComponentFactory; +import org.apache.flink.runtime.entrypoint.component.DefaultDispatcherResourceManagerComponentFactory; import org.apache.flink.runtime.util.EnvironmentInformation; import org.apache.flink.runtime.util.JvmShutdownSafeguard; import org.apache.flink.runtime.util.SignalHandler; @@ -108,8 +107,8 @@ public class MesosSessionClusterEntrypoint extends SessionClusterEntrypoint { } @Override - protected DispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration) { - return new SessionDispatcherResourceManagerComponentFactory( + protected DefaultDispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration) { + return DefaultDispatcherResourceManagerComponentFactory.createSessionComponentFactory( new MesosResourceManagerFactory( mesosServices, mesosConfig, diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java index 5cd8dc3449e94edfb6ebd9251e0ec52c20bb5ac8..4382573ef7f0ec91f50cf4b1aa0241ef17f9bcfe 100755 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java +++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManager.java @@ -48,8 +48,7 @@ import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.JobLeaderIdService; import org.apache.flink.runtime.resourcemanager.ResourceManager; import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException; @@ -155,7 +154,6 @@ public class MesosResourceManager extends ResourceManagerFlinkTuple2} +import org.apache.flink.api.java.tuple.{Tuple2 => FlinkTuple2} import org.apache.flink.configuration.Configuration import org.apache.flink.mesos.Utils import org.apache.flink.mesos.scheduler.LaunchCoordinator._ import org.apache.flink.mesos.scheduler.messages._ import org.apache.flink.mesos.util.MesosResourceAllocation -import org.apache.mesos.{SchedulerDriver, Protos} +import org.apache.mesos.{Protos, SchedulerDriver} import scala.collection.JavaConverters._ import scala.collection.mutable.{Map => MutableMap} import scala.concurrent.duration._ +import org.apache.flink.mesos.configuration.MesosOptions._ /** * The launch coordinator handles offer processing, including @@ -54,6 +56,15 @@ class LaunchCoordinator( val LOG = Logger(getClass) + val declineOfferFilters: Protos.Filters = + Protos.Filters.newBuilder() + .setRefuseSeconds( + Duration(config.getLong(DECLINED_OFFER_REFUSE_DURATION), TimeUnit.MILLISECONDS).toSeconds) + .build() + + val unusedOfferExpirationDuration: Long = + Duration(config.getLong(UNUSED_OFFER_EXPIRATION), TimeUnit.MILLISECONDS).toSeconds + /** * The task placement optimizer. * @@ -68,13 +79,13 @@ class LaunchCoordinator( LOG.info(s"Declined offer ${lease.getId} from ${lease.hostname()} " + s"of memory ${lease.memoryMB()} MB, ${lease.cpuCores()} cpus, " + s"${lease.getScalarValue("gpus")} gpus, " - + s"of disk: ${lease.diskMB()} MB, network: ${lease.networkMbps()} Mbps") - schedulerDriver.declineOffer(lease.getOffer.getId) + + s"of disk: ${lease.diskMB()} MB, network: ${lease.networkMbps()} Mbps " + + s"for the next ${declineOfferFilters.getRefuseSeconds} seconds") + schedulerDriver.declineOffer(lease.getOffer.getId, declineOfferFilters) } }) - // avoid situations where we have lots of expired offers and we only expire a few at a time - .withRejectAllExpiredOffers() - .build + .withLeaseOfferExpirySecs(unusedOfferExpirationDuration) + .withRejectAllExpiredOffers().build } override def postStop(): Unit = { @@ -114,7 +125,9 @@ class LaunchCoordinator( case Event(offers: ResourceOffers, data: GatherData) => // decline any offers that come in schedulerDriver.suppressOffers() - for(offer <- offers.offers().asScala) { schedulerDriver.declineOffer(offer.getId) } + for(offer <- offers.offers().asScala) { + schedulerDriver.declineOffer(offer.getId, declineOfferFilters) + } stay() case Event(msg: Launch, data: GatherData) => diff --git a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java index 8a71d13e1ce1a04952465e134bd16c7ec79daeca..b2eb177dea21f1f98a4ed893fb89b42404327048 100644 --- a/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java +++ b/flink-mesos/src/test/java/org/apache/flink/mesos/runtime/clusterframework/MesosResourceManagerTest.java @@ -53,9 +53,7 @@ import org.apache.flink.runtime.jobmaster.JobMasterRegistrationSuccess; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.MetricRegistryImpl; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.registration.RegistrationResponse; import org.apache.flink.runtime.resourcemanager.JobLeaderIdService; @@ -163,7 +161,6 @@ public class MesosResourceManagerTest extends TestLogger { HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, SlotManager slotManager, - MetricRegistry metricRegistry, JobLeaderIdService jobLeaderIdService, FatalErrorHandler fatalErrorHandler, @@ -173,7 +170,7 @@ public class MesosResourceManagerTest extends TestLogger { MesosConfiguration mesosConfig, MesosTaskManagerParameters taskManagerParameters, ContainerSpecification taskManagerContainerSpec, - JobManagerMetricGroup jobManagerMetricGroup) { + ResourceManagerMetricGroup resourceManagerMetricGroup) { super( rpcService, resourceManagerEndpointId, @@ -181,7 +178,6 @@ public class MesosResourceManagerTest extends TestLogger { highAvailabilityServices, heartbeatServices, slotManager, - metricRegistry, jobLeaderIdService, new ClusterInformation("localhost", 1234), fatalErrorHandler, @@ -191,7 +187,7 @@ public class MesosResourceManagerTest extends TestLogger { taskManagerParameters, taskManagerContainerSpec, null, - jobManagerMetricGroup); + resourceManagerMetricGroup); } @Override @@ -292,7 +288,6 @@ public class MesosResourceManagerTest extends TestLogger { rmServices.highAvailabilityServices, rmServices.heartbeatServices, rmServices.slotManager, - rmServices.metricRegistry, rmServices.jobLeaderIdService, fatalErrorHandler, // Mesos specifics @@ -301,7 +296,7 @@ public class MesosResourceManagerTest extends TestLogger { rmServices.mesosConfig, tmParams, containerSpecification, - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup()); + UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup()); // TaskExecutors task1Executor = mockTaskExecutor(task1); @@ -319,7 +314,6 @@ public class MesosResourceManagerTest extends TestLogger { public final TestingHighAvailabilityServices highAvailabilityServices; public final HeartbeatServices heartbeatServices; - public final MetricRegistry metricRegistry; public final TestingLeaderElectionService rmLeaderElectionService; public final JobLeaderIdService jobLeaderIdService; public final SlotManager slotManager; @@ -333,7 +327,6 @@ public class MesosResourceManagerTest extends TestLogger { rmLeaderElectionService = new TestingLeaderElectionService(); highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService); heartbeatServices = new HeartbeatServices(5L, 5L); - metricRegistry = mock(MetricRegistryImpl.class); slotManager = mock(SlotManager.class); slotManagerStarted = new CompletableFuture<>(); jobLeaderIdService = new JobLeaderIdService( diff --git a/flink-mesos/src/test/scala/org/apache/flink/mesos/scheduler/LaunchCoordinatorTest.scala b/flink-mesos/src/test/scala/org/apache/flink/mesos/scheduler/LaunchCoordinatorTest.scala index 7b913ea7894fc9ea8249e3ceef7f9058ae8a2d42..00c23c6c144fe41403f3902357a5f254b06dc0e6 100644 --- a/flink-mesos/src/test/scala/org/apache/flink/mesos/scheduler/LaunchCoordinatorTest.scala +++ b/flink-mesos/src/test/scala/org/apache/flink/mesos/scheduler/LaunchCoordinatorTest.scala @@ -18,6 +18,7 @@ package org.apache.flink.mesos.scheduler +import java.util.concurrent.TimeUnit import java.util.{Collections, UUID} import java.util.concurrent.atomic.AtomicReference @@ -27,13 +28,13 @@ import akka.testkit._ import com.netflix.fenzo.TaskRequest.{AssignedResources, NamedResourceSetRequest} import com.netflix.fenzo._ import com.netflix.fenzo.functions.{Action1, Action2} -import org.apache.flink.api.java.tuple.{Tuple2 => FlinkTuple2} +import org.apache.flink.api.java.tuple.{Tuple2=>FlinkTuple2} import org.apache.flink.configuration.Configuration import org.apache.flink.mesos.scheduler.LaunchCoordinator._ import org.apache.flink.mesos.scheduler.messages._ import org.apache.flink.runtime.akka.AkkaUtils import org.apache.mesos.Protos.{SlaveID, TaskInfo} -import org.apache.mesos.{Protos, SchedulerDriver} +import org.apache.mesos.{SchedulerDriver, Protos} import org.junit.runner.RunWith import org.mockito.Mockito.{verify, _} import org.mockito.invocation.InvocationOnMock @@ -47,8 +48,11 @@ import scala.collection.JavaConverters._ import org.apache.flink.mesos.Utils.range import org.apache.flink.mesos.Utils.ranges import org.apache.flink.mesos.Utils.scalar +import org.apache.flink.mesos.configuration.MesosOptions.DECLINED_OFFER_REFUSE_DURATION import org.apache.flink.mesos.util.MesosResourceAllocation +import scala.concurrent.duration.Duration + @RunWith(classOf[JUnitRunner]) class LaunchCoordinatorTest extends TestKitBase @@ -202,6 +206,9 @@ class LaunchCoordinatorTest def taskSchedulerBuilder(optimizer: TaskScheduler) = new TaskSchedulerBuilder { var leaseRejectAction: Action1[VirtualMachineLease] = null var rejectAllExpiredOffers: Boolean = false + var leaseOfferExpiry: Long = 0L + var offersToReject: Int = 0 + override def withLeaseRejectAction( action: Action1[VirtualMachineLease]): TaskSchedulerBuilder = { leaseRejectAction = action @@ -212,7 +219,13 @@ class LaunchCoordinatorTest this } + override def withLeaseOfferExpirySecs(leaseOfferExpirySecs: Long): TaskSchedulerBuilder = { + leaseOfferExpiry = leaseOfferExpirySecs + this + } + override def build(): TaskScheduler = optimizer + } /** @@ -239,6 +252,11 @@ class LaunchCoordinatorTest val trace = Mockito.inOrder(schedulerDriver) val fsm = TestFSMRef(new LaunchCoordinator(testActor, config, schedulerDriver, optimizerBuilder)) + val refuseFilter = + Protos.Filters.newBuilder() + .setRefuseSeconds( + Duration(config.getLong(DECLINED_OFFER_REFUSE_DURATION), TimeUnit.MILLISECONDS).toSeconds) + .build() val framework = randomFramework val task1 = randomTask @@ -321,8 +339,8 @@ class LaunchCoordinatorTest "stays in Idle with offers declined" in new Context { fsm.setState(Idle) fsm ! new ResourceOffers(Seq(slave1._3, slave1._4).asJava) - verify(schedulerDriver).declineOffer(slave1._3.getId) - verify(schedulerDriver).declineOffer(slave1._4.getId) + verify(schedulerDriver).declineOffer(slave1._3.getId, refuseFilter) + verify(schedulerDriver).declineOffer(slave1._4.getId, refuseFilter) fsm.stateName should be (Idle) } } @@ -462,7 +480,7 @@ class LaunchCoordinatorTest fsm.setState(GatheringOffers, GatherData(tasks = Seq(task1._2), newLeases = Seq(lease(slave1._3)))) fsm ! StateTimeout - verify(schedulerDriver).declineOffer(slave1._3.getId) + verify(schedulerDriver).declineOffer(slave1._3.getId, refuseFilter) } } diff --git a/flink-metrics/flink-metrics-influxdb/src/main/java/org/apache/flink/metrics/influxdb/InfluxdbReporter.java b/flink-metrics/flink-metrics-influxdb/src/main/java/org/apache/flink/metrics/influxdb/InfluxdbReporter.java index 28d9eee184a763a0682a88831c5066cafbd61a03..19404ffbe0cbbed15a770ffba0ca181480f09106 100644 --- a/flink-metrics/flink-metrics-influxdb/src/main/java/org/apache/flink/metrics/influxdb/InfluxdbReporter.java +++ b/flink-metrics/flink-metrics-influxdb/src/main/java/org/apache/flink/metrics/influxdb/InfluxdbReporter.java @@ -27,6 +27,7 @@ import org.apache.flink.metrics.MetricConfig; import org.apache.flink.metrics.reporter.MetricReporter; import org.apache.flink.metrics.reporter.Scheduled; +import okhttp3.OkHttpClient; import org.influxdb.InfluxDB; import org.influxdb.InfluxDBFactory; import org.influxdb.dto.BatchPoints; @@ -37,13 +38,16 @@ import java.time.Instant; import java.util.ConcurrentModificationException; import java.util.Map; import java.util.NoSuchElementException; +import java.util.concurrent.TimeUnit; +import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.CONNECT_TIMEOUT; import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.DB; import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.HOST; import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.PASSWORD; import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.PORT; import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.RETENTION_POLICY; import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.USERNAME; +import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.WRITE_TIMEOUT; import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.getInteger; import static org.apache.flink.metrics.influxdb.InfluxdbReporterOptions.getString; @@ -77,10 +81,17 @@ public class InfluxdbReporter extends AbstractReporter implemen this.database = database; this.retentionPolicy = getString(config, RETENTION_POLICY); + + int connectTimeout = getInteger(config, CONNECT_TIMEOUT); + int writeTimeout = getInteger(config, WRITE_TIMEOUT); + OkHttpClient.Builder client = new OkHttpClient.Builder() + .connectTimeout(connectTimeout, TimeUnit.MILLISECONDS) + .writeTimeout(writeTimeout, TimeUnit.MILLISECONDS); + if (username != null && password != null) { - influxDB = InfluxDBFactory.connect(url, username, password); + influxDB = InfluxDBFactory.connect(url, username, password, client); } else { - influxDB = InfluxDBFactory.connect(url); + influxDB = InfluxDBFactory.connect(url, client); } log.info("Configured InfluxDBReporter with {host:{}, port:{}, db:{}, and retentionPolicy:{}}", host, port, database, retentionPolicy); diff --git a/flink-metrics/flink-metrics-influxdb/src/main/java/org/apache/flink/metrics/influxdb/InfluxdbReporterOptions.java b/flink-metrics/flink-metrics-influxdb/src/main/java/org/apache/flink/metrics/influxdb/InfluxdbReporterOptions.java index fe21fb083cb7278590147ea656c40b041f387bbc..cc88eb5f33120a7e5a95a3cca6ae474c62f576e2 100644 --- a/flink-metrics/flink-metrics-influxdb/src/main/java/org/apache/flink/metrics/influxdb/InfluxdbReporterOptions.java +++ b/flink-metrics/flink-metrics-influxdb/src/main/java/org/apache/flink/metrics/influxdb/InfluxdbReporterOptions.java @@ -57,6 +57,16 @@ public class InfluxdbReporterOptions { .defaultValue("") .withDescription("(optional) the InfluxDB retention policy for metrics"); + public static final ConfigOption CONNECT_TIMEOUT = ConfigOptions + .key("connectTimeout") + .defaultValue(10000) + .withDescription("(optional) the InfluxDB connect timeout for metrics"); + + public static final ConfigOption WRITE_TIMEOUT = ConfigOptions + .key("writeTimeout") + .defaultValue(10000) + .withDescription("(optional) the InfluxDB write timeout for metrics"); + static String getString(MetricConfig config, ConfigOption key) { return config.getString(key.key(), key.defaultValue()); } diff --git a/flink-ml-parent/flink-ml-lib/pom.xml b/flink-ml-parent/flink-ml-lib/pom.xml index 391b2acd86b12c5ff167aa74945dc531d9c61db8..5cd4793ccdfba98038dd89b0e03db2ab38f09538 100644 --- a/flink-ml-parent/flink-ml-lib/pom.xml +++ b/flink-ml-parent/flink-ml-lib/pom.xml @@ -26,7 +26,8 @@ under the License. 1.10-SNAPSHOT - flink-ml-lib + flink-ml-lib_${scala.binary.version} + flink-ml-lib @@ -34,6 +35,22 @@ under the License. flink-ml-api ${project.version} + + org.apache.flink + flink-table-api-java + ${project.version} + + + org.apache.flink + flink-table-api-java-bridge_${scala.binary.version} + ${project.version} + + + org.apache.flink + flink-table-planner_${scala.binary.version} + ${project.version} + test + com.github.fommil.netlib core diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java new file mode 100644 index 0000000000000000000000000000000000000000..f9deceaec6c5b44d7b668e34313af521aae3a0ad --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.common; + +import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.java.StreamTableEnvironment; + +/** + * The MLEnvironment stores the necessary context in Flink. + * Each MLEnvironment will be associated with a unique ID. + * The operations associated with the same MLEnvironment ID + * will share the same Flink job context. + * + *

Both MLEnvironment ID and MLEnvironment can only be retrieved from MLEnvironmentFactory. + * + * @see ExecutionEnvironment + * @see StreamExecutionEnvironment + * @see BatchTableEnvironment + * @see StreamTableEnvironment + */ +public class MLEnvironment { + private ExecutionEnvironment env; + private StreamExecutionEnvironment streamEnv; + private BatchTableEnvironment batchTableEnv; + private StreamTableEnvironment streamTableEnv; + + /** + * Construct with null that the class can load the environment in the `get` method. + */ + public MLEnvironment() { + this(null, null, null, null); + } + + /** + * Construct with the batch environment and the the batch table environment given by user. + * + *

The env can be null which will be loaded in the `get` method. + * + * @param batchEnv the ExecutionEnvironment + * @param batchTableEnv the BatchTableEnvironment + */ + public MLEnvironment( + ExecutionEnvironment batchEnv, + BatchTableEnvironment batchTableEnv) { + this(batchEnv, batchTableEnv, null, null); + } + + /** + * Construct with the stream environment and the the stream table environment given by user. + * + *

The env can be null which will be loaded in the `get` method. + * + * @param streamEnv the StreamExecutionEnvironment + * @param streamTableEnv the StreamTableEnvironment + */ + public MLEnvironment( + StreamExecutionEnvironment streamEnv, + StreamTableEnvironment streamTableEnv) { + this(null, null, streamEnv, streamTableEnv); + } + + /** + * Construct with env given by user. + * + *

The env can be null which will be loaded in the `get` method. + * + * @param batchEnv the ExecutionEnvironment + * @param batchTableEnv the BatchTableEnvironment + * @param streamEnv the StreamExecutionEnvironment + * @param streamTableEnv the StreamTableEnvironment + */ + public MLEnvironment( + ExecutionEnvironment batchEnv, + BatchTableEnvironment batchTableEnv, + StreamExecutionEnvironment streamEnv, + StreamTableEnvironment streamTableEnv) { + this.env = batchEnv; + this.batchTableEnv = batchTableEnv; + this.streamEnv = streamEnv; + this.streamTableEnv = streamTableEnv; + } + + /** + * Get the ExecutionEnvironment. + * if the ExecutionEnvironment has not been set, it initial the ExecutionEnvironment + * with default Configuration. + * + * @return the batch {@link ExecutionEnvironment} + */ + public ExecutionEnvironment getExecutionEnvironment() { + if (null == env) { + env = ExecutionEnvironment.getExecutionEnvironment(); + } + return env; + } + + /** + * Get the StreamExecutionEnvironment. + * if the StreamExecutionEnvironment has not been set, it initial the StreamExecutionEnvironment + * with default Configuration. + * + * @return the {@link StreamExecutionEnvironment} + */ + public StreamExecutionEnvironment getStreamExecutionEnvironment() { + if (null == streamEnv) { + streamEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + } + return streamEnv; + } + + /** + * Get the BatchTableEnvironment. + * if the BatchTableEnvironment has not been set, it initial the BatchTableEnvironment + * with default Configuration. + * + * @return the {@link BatchTableEnvironment} + */ + public BatchTableEnvironment getBatchTableEnvironment() { + if (null == batchTableEnv) { + batchTableEnv = BatchTableEnvironment.create(getExecutionEnvironment()); + } + return batchTableEnv; + } + + /** + * Get the StreamTableEnvironment. + * if the StreamTableEnvironment has not been set, it initial the StreamTableEnvironment + * with default Configuration. + * + * @return the {@link StreamTableEnvironment} + */ + public StreamTableEnvironment getStreamTableEnvironment() { + if (null == streamTableEnv) { + streamTableEnv = StreamTableEnvironment.create(getStreamExecutionEnvironment()); + } + return streamTableEnv; + } +} + diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironmentFactory.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironmentFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..c1b298632d2f693889d6c12a12acb6ac9e4129c3 --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironmentFactory.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.common; + +import org.apache.flink.util.Preconditions; + +import java.util.HashMap; + +/** + * Factory to get the MLEnvironment using a MLEnvironmentId. + * + *

The following code snippet shows how to interact with MLEnvironmentFactory. + * 

+ * {@code
+ * long mlEnvId = MLEnvironmentFactory.getNewMLEnvironmentId();
+ * MLEnvironment mlEnv = MLEnvironmentFactory.get(mlEnvId);
+ * }
+ *
+ */ +public class MLEnvironmentFactory { + + /** + * The default MLEnvironmentId. + */ + public static final Long DEFAULT_ML_ENVIRONMENT_ID = 0L; + + /** + * A monotonically increasing id for the MLEnvironments. + * Each id uniquely identifies an MLEnvironment. + */ + private static Long nextId = 1L; + + /** + * Map that hold the MLEnvironment and use the MLEnvironmentId as its key. + */ + private static final HashMap map = new HashMap<>(); + + static { + map.put(DEFAULT_ML_ENVIRONMENT_ID, new MLEnvironment()); + } + + /** + * Get the MLEnvironment using a MLEnvironmentId. + * + * @param mlEnvId the MLEnvironmentId + * @return the MLEnvironment + */ + public static synchronized MLEnvironment get(Long mlEnvId) { + if (!map.containsKey(mlEnvId)) { + throw new IllegalArgumentException( + String.format("Cannot find MLEnvironment for MLEnvironmentId %s." + + " Did you get the MLEnvironmentId by calling getNewMLEnvironmentId?", mlEnvId)); + } + + return map.get(mlEnvId); + } + + /** + * Get the MLEnvironment use the default MLEnvironmentId. + * + * @return the default MLEnvironment. + */ + public static synchronized MLEnvironment getDefault() { + return get(DEFAULT_ML_ENVIRONMENT_ID); + } + + /** + * Create a unique MLEnvironment id and register a new MLEnvironment in the factory. + * + * @return the MLEnvironment id. + */ + public static synchronized Long getNewMLEnvironmentId() { + return registerMLEnvironment(new MLEnvironment()); + } + + /** + * Register a new MLEnvironment to the factory and return a new MLEnvironment id. + * + * @param env the MLEnvironment that will be stored in the factory. + * @return the MLEnvironment id. + */ + public static synchronized Long registerMLEnvironment(MLEnvironment env) { + map.put(nextId, env); + return nextId++; + } + + /** + * Remove the MLEnvironment using the MLEnvironmentId. + * + * @param mlEnvId the id. + * @return the removed MLEnvironment + */ + public static synchronized MLEnvironment remove(Long mlEnvId) { + Preconditions.checkNotNull(mlEnvId, "The environment id cannot be null."); + // Never remove the default MLEnvironment. Just return the default environment. + if (DEFAULT_ML_ENVIRONMENT_ID.equals(mlEnvId)) { + return getDefault(); + } else { + return map.remove(mlEnvId); + } + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/AlgoOperator.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/AlgoOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..43c9a89cadfb897b1072f66e99af29a590acd3fb --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/AlgoOperator.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.operator; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.ml.api.misc.param.Params; +import org.apache.flink.ml.api.misc.param.WithParams; +import org.apache.flink.ml.params.shared.HasMLEnvironmentId; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.util.Preconditions; + +import java.io.Serializable; + +/** + * Base class for algorithm operators. + * + *

Base class for the algorithm operators. It hosts the parameters and output + * tables of an algorithm operator. Each AlgoOperator may have one or more output tables. + * One of the output table is the primary output table which can be obtained by calling + * {@link #getOutput}. The other output tables are side output tables that can be obtained + * by calling {@link #getSideOutputs()}. + * + *

The input of an AlgoOperator is defined in the subclasses of the AlgoOperator. + * + * @param The class type of the {@link AlgoOperator} implementation itself + */ +public abstract class AlgoOperator> + implements WithParams, HasMLEnvironmentId, Serializable { + + /** + * Params for algorithms. + */ + private Params params; + + /** + * The table held by operator. + */ + private Table output = null; + + /** + * The side outputs of operator that be similar to the stream's side outputs. + */ + private Table[] sideOutputs = null; + + /** + * Construct the operator with empty Params. + * + *

This constructor is especially useful when users want to set parameters + * for the algorithm operators. For example: + * SplitBatchOp is widely used in ML data pre-processing, + * which splits one dataset into two dataset: training set and validation set. + * It is very convenient for us to write code like this: + * 

+	 * {@code
+	 * new SplitBatchOp().setSplitRatio(0.9)
+	 * }
+	 *
+ */ + protected AlgoOperator() { + this(null); + } + + /** + * Construct the operator with the initial Params. + */ + protected AlgoOperator(Params params) { + if (null == params) { + this.params = new Params(); + } else { + this.params = params.clone(); + } + } + + @Override + public Params getParams() { + return this.params; + } + + /** + * Returns the table held by operator. + */ + public Table getOutput() { + return this.output; + } + + /** + * Returns the side outputs. + */ + public Table[] getSideOutputs() { + return this.sideOutputs; + } + + /** + * Set the side outputs. + * + * @param sideOutputs the side outputs set the operator. + */ + protected void setSideOutputs(Table[] sideOutputs) { + this.sideOutputs = sideOutputs; + } + + /** + * Set the table held by operator. + * + * @param output the output table. + */ + protected void setOutput(Table output) { + this.output = output; + } + + /** + * Returns the column names of the output table. + */ + public String[] getColNames() { + return getSchema().getFieldNames(); + } + + /** + * Returns the column types of the output table. + */ + public TypeInformation[] getColTypes() { + return getSchema().getFieldTypes(); + } + + /** + * Get the column names of the specified side-output table. + * + * @param index the index of the table. + * @return the column types of the table. + */ + public String[] getSideOutputColNames(int index) { + checkSideOutputAccessibility(index); + + return sideOutputs[index].getSchema().getFieldNames(); + } + + /** + * Get the column types of the specified side-output table. + * + * @param index the index of the table. + * @return the column types of the table. + */ + public TypeInformation[] getSideOutputColTypes(int index) { + checkSideOutputAccessibility(index); + + return sideOutputs[index].getSchema().getFieldTypes(); + } + + /** + * Returns the schema of the output table. + */ + public TableSchema getSchema() { + return this.getOutput().getSchema(); + } + + @Override + public String toString() { + return getOutput().toString(); + } + + protected static void checkOpSize(int size, AlgoOperator... inputs) { + Preconditions.checkNotNull(inputs, "Operators should not be null."); + Preconditions.checkState(inputs.length == size, "The size of operators should be equal to " + + size + ", current: " + inputs.length); + } + + protected static void checkMinOpSize(int size, AlgoOperator... inputs) { + Preconditions.checkNotNull(inputs, "Operators should not be null."); + Preconditions.checkState(inputs.length >= size, "The size of operators should be equal or greater than " + + size + ", current: " + inputs.length); + } + + private void checkSideOutputAccessibility(int index) { + Preconditions.checkNotNull(sideOutputs, + "There is not side-outputs in this AlgoOperator."); + Preconditions.checkState(index >= 0 && index < sideOutputs.length, + String.format("The index(%s) of side-outputs is out of bound.", index)); + Preconditions.checkNotNull(sideOutputs[index], + String.format("The %snd of side-outputs is null. Maybe the operator has not been linked.", index)); + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/BatchOperator.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/BatchOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..d655fe8f2a870799e6c94fca10894299c7f3e452 --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/BatchOperator.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.operator.batch; + +import org.apache.flink.ml.api.misc.param.Params; +import org.apache.flink.ml.operator.AlgoOperator; +import org.apache.flink.ml.operator.batch.source.TableSourceBatchOp; +import org.apache.flink.table.api.Table; + +/** + * Base class of batch algorithm operators. + * + *

This class extends {@link AlgoOperator} to support data transmission between BatchOperators. + */ +public abstract class BatchOperator> extends AlgoOperator { + + public BatchOperator() { + super(); + } + + /** + * The constructor of BatchOperator with {@link Params}. + * @param params the initial Params. + */ + public BatchOperator(Params params) { + super(params); + } + + /** + * Link to another {@link BatchOperator}. + * + *

Link the next BatchOperator using this BatchOperator as its input. + * + *

For example: + * + * 

+	 * {@code
+	 * BatchOperator a = ...;
+	 * BatchOperator b = ...;
+	 * BatchOperator c = a.link(b)
+	 * }
+	 *
+ * + *

The BatchOperator c in the above code + * is the same instance as b which takes + * a as its input. + * Note that BatchOperator b will be changed + * to link from BatchOperator a. + * + * @param next The operator that will be modified to add this operator to its input. + * @param type of BatchOperator returned + * @return the linked next + * @see #linkFrom(BatchOperator[]) + */ + public > B link(B next) { + next.linkFrom(this); + return next; + } + + /** + * Link from others {@link BatchOperator}. + * + *

Link this object to BatchOperator using the BatchOperators as its input. + * + *

For example: + * + * 

+	 * {@code
+	 * BatchOperator a = ...;
+	 * BatchOperator b = ...;
+	 * BatchOperator c = ...;
+	 *
+	 * BatchOperator d = c.linkFrom(a, b)
+	 * }
+	 *
+ * + *

The d in the above code is the same + * instance as BatchOperator c which takes + * both a and b as its input. + * + *

note: It is not recommended to linkFrom itself or linkFrom the same group inputs twice. + * + * @param inputs the linked inputs + * @return the linked this object + */ + public abstract T linkFrom(BatchOperator... inputs); + + /** + * create a new BatchOperator from table. + * @param table the input table + * @return the new BatchOperator + */ + public static BatchOperator fromTable(Table table) { + return new TableSourceBatchOp(table); + } + + protected static BatchOperator checkAndGetFirst(BatchOperator ... inputs) { + checkOpSize(1, inputs); + return inputs[0]; + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/source/TableSourceBatchOp.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/source/TableSourceBatchOp.java new file mode 100644 index 0000000000000000000000000000000000000000..7f8d6af5df9c6f9da0aaf0ca9d420d0436fcf82f --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/source/TableSourceBatchOp.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.operator.batch.source; + +import org.apache.flink.ml.operator.batch.BatchOperator; +import org.apache.flink.table.api.Table; +import org.apache.flink.util.Preconditions; + +/** + * Transform the Table to SourceBatchOp. + */ +public final class TableSourceBatchOp extends BatchOperator { + + public TableSourceBatchOp(Table table) { + super(null); + Preconditions.checkArgument(table != null, "The source table cannot be null."); + this.setOutput(table); + } + + @Override + public TableSourceBatchOp linkFrom(BatchOperator... inputs) { + throw new UnsupportedOperationException("Table source operator should not have any upstream to link from."); + } + +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/StreamOperator.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/StreamOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..870623afae3ddc1387e1a4428f2837689cb8ad06 --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/StreamOperator.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.operator.stream; + +import org.apache.flink.ml.api.misc.param.Params; +import org.apache.flink.ml.operator.AlgoOperator; +import org.apache.flink.ml.operator.stream.source.TableSourceStreamOp; +import org.apache.flink.table.api.Table; + +/** + * Base class of stream algorithm operators. + * + *

This class extends {@link AlgoOperator} to support data transmission between StreamOperator. + */ +public abstract class StreamOperator> extends AlgoOperator { + + public StreamOperator() { + super(); + } + + /** + * The constructor of StreamOperator with {@link Params}. + * + * @param params the initial Params. + */ + public StreamOperator(Params params) { + super(params); + } + + /** + * Link to another {@link StreamOperator}. + * + *

Link the next StreamOperator using this StreamOperator as its input. + * + *

For example: + * + * 

+	 * {@code
+	 * StreamOperator a = ...;
+	 * StreamOperator b = ...;
+	 *
+	 * StreamOperator c = a.link(b)
+	 * }
+	 *
+ * + *

The StreamOperator c in the above code + * is the same instance as b which takes + * a as its input. + * Note that StreamOperator b will be changed + * to link from StreamOperator a. + * + * @param next the linked StreamOperator + * @param type of StreamOperator returned + * @return the linked next + * @see #linkFrom(StreamOperator[]) + */ + public > S link(S next) { + next.linkFrom(this); + return next; + } + + /** + * Link from others {@link StreamOperator}. + * + *

Link this object to StreamOperator using the StreamOperators as its input. + * + *

For example: + * + * 

+	 * {@code
+	 * StreamOperator a = ...;
+	 * StreamOperator b = ...;
+	 * StreamOperator c = ...;
+	 *
+	 * StreamOperator d = c.linkFrom(a, b)
+	 * }
+	 *
+ * + *

The d in the above code is the same + * instance as StreamOperator c which takes + * both a and b as its input. + * + *

note: It is not recommended to linkFrom itself or linkFrom the same group inputs twice. + * + * @param inputs the linked inputs + * @return the linked this object + */ + public abstract T linkFrom(StreamOperator... inputs); + + /** + * create a new StreamOperator from table. + * + * @param table the input table + * @return the new StreamOperator + */ + public static StreamOperator fromTable(Table table) { + return new TableSourceStreamOp(table); + } + + protected static StreamOperator checkAndGetFirst(StreamOperator... inputs) { + checkOpSize(1, inputs); + return inputs[0]; + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/source/TableSourceStreamOp.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/source/TableSourceStreamOp.java new file mode 100644 index 0000000000000000000000000000000000000000..8ea2f02c154461ec7ae7ead1d627919d7c671478 --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/source/TableSourceStreamOp.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.operator.stream.source; + +import org.apache.flink.ml.operator.stream.StreamOperator; +import org.apache.flink.table.api.Table; +import org.apache.flink.util.Preconditions; + +/** + * Transform the Table to SourceStreamOp. + */ +public final class TableSourceStreamOp extends StreamOperator { + + public TableSourceStreamOp(Table table) { + super(null); + Preconditions.checkArgument(table != null, "The source table cannot be null."); + this.setOutput(table); + } + + @Override + public TableSourceStreamOp linkFrom(StreamOperator... inputs) { + throw new UnsupportedOperationException("Table source operator should not have any upstream to link from."); + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/HasMLEnvironmentId.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/HasMLEnvironmentId.java new file mode 100644 index 0000000000000000000000000000000000000000..ae7c7fe1980936492ea024c899b65d060333f8b1 --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/HasMLEnvironmentId.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.params.shared; + +import org.apache.flink.ml.api.misc.param.ParamInfo; +import org.apache.flink.ml.api.misc.param.ParamInfoFactory; +import org.apache.flink.ml.api.misc.param.WithParams; +import org.apache.flink.ml.common.MLEnvironmentFactory; + +/** + * An interface for classes with a parameter specifying the id of MLEnvironment. + */ +public interface HasMLEnvironmentId extends WithParams { + + ParamInfo ML_ENVIRONMENT_ID = ParamInfoFactory + .createParamInfo("MLEnvironmentId", Long.class) + .setDescription("ID of ML environment.") + .setHasDefaultValue(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID) + .build(); + + default Long getMLEnvironmentId() { + return get(ML_ENVIRONMENT_ID); + } + + default T setMLEnvironmentId(Long value) { + return set(ML_ENVIRONMENT_ID, value); + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java new file mode 100644 index 0000000000000000000000000000000000000000..3b6bcc143d71ac8ffe4767966bdb9d8303606e37 --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.pipeline; + +import org.apache.flink.ml.api.core.Estimator; +import org.apache.flink.ml.api.misc.param.Params; +import org.apache.flink.ml.operator.batch.BatchOperator; +import org.apache.flink.ml.operator.batch.source.TableSourceBatchOp; +import org.apache.flink.ml.operator.stream.StreamOperator; +import org.apache.flink.ml.operator.stream.source.TableSourceStreamOp; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.internal.TableImpl; +import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.util.Preconditions; + +/** + * The base class for estimator implementations. + * + * @param A subclass of the {@link EstimatorBase}, used by + * {@link org.apache.flink.ml.api.misc.param.WithParams} + * @param class type of the {@link ModelBase} this Estimator produces. + */ +public abstract class EstimatorBase, M extends ModelBase> + extends PipelineStageBase implements Estimator { + + public EstimatorBase() { + super(); + } + + public EstimatorBase(Params params) { + super(params); + } + + @Override + public M fit(TableEnvironment tEnv, Table input) { + Preconditions.checkArgument(input != null, "Input CAN NOT BE null!"); + Preconditions.checkArgument( + tableEnvOf(input) == tEnv, + "The input table is not in the specified table environment."); + return fit(input); + } + + /** + * Train and produce a {@link ModelBase} which fits the records in the given {@link Table}. + * + * @param input the table with records to train the Model. + * @return a model trained to fit on the given Table. + */ + public M fit(Table input) { + Preconditions.checkArgument(input != null, "Input CAN NOT BE null!"); + if (((TableImpl) input).getTableEnvironment() instanceof StreamTableEnvironment) { + TableSourceStreamOp source = new TableSourceStreamOp(input); + if (this.params.contains(ML_ENVIRONMENT_ID)) { + source.setMLEnvironmentId(this.params.get(ML_ENVIRONMENT_ID)); + } + return fit(source); + } else { + TableSourceBatchOp source = new TableSourceBatchOp(input); + if (this.params.contains(ML_ENVIRONMENT_ID)) { + source.setMLEnvironmentId(this.params.get(ML_ENVIRONMENT_ID)); + } + return fit(source); + } + } + + /** + * Train and produce a {@link ModelBase} which fits the records from the given {@link BatchOperator}. + * + * @param input the table with records to train the Model. + * @return a model trained to fit on the given Table. + */ + protected abstract M fit(BatchOperator input); + + /** + * Online learning and produce {@link ModelBase} series which fit the streaming records from the given {@link + * StreamOperator}. + * + * @param input the StreamOperator with streaming records to online train the Model series. + * @return the model series trained to fit on the streaming data from given StreamOperator. + */ + protected M fit(StreamOperator input) { + throw new UnsupportedOperationException("NOT supported yet!"); + } + +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/ModelBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/ModelBase.java new file mode 100644 index 0000000000000000000000000000000000000000..98f007fd4e8527675709ea0983aa8a0c355b0a7b --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/ModelBase.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.pipeline; + +import org.apache.flink.ml.api.core.Model; +import org.apache.flink.ml.api.misc.param.Params; +import org.apache.flink.table.api.Table; + +/** + * The base class for a machine learning model. + * + * @param The class type of the {@link ModelBase} implementation itself + */ +public abstract class ModelBase> extends TransformerBase + implements Model { + + protected Table modelData; + + public ModelBase() { + super(); + } + + public ModelBase(Params params) { + super(params); + } + + /** + * Get model data as Table representation. + * + * @return the Table + */ + public Table getModelData() { + return this.modelData; + } + + /** + * Set the model data using the Table. + * + * @param modelData the Table. + * @return {@link ModelBase} itself + */ + public M setModelData(Table modelData) { + this.modelData = modelData; + return (M) this; + } + + @Override + public M clone() throws CloneNotSupportedException { + return (M) super.clone().setModelData(this.modelData); + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/PipelineStageBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/PipelineStageBase.java new file mode 100644 index 0000000000000000000000000000000000000000..cb932569c7326e85c5b347171ed8a2d989d128d7 --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/PipelineStageBase.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.pipeline; + +import org.apache.flink.ml.api.misc.param.Params; +import org.apache.flink.ml.api.misc.param.WithParams; +import org.apache.flink.ml.params.shared.HasMLEnvironmentId; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.internal.TableImpl; + +/** + * The base class for a stage in a pipeline, either an [[EstimatorBase]] or a [[TransformerBase]]. + * + *

The PipelineStageBase maintains the parameters for the stage. + * A default constructor is needed in order to restore a pipeline stage. + * + * @param The class type of the {@link PipelineStageBase} implementation itself, used by {@link + * org.apache.flink.ml.api.misc.param.WithParams} and Cloneable. + */ +public abstract class PipelineStageBase> + implements WithParams , HasMLEnvironmentId, Cloneable { + protected Params params; + + public PipelineStageBase() { + this(null); + } + + public PipelineStageBase(Params params) { + if (null == params) { + this.params = new Params(); + } else { + this.params = params.clone(); + } + } + + @Override + public Params getParams() { + if (null == this.params) { + this.params = new Params(); + } + return this.params; + } + + @Override + public S clone() throws CloneNotSupportedException { + PipelineStageBase result = (PipelineStageBase) super.clone(); + result.params = this.params.clone(); + return (S) result; + } + + protected static TableEnvironment tableEnvOf(Table table) { + return ((TableImpl) table).getTableEnvironment(); + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java new file mode 100644 index 0000000000000000000000000000000000000000..ed3c374cbd90b46c64253f7fda82e7204bc286b9 --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.pipeline; + +import org.apache.flink.ml.api.core.Transformer; +import org.apache.flink.ml.api.misc.param.Params; +import org.apache.flink.ml.operator.batch.BatchOperator; +import org.apache.flink.ml.operator.batch.source.TableSourceBatchOp; +import org.apache.flink.ml.operator.stream.StreamOperator; +import org.apache.flink.ml.operator.stream.source.TableSourceStreamOp; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.java.StreamTableEnvironment; +import org.apache.flink.util.Preconditions; + +/** + * The base class for transformer implementations. + * + * @param A subclass of {@link TransformerBase}, used by {@link + * org.apache.flink.ml.api.misc.param.WithParams} + */ +public abstract class TransformerBase> + extends PipelineStageBase implements Transformer { + + public TransformerBase() { + super(); + } + + public TransformerBase(Params params) { + super(params); + } + + @Override + public Table transform(TableEnvironment tEnv, Table input) { + Preconditions.checkArgument(input != null, "Input CAN NOT BE null!"); + Preconditions.checkArgument( + tableEnvOf(input) == tEnv, + "The input table is not in the specified table environment."); + return transform(input); + } + + /** + * Applies the transformer on the input table, and returns the result table. + * + * @param input the table to be transformed + * @return the transformed table + */ + public Table transform(Table input) { + Preconditions.checkArgument(input != null, "Input CAN NOT BE null!"); + if (tableEnvOf(input) instanceof StreamTableEnvironment) { + TableSourceStreamOp source = new TableSourceStreamOp(input); + if (this.params.contains(ML_ENVIRONMENT_ID)) { + source.setMLEnvironmentId(this.params.get(ML_ENVIRONMENT_ID)); + } + return transform(source).getOutput(); + } else { + TableSourceBatchOp source = new TableSourceBatchOp(input); + if (this.params.contains(ML_ENVIRONMENT_ID)) { + source.setMLEnvironmentId(this.params.get(ML_ENVIRONMENT_ID)); + } + return transform(source).getOutput(); + } + } + + /** + * Applies the transformer on the input batch data from BatchOperator, and returns the batch result data with + * BatchOperator. + * + * @param input the input batch data from BatchOperator + * @return the transformed batch result data + */ + protected abstract BatchOperator transform(BatchOperator input); + + /** + * Applies the transformer on the input streaming data from StreamOperator, and returns the streaming result data + * with StreamOperator. + * + * @param input the input streaming data from StreamOperator + * @return the transformed streaming result data + */ + protected abstract StreamOperator transform(StreamOperator input); + +} diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java new file mode 100644 index 0000000000000000000000000000000000000000..60b18fe595598d93cb392ec1b62576391092d0ba --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.common; + +import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.java.BatchTableEnvironment; +import org.apache.flink.table.api.java.StreamTableEnvironment; + +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +/** + * Test cases for MLEnvironment. + */ +public class MLEnvironmentTest { + @Test + public void testDefaultConstructor() { + MLEnvironment mlEnvironment = new MLEnvironment(); + Assert.assertNotNull(mlEnvironment.getExecutionEnvironment()); + Assert.assertNotNull(mlEnvironment.getBatchTableEnvironment()); + Assert.assertNotNull(mlEnvironment.getStreamExecutionEnvironment()); + Assert.assertNotNull(mlEnvironment.getStreamTableEnvironment()); + } + + @Test + public void testConstructWithBatchEnv() { + ExecutionEnvironment executionEnvironment = ExecutionEnvironment.getExecutionEnvironment(); + BatchTableEnvironment batchTableEnvironment = BatchTableEnvironment.create(executionEnvironment); + + MLEnvironment mlEnvironment = new MLEnvironment(executionEnvironment, batchTableEnvironment); + + Assert.assertSame(mlEnvironment.getExecutionEnvironment(), executionEnvironment); + Assert.assertSame(mlEnvironment.getBatchTableEnvironment(), batchTableEnvironment); + } + + @Test + public void testConstructWithStreamEnv() { + StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment(); + StreamTableEnvironment streamTableEnvironment = StreamTableEnvironment.create(streamExecutionEnvironment); + + MLEnvironment mlEnvironment = new MLEnvironment(streamExecutionEnvironment, streamTableEnvironment); + + Assert.assertSame(mlEnvironment.getStreamExecutionEnvironment(), streamExecutionEnvironment); + Assert.assertSame(mlEnvironment.getStreamTableEnvironment(), streamTableEnvironment); + } + + @Test + public void testRemoveDefaultMLEnvironment() { + MLEnvironment defaultEnv = MLEnvironmentFactory.getDefault(); + MLEnvironmentFactory.remove(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID); + assertEquals("The default MLEnvironment should not have been removed", + defaultEnv, MLEnvironmentFactory.get(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID)); + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/EstimatorBaseTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/EstimatorBaseTest.java new file mode 100644 index 0000000000000000000000000000000000000000..17b36e25a181dd9a2c09a4cfb5b176fec965087a --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/EstimatorBaseTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.ml.pipeline; + +import org.apache.flink.api.java.DataSet; +import org.apache.flink.ml.common.MLEnvironment; +import org.apache.flink.ml.common.MLEnvironmentFactory; +import org.apache.flink.ml.operator.batch.BatchOperator; +import org.apache.flink.ml.operator.stream.StreamOperator; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.table.api.Table; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Test for {@link EstimatorBase}. + */ +public class EstimatorBaseTest extends PipelineStageTestBase { + + /** + * This fake estimator simply record which fit method is invoked. + */ + private static class FakeEstimator extends EstimatorBase { + + boolean batchFitted = false; + boolean streamFitted = false; + + @Override + public ModelBase fit(BatchOperator input) { + batchFitted = true; + return null; + } + + @Override + public ModelBase fit(StreamOperator input) { + streamFitted = true; + return null; + } + } + + @Override + protected PipelineStageBase createPipelineStage() { + return new FakeEstimator(); + } + + @Test + public void testFitBatchTable() { + Long id = MLEnvironmentFactory.getNewMLEnvironmentId(); + MLEnvironment env = MLEnvironmentFactory.get(id); + DataSet input = env.getExecutionEnvironment().fromElements(1, 2, 3); + Table table = env.getBatchTableEnvironment().fromDataSet(input); + + FakeEstimator estimator = new FakeEstimator(); + estimator.setMLEnvironmentId(id); + estimator.fit(env.getBatchTableEnvironment(), table); + + Assert.assertTrue(estimator.batchFitted); + Assert.assertFalse(estimator.streamFitted); + } + + @Test + public void testFitStreamTable() { + Long id = MLEnvironmentFactory.getNewMLEnvironmentId(); + MLEnvironment env = MLEnvironmentFactory.get(id); + DataStream input = env.getStreamExecutionEnvironment().fromElements(1, 2, 3); + Table table = env.getStreamTableEnvironment().fromDataStream(input); + + FakeEstimator estimator = new FakeEstimator(); + estimator.setMLEnvironmentId(id); + estimator.fit(env.getStreamTableEnvironment(), table); + + Assert.assertFalse(estimator.batchFitted); + Assert.assertTrue(estimator.streamFitted); + } +} diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/PipelineStageTestBase.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/PipelineStageTestBase.java new file mode 100644 index 0000000000000000000000000000000000000000..292bffcda7cfa3ef48c0c5d9c57e9a7c38aa517d --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/PipelineStageTestBase.java @@ -0,0 +1,67 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package org.apache.flink.ml.pipeline; + +import org.apache.flink.api.java.DataSet; +import org.apache.flink.ml.api.core.Estimator; +import org.apache.flink.ml.api.core.Transformer; +import org.apache.flink.ml.common.MLEnvironment; +import org.apache.flink.ml.common.MLEnvironmentFactory; +import org.apache.flink.table.api.Table; + +import org.junit.Test; + +/** + * The base class for testing the base implementation of pipeline stages, i.e. Estimators and Transformers. + * This class is package private because we do not expect extension outside of the package. + */ +abstract class PipelineStageTestBase { + + @Test(expected = IllegalArgumentException.class) + public void testMismatchTableEnvironment() { + Long id = MLEnvironmentFactory.getNewMLEnvironmentId(); + MLEnvironment env = MLEnvironmentFactory.get(id); + DataSet input = env.getExecutionEnvironment().fromElements(1, 2, 3); + Table t = env.getBatchTableEnvironment().fromDataSet(input); + + PipelineStageBase pipelineStageBase = createPipelineStage(); + pipelineStageBase.setMLEnvironmentId(id); + if (pipelineStageBase instanceof EstimatorBase) { + ((Estimator) pipelineStageBase).fit(MLEnvironmentFactory.getDefault().getBatchTableEnvironment(), t); + } else { + ((Transformer) pipelineStageBase).transform(MLEnvironmentFactory.getDefault().getBatchTableEnvironment(), t); + } + } + + @Test(expected = IllegalArgumentException.class) + public void testNullInputTable() { + Long id = MLEnvironmentFactory.getNewMLEnvironmentId(); + MLEnvironment env = MLEnvironmentFactory.get(id); + + PipelineStageBase pipelineStageBase = createPipelineStage(); + pipelineStageBase.setMLEnvironmentId(id); + if (pipelineStageBase instanceof Estimator) { + ((Estimator) pipelineStageBase).fit(env.getBatchTableEnvironment(), null); + } else { + ((Transformer) pipelineStageBase).transform(env.getBatchTableEnvironment(), null); + } + } + + protected abstract PipelineStageBase createPipelineStage(); +} diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/TransformerBaseTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/TransformerBaseTest.java new file mode 100644 index 0000000000000000000000000000000000000000..2797685a760be67d4aa8626a9af4875d3311ac7a --- /dev/null +++ b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/TransformerBaseTest.java @@ -0,0 +1,92 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +package org.apache.flink.ml.pipeline; + +import org.apache.flink.api.java.DataSet; +import org.apache.flink.ml.common.MLEnvironment; +import org.apache.flink.ml.common.MLEnvironmentFactory; +import org.apache.flink.ml.operator.batch.BatchOperator; +import org.apache.flink.ml.operator.stream.StreamOperator; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.table.api.Table; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Unit test for {@link TransformerBase}. + */ +public class TransformerBaseTest extends PipelineStageTestBase { + + /** + * This fake transformer simply record which transform method is invoked. + */ + private static class FakeTransFormer extends TransformerBase { + + boolean batchTransformed = false; + boolean streamTransformed = false; + + @Override + protected BatchOperator transform(BatchOperator input) { + batchTransformed = true; + return input; + } + + @Override + protected StreamOperator transform(StreamOperator input) { + streamTransformed = true; + return input; + } + } + + @Override + protected PipelineStageBase createPipelineStage() { + return new FakeTransFormer(); + } + + @Test + public void testFitBatchTable() { + Long id = MLEnvironmentFactory.getNewMLEnvironmentId(); + MLEnvironment env = MLEnvironmentFactory.get(id); + DataSet input = env.getExecutionEnvironment().fromElements(1, 2, 3); + Table table = env.getBatchTableEnvironment().fromDataSet(input); + + FakeTransFormer transFormer = new FakeTransFormer(); + transFormer.setMLEnvironmentId(id); + transFormer.transform(env.getBatchTableEnvironment(), table); + + Assert.assertTrue(transFormer.batchTransformed); + Assert.assertFalse(transFormer.streamTransformed); + } + + @Test + public void testFitStreamTable() { + Long id = MLEnvironmentFactory.getNewMLEnvironmentId(); + MLEnvironment env = MLEnvironmentFactory.get(id); + DataStream input = env.getStreamExecutionEnvironment().fromElements(1, 2, 3); + Table table = env.getStreamTableEnvironment().fromDataStream(input); + + FakeTransFormer transFormer = new FakeTransFormer(); + transFormer.setMLEnvironmentId(id); + transFormer.transform(env.getStreamTableEnvironment(), table); + + Assert.assertFalse(transFormer.batchTransformed); + Assert.assertTrue(transFormer.streamTransformed); + } +} diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java index 311c2861e747c192ec0f376ced7e65f633a6a996..3e8f2f02f6662b28b1fe25f6a94f5885f47a7341 100644 --- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java +++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/plan/OptimizedPlan.java @@ -32,7 +32,7 @@ import org.apache.flink.util.Visitor; * all operator strategies (sorting-merge join, hash join, sorted grouping, ...), * and the data exchange modes (batched, pipelined).

*/ -public class OptimizedPlan implements FlinkPlan, Visitable { +public class OptimizedPlan implements Visitable { /** The data sources in the plan. */ private final Collection dataSources; diff --git a/flink-python/README.md b/flink-python/README.md index 7f8c9361441160900e600fb07a4d8d2715e7ba74..29dc3c419fbd6eb9cb8f9287123dbbfe129a06c3 100644 --- a/flink-python/README.md +++ b/flink-python/README.md @@ -27,4 +27,17 @@ We can enter the directory where this README.md file is located and run test cas ## Python Requirements -PyFlink depends on Py4J (currently version 0.10.8.1). +PyFlink depends on Py4J (currently version 0.10.8.1) and CloudPickle (currently version 1.2.2). + +## Development notices + +Protocol buffer is used in this module and file `flink_fn_execution_pb2.py` is generated from `flink-fn-execution.proto`. Whenever `flink-fn-execution.proto` is updated, please re-generate `flink_fn_execution_pb2.py` by executing + +``` +python pyflink/gen_protos.py +``` + +PyFlink depends on the following libraries to execute the above script: +1. grpcio-tools (>=1.3.5,<=1.14.2) +2. setuptools (>=37.0.0) +3. pip (>=7.1.0) diff --git a/flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh b/flink-python/bin/pyflink-gateway-server.sh similarity index 100% rename from flink-dist/src/main/flink-bin/bin/pyflink-gateway-server.sh rename to flink-python/bin/pyflink-gateway-server.sh diff --git a/flink-dist/src/main/flink-bin/bin/pyflink-shell.sh b/flink-python/bin/pyflink-shell.sh similarity index 94% rename from flink-dist/src/main/flink-bin/bin/pyflink-shell.sh rename to flink-python/bin/pyflink-shell.sh index 2c83cc76af575361f58d5f7763d07398772ec79a..8722e2083181162752ff98d5ab9494aa890f01b8 100755 --- a/flink-dist/src/main/flink-bin/bin/pyflink-shell.sh +++ b/flink-python/bin/pyflink-shell.sh @@ -34,10 +34,11 @@ PYFLINK_PYTHON="${PYFLINK_PYTHON:-"python"}" export FLINK_BIN_DIR=$FLINK_BIN_DIR export FLINK_HOME -# Add pyflink & py4j to PYTHONPATH +# Add pyflink & py4j & cloudpickle to PYTHONPATH export PYTHONPATH="$FLINK_OPT_DIR/python/pyflink.zip:$PYTHONPATH" PY4J_ZIP=`echo "$FLINK_OPT_DIR"/python/py4j-*-src.zip` -export PYTHONPATH="$PY4J_ZIP:$PYTHONPATH" +CLOUDPICKLE_ZIP=`echo "$FLINK_OPT_DIR"/python/cloudpickle-*-src.zip` +export PYTHONPATH="$PY4J_ZIP:$CLOUDPICKLE_ZIP:$PYTHONPATH" PARSER="org.apache.flink.client.python.PythonShellParser" function parse_options() { diff --git a/flink-dist/src/main/flink-bin/bin/pyflink-udf-runner.sh b/flink-python/bin/pyflink-udf-runner.sh similarity index 60% rename from flink-dist/src/main/flink-bin/bin/pyflink-udf-runner.sh rename to flink-python/bin/pyflink-udf-runner.sh index d2f96e875a77632624a7894d35fca3aaa25e6471..edfb85c718ab88e98e5f0a62dbedf900ad569f9e 100755 --- a/flink-dist/src/main/flink-bin/bin/pyflink-udf-runner.sh +++ b/flink-python/bin/pyflink-udf-runner.sh @@ -32,16 +32,38 @@ if [[ "$python" = "" ]]; then python="python" fi -# Add pyflink & py4j to PYTHONPATH -PYFLINK_ZIP="$FLINK_OPT_DIR/python/pyflink.zip" -if [[ ! ${PYTHONPATH} =~ ${PYFLINK_ZIP} ]]; then +CURRENT_DIR=`pwd -P` +FLINK_SOURCE_ROOT_DIR=`cd $bin/../../../../../; pwd -P` +cd $CURRENT_DIR + +# Add pyflink to PYTHONPATH +FLINK_PYTHON="${FLINK_SOURCE_ROOT_DIR}/flink-python" +if [[ ! -f "${FLINK_PYTHON}/pyflink/fn_execution/boot.py" ]]; then + # Add pyflink.zip to PYTHONPATH if directory pyflink does not exist + PYFLINK_ZIP="$FLINK_OPT_DIR/python/pyflink.zip" + if [[ ! ${PYTHONPATH} =~ ${PYFLINK_ZIP} ]]; then export PYTHONPATH="$PYFLINK_ZIP:$PYTHONPATH" + fi +else + # Add directory flink-python/pyflink to PYTHONPATH if it exists, this is helpful during + # development as this script is used to start up the Python worker and putting the + # directory of flink-python/pyflink to PYTHONPATH makes sure the Python source code will + # take effect immediately after changed. + export PYTHONPATH="$FLINK_PYTHON:$PYTHONPATH" fi + +# Add py4j to PYTHONPATH PY4J_ZIP=`echo "$FLINK_OPT_DIR"/python/py4j-*-src.zip` if [[ ! ${PYTHONPATH} =~ ${PY4J_ZIP} ]]; then export PYTHONPATH="$PY4J_ZIP:$PYTHONPATH" fi +# Add cloudpickle to PYTHONPATH +CLOUDPICKLE_ZIP=`echo "$FLINK_OPT_DIR"/python/cloudpickle-*-src.zip` +if [[ ! ${PYTHONPATH} =~ ${CLOUDPICKLE_ZIP} ]]; then + export PYTHONPATH="$CLOUDPICKLE_ZIP:$PYTHONPATH" +fi + log="$FLINK_LOG_DIR/flink-$FLINK_IDENT_STRING-python-udf-boot-$HOSTNAME.log" ${python} -m pyflink.fn_execution.boot $@ 2>&1 | tee -a ${log} diff --git a/flink-python/lib/cloudpickle-1.2.2-src.zip b/flink-python/lib/cloudpickle-1.2.2-src.zip new file mode 100644 index 0000000000000000000000000000000000000000..82bc04ec2740210a4fbd2c4cd1ba0947be478b07 Binary files /dev/null and b/flink-python/lib/cloudpickle-1.2.2-src.zip differ diff --git a/licenses-binary/LICENSE-protobuf b/flink-python/lib/cloudpickle-LICENSE.txt similarity index 35% rename from licenses-binary/LICENSE-protobuf rename to flink-python/lib/cloudpickle-LICENSE.txt index 21645bec9d5d13871408aed2b3e1ee13e3db4a93..d112c4806aa90b4c911b360fa0ce1d0ee7c031ce 100644 --- a/licenses-binary/LICENSE-protobuf +++ b/flink-python/lib/cloudpickle-LICENSE.txt @@ -1,36 +1,32 @@ -Copyright 2008, Google Inc. +This module was extracted from the `cloud` package, developed by +PiCloud, Inc. + +Copyright (c) 2015, Cloudpickle contributors. +Copyright (c) 2012, Regents of the University of California. +Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. All rights reserved. Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - +modification, are permitted provided that the following conditions +are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - * Neither the name of Google Inc. nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California, Berkeley nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. - +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-python/pom.xml b/flink-python/pom.xml index 4c63f39561f853e9e2e28a5d97ba8cc9b6a3feb9..ed16daa09e3a8946087e92653be9210f8fe107ea 100644 --- a/flink-python/pom.xml +++ b/flink-python/pom.xml @@ -74,6 +74,12 @@ under the License. ${project.version} provided + + org.apache.flink + flink-table-planner-blink_${scala.binary.version} + ${project.version} + provided + @@ -197,9 +203,15 @@ under the License. - + + includes="**/*.pyc,**/__pycache__"/> + + + + + + @@ -217,9 +229,15 @@ under the License. - + + includes="**/*.pyc,**/__pycache__"/> + + + + + + diff --git a/flink-python/pyflink/fn_execution/boot.py b/flink-python/pyflink/fn_execution/boot.py index e42ad2eea58857da110c7a139507ebe53de2e865..bf913846fb2bfc67c71d86d3efd85aba8d16dd45 100644 --- a/flink-python/pyflink/fn_execution/boot.py +++ b/flink-python/pyflink/fn_execution/boot.py @@ -145,5 +145,5 @@ env = dict(os.environ) if "FLINK_BOOT_TESTING" in os.environ and os.environ["FLINK_BOOT_TESTING"] == "1": exit(0) -call([sys.executable, "-m", "apache_beam.runners.worker.sdk_worker_main"], +call([sys.executable, "-m", "pyflink.fn_execution.sdk_worker_main"], stdout=sys.stdout, stderr=sys.stderr, env=env) diff --git a/flink-python/pyflink/fn_execution/coder_impl.py b/flink-python/pyflink/fn_execution/coder_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..2f7c5d3ab6536f83195b75f96d78ec614958dd50 --- /dev/null +++ b/flink-python/pyflink/fn_execution/coder_impl.py @@ -0,0 +1,80 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +import sys + +from apache_beam.coders.coder_impl import StreamCoderImpl + +if sys.version > '3': + xrange = range + + +class RowCoderImpl(StreamCoderImpl): + + def __init__(self, field_coders): + self._field_coders = field_coders + + def encode_to_stream(self, value, out_stream, nested): + self.write_null_mask(value, out_stream) + for i in xrange(len(self._field_coders)): + self._field_coders[i].encode_to_stream(value[i], out_stream, nested) + + def decode_from_stream(self, in_stream, nested): + from pyflink.table import Row + null_mask = self.read_null_mask(len(self._field_coders), in_stream) + assert len(null_mask) == len(self._field_coders) + return Row(*[None if null_mask[idx] else self._field_coders[idx].decode_from_stream( + in_stream, nested) for idx in xrange(0, len(null_mask))]) + + @staticmethod + def write_null_mask(value, out_stream): + field_pos = 0 + field_count = len(value) + while field_pos < field_count: + b = 0x00 + # set bits in byte + num_pos = min(8, field_count - field_pos) + byte_pos = 0 + while byte_pos < num_pos: + b = b << 1 + # set bit if field is null + if value[field_pos + byte_pos] is None: + b |= 0x01 + byte_pos += 1 + field_pos += num_pos + # shift bits if last byte is not completely filled + b <<= (8 - byte_pos) + # write byte + out_stream.write_byte(b) + + @staticmethod + def read_null_mask(field_count, in_stream): + null_mask = [] + field_pos = 0 + while field_pos < field_count: + b = in_stream.read_byte() + num_pos = min(8, field_count - field_pos) + byte_pos = 0 + while byte_pos < num_pos: + null_mask.append((b & 0x80) > 0) + b = b << 1 + byte_pos += 1 + field_pos += num_pos + return null_mask + + def __repr__(self): + return 'RowCoderImpl[%s]' % ', '.join(str(c) for c in self._field_coders) diff --git a/flink-python/pyflink/fn_execution/coders.py b/flink-python/pyflink/fn_execution/coders.py new file mode 100644 index 0000000000000000000000000000000000000000..c6add86477aba09836296cacc8c1887ff2e0a8c7 --- /dev/null +++ b/flink-python/pyflink/fn_execution/coders.py @@ -0,0 +1,86 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +import sys + +from apache_beam.coders import Coder, VarIntCoder +from apache_beam.coders.coders import FastCoder + +from pyflink.fn_execution import coder_impl +from pyflink.fn_execution import flink_fn_execution_pb2 + +FLINK_SCHEMA_CODER_URN = "flink:coder:schema:v1" + +if sys.version > '3': + xrange = range + + +__all__ = ['RowCoder'] + + +class RowCoder(FastCoder): + """ + Coder for Row. + """ + + def __init__(self, field_coders): + self._field_coders = field_coders + + def _create_impl(self): + return coder_impl.RowCoderImpl([c.get_impl() for c in self._field_coders]) + + def is_deterministic(self): + return all(c.is_deterministic() for c in self._field_coders) + + def to_type_hint(self): + from pyflink.table import Row + return Row + + def __repr__(self): + return 'RowCoder[%s]' % ', '.join(str(c) for c in self._field_coders) + + def __eq__(self, other): + return (self.__class__ == other.__class__ + and len(self._field_coders) == len(other._field_coders) + and [self._field_coders[i] == other._field_coders[i] for i in + xrange(len(self._field_coders))]) + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self._field_coders) + + +@Coder.register_urn(FLINK_SCHEMA_CODER_URN, flink_fn_execution_pb2.Schema) +def _pickle_from_runner_api_parameter(schema_proto, unused_components, unused_context): + return RowCoder([from_proto(f.type) for f in schema_proto.fields]) + + +def from_proto(field_type): + """ + Creates the corresponding :class:`Coder` given the protocol representation of the field type. + + :param field_type: the protocol representation of the field type + :return: :class:`Coder` + """ + if field_type.type_name == flink_fn_execution_pb2.Schema.TypeName.BIGINT: + return VarIntCoder() + elif field_type.type_name == flink_fn_execution_pb2.Schema.TypeName.ROW: + return RowCoder([from_proto(f.type) for f in field_type.row_schema.fields]) + else: + raise ValueError("field_type %s is not supported." % field_type) diff --git a/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..f6106a0fb1c56f110e6c03b4ba617255ab3b4094 --- /dev/null +++ b/flink-python/pyflink/fn_execution/flink_fn_execution_pb2.py @@ -0,0 +1,519 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: flink-fn-execution.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='flink-fn-execution.proto', + package='org.apache.flink.fn_execution.v1', + syntax='proto3', + serialized_pb=_b('\n\x18\x66link-fn-execution.proto\x12 org.apache.flink.fn_execution.v1\"\xfc\x01\n\x13UserDefinedFunction\x12\x0f\n\x07payload\x18\x01 \x01(\x0c\x12K\n\x06inputs\x18\x02 \x03(\x0b\x32;.org.apache.flink.fn_execution.v1.UserDefinedFunction.Input\x1a\x86\x01\n\x05Input\x12\x44\n\x03udf\x18\x01 \x01(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunctionH\x00\x12\x15\n\x0binputOffset\x18\x02 \x01(\x05H\x00\x12\x17\n\rinputConstant\x18\x03 \x01(\x0cH\x00\x42\x07\n\x05input\"[\n\x14UserDefinedFunctions\x12\x43\n\x04udfs\x18\x01 \x03(\x0b\x32\x35.org.apache.flink.fn_execution.v1.UserDefinedFunction\"\x8d\x07\n\x06Schema\x12>\n\x06\x66ields\x18\x01 \x03(\x0b\x32..org.apache.flink.fn_execution.v1.Schema.Field\x1a\x97\x01\n\x07MapType\x12\x44\n\x08key_type\x18\x01 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x12\x46\n\nvalue_type\x18\x02 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\x1a\xcd\x02\n\tFieldType\x12\x44\n\ttype_name\x18\x01 \x01(\x0e\x32\x31.org.apache.flink.fn_execution.v1.Schema.TypeName\x12\x10\n\x08nullable\x18\x02 \x01(\x08\x12U\n\x17\x63ollection_element_type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldTypeH\x00\x12\x44\n\x08map_type\x18\x04 \x01(\x0b\x32\x30.org.apache.flink.fn_execution.v1.Schema.MapTypeH\x00\x12>\n\nrow_schema\x18\x05 \x01(\x0b\x32(.org.apache.flink.fn_execution.v1.SchemaH\x00\x42\x0b\n\ttype_info\x1al\n\x05\x46ield\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12@\n\x04type\x18\x03 \x01(\x0b\x32\x32.org.apache.flink.fn_execution.v1.Schema.FieldType\"\xea\x01\n\x08TypeName\x12\x07\n\x03ROW\x10\x00\x12\x0b\n\x07TINYINT\x10\x01\x12\x0c\n\x08SMALLINT\x10\x02\x12\x07\n\x03INT\x10\x03\x12\n\n\x06\x42IGINT\x10\x04\x12\x0b\n\x07\x44\x45\x43IMAL\x10\x05\x12\t\n\x05\x46LOAT\x10\x06\x12\n\n\x06\x44OUBLE\x10\x07\x12\x08\n\x04\x44\x41TE\x10\x08\x12\x08\n\x04TIME\x10\t\x12\x0c\n\x08\x44\x41TETIME\x10\n\x12\x0b\n\x07\x42OOLEAN\x10\x0b\x12\n\n\x06\x42INARY\x10\x0c\x12\r\n\tVARBINARY\x10\r\x12\x08\n\x04\x43HAR\x10\x0e\x12\x0b\n\x07VARCHAR\x10\x0f\x12\t\n\x05\x41RRAY\x10\x10\x12\x07\n\x03MAP\x10\x11\x12\x0c\n\x08MULTISET\x10\x12\x42-\n\x1forg.apache.flink.fnexecution.v1B\nFlinkFnApib\x06proto3') +) + + + +_SCHEMA_TYPENAME = _descriptor.EnumDescriptor( + name='TypeName', + full_name='org.apache.flink.fn_execution.v1.Schema.TypeName', + filename=None, + file=DESCRIPTOR, + values=[ + _descriptor.EnumValueDescriptor( + name='ROW', index=0, number=0, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TINYINT', index=1, number=1, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='SMALLINT', index=2, number=2, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='INT', index=3, number=3, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='BIGINT', index=4, number=4, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DECIMAL', index=5, number=5, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='FLOAT', index=6, number=6, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DOUBLE', index=7, number=7, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DATE', index=8, number=8, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='TIME', index=9, number=9, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='DATETIME', index=10, number=10, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='BOOLEAN', index=11, number=11, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='BINARY', index=12, number=12, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='VARBINARY', index=13, number=13, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='CHAR', index=14, number=14, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='VARCHAR', index=15, number=15, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='ARRAY', index=16, number=16, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MAP', index=17, number=17, + options=None, + type=None), + _descriptor.EnumValueDescriptor( + name='MULTISET', index=18, number=18, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=1086, + serialized_end=1320, +) +_sym_db.RegisterEnumDescriptor(_SCHEMA_TYPENAME) + + +_USERDEFINEDFUNCTION_INPUT = _descriptor.Descriptor( + name='Input', + full_name='org.apache.flink.fn_execution.v1.UserDefinedFunction.Input', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='udf', full_name='org.apache.flink.fn_execution.v1.UserDefinedFunction.Input.udf', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='inputOffset', full_name='org.apache.flink.fn_execution.v1.UserDefinedFunction.Input.inputOffset', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='inputConstant', full_name='org.apache.flink.fn_execution.v1.UserDefinedFunction.Input.inputConstant', index=2, + number=3, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='input', full_name='org.apache.flink.fn_execution.v1.UserDefinedFunction.Input.input', + index=0, containing_type=None, fields=[]), + ], + serialized_start=181, + serialized_end=315, +) + +_USERDEFINEDFUNCTION = _descriptor.Descriptor( + name='UserDefinedFunction', + full_name='org.apache.flink.fn_execution.v1.UserDefinedFunction', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='payload', full_name='org.apache.flink.fn_execution.v1.UserDefinedFunction.payload', index=0, + number=1, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=_b(""), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='inputs', full_name='org.apache.flink.fn_execution.v1.UserDefinedFunction.inputs', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_USERDEFINEDFUNCTION_INPUT, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=63, + serialized_end=315, +) + + +_USERDEFINEDFUNCTIONS = _descriptor.Descriptor( + name='UserDefinedFunctions', + full_name='org.apache.flink.fn_execution.v1.UserDefinedFunctions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='udfs', full_name='org.apache.flink.fn_execution.v1.UserDefinedFunctions.udfs', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=317, + serialized_end=408, +) + + +_SCHEMA_MAPTYPE = _descriptor.Descriptor( + name='MapType', + full_name='org.apache.flink.fn_execution.v1.Schema.MapType', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key_type', full_name='org.apache.flink.fn_execution.v1.Schema.MapType.key_type', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='value_type', full_name='org.apache.flink.fn_execution.v1.Schema.MapType.value_type', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=486, + serialized_end=637, +) + +_SCHEMA_FIELDTYPE = _descriptor.Descriptor( + name='FieldType', + full_name='org.apache.flink.fn_execution.v1.Schema.FieldType', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='type_name', full_name='org.apache.flink.fn_execution.v1.Schema.FieldType.type_name', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='nullable', full_name='org.apache.flink.fn_execution.v1.Schema.FieldType.nullable', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='collection_element_type', full_name='org.apache.flink.fn_execution.v1.Schema.FieldType.collection_element_type', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='map_type', full_name='org.apache.flink.fn_execution.v1.Schema.FieldType.map_type', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='row_schema', full_name='org.apache.flink.fn_execution.v1.Schema.FieldType.row_schema', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='type_info', full_name='org.apache.flink.fn_execution.v1.Schema.FieldType.type_info', + index=0, containing_type=None, fields=[]), + ], + serialized_start=640, + serialized_end=973, +) + +_SCHEMA_FIELD = _descriptor.Descriptor( + name='Field', + full_name='org.apache.flink.fn_execution.v1.Schema.Field', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='org.apache.flink.fn_execution.v1.Schema.Field.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='description', full_name='org.apache.flink.fn_execution.v1.Schema.Field.description', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='type', full_name='org.apache.flink.fn_execution.v1.Schema.Field.type', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=975, + serialized_end=1083, +) + +_SCHEMA = _descriptor.Descriptor( + name='Schema', + full_name='org.apache.flink.fn_execution.v1.Schema', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='fields', full_name='org.apache.flink.fn_execution.v1.Schema.fields', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_SCHEMA_MAPTYPE, _SCHEMA_FIELDTYPE, _SCHEMA_FIELD, ], + enum_types=[ + _SCHEMA_TYPENAME, + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=411, + serialized_end=1320, +) + +_USERDEFINEDFUNCTION_INPUT.fields_by_name['udf'].message_type = _USERDEFINEDFUNCTION +_USERDEFINEDFUNCTION_INPUT.containing_type = _USERDEFINEDFUNCTION +_USERDEFINEDFUNCTION_INPUT.oneofs_by_name['input'].fields.append( + _USERDEFINEDFUNCTION_INPUT.fields_by_name['udf']) +_USERDEFINEDFUNCTION_INPUT.fields_by_name['udf'].containing_oneof = _USERDEFINEDFUNCTION_INPUT.oneofs_by_name['input'] +_USERDEFINEDFUNCTION_INPUT.oneofs_by_name['input'].fields.append( + _USERDEFINEDFUNCTION_INPUT.fields_by_name['inputOffset']) +_USERDEFINEDFUNCTION_INPUT.fields_by_name['inputOffset'].containing_oneof = _USERDEFINEDFUNCTION_INPUT.oneofs_by_name['input'] +_USERDEFINEDFUNCTION_INPUT.oneofs_by_name['input'].fields.append( + _USERDEFINEDFUNCTION_INPUT.fields_by_name['inputConstant']) +_USERDEFINEDFUNCTION_INPUT.fields_by_name['inputConstant'].containing_oneof = _USERDEFINEDFUNCTION_INPUT.oneofs_by_name['input'] +_USERDEFINEDFUNCTION.fields_by_name['inputs'].message_type = _USERDEFINEDFUNCTION_INPUT +_USERDEFINEDFUNCTIONS.fields_by_name['udfs'].message_type = _USERDEFINEDFUNCTION +_SCHEMA_MAPTYPE.fields_by_name['key_type'].message_type = _SCHEMA_FIELDTYPE +_SCHEMA_MAPTYPE.fields_by_name['value_type'].message_type = _SCHEMA_FIELDTYPE +_SCHEMA_MAPTYPE.containing_type = _SCHEMA +_SCHEMA_FIELDTYPE.fields_by_name['type_name'].enum_type = _SCHEMA_TYPENAME +_SCHEMA_FIELDTYPE.fields_by_name['collection_element_type'].message_type = _SCHEMA_FIELDTYPE +_SCHEMA_FIELDTYPE.fields_by_name['map_type'].message_type = _SCHEMA_MAPTYPE +_SCHEMA_FIELDTYPE.fields_by_name['row_schema'].message_type = _SCHEMA +_SCHEMA_FIELDTYPE.containing_type = _SCHEMA +_SCHEMA_FIELDTYPE.oneofs_by_name['type_info'].fields.append( + _SCHEMA_FIELDTYPE.fields_by_name['collection_element_type']) +_SCHEMA_FIELDTYPE.fields_by_name['collection_element_type'].containing_oneof = _SCHEMA_FIELDTYPE.oneofs_by_name['type_info'] +_SCHEMA_FIELDTYPE.oneofs_by_name['type_info'].fields.append( + _SCHEMA_FIELDTYPE.fields_by_name['map_type']) +_SCHEMA_FIELDTYPE.fields_by_name['map_type'].containing_oneof = _SCHEMA_FIELDTYPE.oneofs_by_name['type_info'] +_SCHEMA_FIELDTYPE.oneofs_by_name['type_info'].fields.append( + _SCHEMA_FIELDTYPE.fields_by_name['row_schema']) +_SCHEMA_FIELDTYPE.fields_by_name['row_schema'].containing_oneof = _SCHEMA_FIELDTYPE.oneofs_by_name['type_info'] +_SCHEMA_FIELD.fields_by_name['type'].message_type = _SCHEMA_FIELDTYPE +_SCHEMA_FIELD.containing_type = _SCHEMA +_SCHEMA.fields_by_name['fields'].message_type = _SCHEMA_FIELD +_SCHEMA_TYPENAME.containing_type = _SCHEMA +DESCRIPTOR.message_types_by_name['UserDefinedFunction'] = _USERDEFINEDFUNCTION +DESCRIPTOR.message_types_by_name['UserDefinedFunctions'] = _USERDEFINEDFUNCTIONS +DESCRIPTOR.message_types_by_name['Schema'] = _SCHEMA +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +UserDefinedFunction = _reflection.GeneratedProtocolMessageType('UserDefinedFunction', (_message.Message,), dict( + + Input = _reflection.GeneratedProtocolMessageType('Input', (_message.Message,), dict( + DESCRIPTOR = _USERDEFINEDFUNCTION_INPUT, + __module__ = 'flink_fn_execution_pb2' + # @@protoc_insertion_point(class_scope:org.apache.flink.fn_execution.v1.UserDefinedFunction.Input) + )) + , + DESCRIPTOR = _USERDEFINEDFUNCTION, + __module__ = 'flink_fn_execution_pb2' + # @@protoc_insertion_point(class_scope:org.apache.flink.fn_execution.v1.UserDefinedFunction) + )) +_sym_db.RegisterMessage(UserDefinedFunction) +_sym_db.RegisterMessage(UserDefinedFunction.Input) + +UserDefinedFunctions = _reflection.GeneratedProtocolMessageType('UserDefinedFunctions', (_message.Message,), dict( + DESCRIPTOR = _USERDEFINEDFUNCTIONS, + __module__ = 'flink_fn_execution_pb2' + # @@protoc_insertion_point(class_scope:org.apache.flink.fn_execution.v1.UserDefinedFunctions) + )) +_sym_db.RegisterMessage(UserDefinedFunctions) + +Schema = _reflection.GeneratedProtocolMessageType('Schema', (_message.Message,), dict( + + MapType = _reflection.GeneratedProtocolMessageType('MapType', (_message.Message,), dict( + DESCRIPTOR = _SCHEMA_MAPTYPE, + __module__ = 'flink_fn_execution_pb2' + # @@protoc_insertion_point(class_scope:org.apache.flink.fn_execution.v1.Schema.MapType) + )) + , + + FieldType = _reflection.GeneratedProtocolMessageType('FieldType', (_message.Message,), dict( + DESCRIPTOR = _SCHEMA_FIELDTYPE, + __module__ = 'flink_fn_execution_pb2' + # @@protoc_insertion_point(class_scope:org.apache.flink.fn_execution.v1.Schema.FieldType) + )) + , + + Field = _reflection.GeneratedProtocolMessageType('Field', (_message.Message,), dict( + DESCRIPTOR = _SCHEMA_FIELD, + __module__ = 'flink_fn_execution_pb2' + # @@protoc_insertion_point(class_scope:org.apache.flink.fn_execution.v1.Schema.Field) + )) + , + DESCRIPTOR = _SCHEMA, + __module__ = 'flink_fn_execution_pb2' + # @@protoc_insertion_point(class_scope:org.apache.flink.fn_execution.v1.Schema) + )) +_sym_db.RegisterMessage(Schema) +_sym_db.RegisterMessage(Schema.MapType) +_sym_db.RegisterMessage(Schema.FieldType) +_sym_db.RegisterMessage(Schema.Field) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\037org.apache.flink.fnexecution.v1B\nFlinkFnApi')) +# @@protoc_insertion_point(module_scope) diff --git a/flink-python/pyflink/fn_execution/operations.py b/flink-python/pyflink/fn_execution/operations.py new file mode 100644 index 0000000000000000000000000000000000000000..4bd955cc71e3b2eb33ca3e350e4b0dd13acccc20 --- /dev/null +++ b/flink-python/pyflink/fn_execution/operations.py @@ -0,0 +1,304 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import datetime +from abc import abstractmethod, ABCMeta + +from apache_beam.runners.worker import operation_specs +from apache_beam.runners.worker import bundle_processor +from apache_beam.runners.worker.operations import Operation + +from pyflink.fn_execution import flink_fn_execution_pb2 +from pyflink.serializers import PickleSerializer + +SCALAR_FUNCTION_URN = "flink:transform:scalar_function:v1" + + +class InputGetter(object): + """ + Base class for get an input argument for a :class:`UserDefinedFunction`. + """ + __metaclass__ = ABCMeta + + def open(self): + pass + + def close(self): + pass + + @abstractmethod + def get(self, value): + pass + + +class OffsetInputGetter(InputGetter): + """ + InputGetter for the input argument which is a column of the input row. + + :param input_offset: the offset of the column in the input row + """ + + def __init__(self, input_offset): + self.input_offset = input_offset + + def get(self, value): + return value[self.input_offset] + + +class ScalarFunctionInputGetter(InputGetter): + """ + InputGetter for the input argument which is a Python :class:`ScalarFunction`. This is used for + chaining Python functions. + + :param scalar_function_proto: the proto representation of the Python :class:`ScalarFunction` + """ + + def __init__(self, scalar_function_proto): + self.scalar_function_invoker = create_scalar_function_invoker(scalar_function_proto) + + def open(self): + self.scalar_function_invoker.invoke_open() + + def close(self): + self.scalar_function_invoker.invoke_close() + + def get(self, value): + return self.scalar_function_invoker.invoke_eval(value) + + +class ConstantInputGetter(InputGetter): + """ + InputGetter for the input argument which is a constant value. + + :param constant_value: the constant value of the column + """ + + def __init__(self, constant_value): + j_type = constant_value[0] + serializer = PickleSerializer() + pickled_data = serializer.loads(constant_value[1:]) + # the type set contains + # TINYINT,SMALLINT,INTEGER,BIGINT,FLOAT,DOUBLE,DECIMAL,CHAR,VARCHAR,NULL,BOOLEAN + # the pickled_data doesn't need to transfer to anther python object + if j_type == '\x00' or j_type == 0: + self._constant_value = pickled_data + # the type is DATE + elif j_type == '\x01' or j_type == 1: + self._constant_value = \ + datetime.date(year=1970, month=1, day=1) + datetime.timedelta(days=pickled_data) + # the type is TIME + elif j_type == '\x02' or j_type == 2: + seconds, milliseconds = divmod(pickled_data, 1000) + minutes, seconds = divmod(seconds, 60) + hours, minutes = divmod(minutes, 60) + self._constant_value = datetime.time(hours, minutes, seconds, milliseconds * 1000) + # the type is TIMESTAMP + elif j_type == '\x03' or j_type == 3: + self._constant_value = \ + datetime.datetime(year=1970, month=1, day=1, hour=0, minute=0, second=0) \ + + datetime.timedelta(milliseconds=pickled_data) + else: + raise Exception("Unknown type %s, should never happen" % str(j_type)) + + def get(self, value): + return self._constant_value + + +class ScalarFunctionInvoker(object): + """ + An abstraction that can be used to execute :class:`ScalarFunction` methods. + + A ScalarFunctionInvoker describes a particular way for invoking methods of a + :class:`ScalarFunction`. + + :param scalar_function: the :class:`ScalarFunction` to execute + :param inputs: the input arguments for the :class:`ScalarFunction` + """ + + def __init__(self, scalar_function, inputs): + self.scalar_function = scalar_function + self.input_getters = [] + for input in inputs: + if input.HasField("udf"): + # for chaining Python UDF input: the input argument is a Python ScalarFunction + self.input_getters.append(ScalarFunctionInputGetter(input.udf)) + elif input.HasField("inputOffset"): + # the input argument is a column of the input row + self.input_getters.append(OffsetInputGetter(input.inputOffset)) + else: + # the input argument is a constant value + self.input_getters.append(ConstantInputGetter(input.inputConstant)) + + def invoke_open(self): + """ + Invokes the ScalarFunction.open() function. + """ + for input_getter in self.input_getters: + input_getter.open() + # set the FunctionContext to None for now + self.scalar_function.open(None) + + def invoke_close(self): + """ + Invokes the ScalarFunction.close() function. + """ + for input_getter in self.input_getters: + input_getter.close() + self.scalar_function.close() + + def invoke_eval(self, value): + """ + Invokes the ScalarFunction.eval() function. + + :param value: the input element for which eval() method should be invoked + """ + args = [input_getter.get(value) for input_getter in self.input_getters] + return self.scalar_function.eval(*args) + + +def create_scalar_function_invoker(scalar_function_proto): + """ + Creates :class:`ScalarFunctionInvoker` from the proto representation of a + :class:`ScalarFunction`. + + :param scalar_function_proto: the proto representation of the Python :class:`ScalarFunction` + :return: :class:`ScalarFunctionInvoker`. + """ + import cloudpickle + scalar_function = cloudpickle.loads(scalar_function_proto.payload) + return ScalarFunctionInvoker(scalar_function, scalar_function_proto.inputs) + + +class ScalarFunctionRunner(object): + """ + The runner which is responsible for executing the scalar functions and send the + execution results back to the remote Java operator. + + :param udfs_proto: protocol representation for the scalar functions to execute + """ + + def __init__(self, udfs_proto): + self.scalar_function_invokers = [create_scalar_function_invoker(f) for f in + udfs_proto] + + def setup(self, main_receivers): + """ + Set up the ScalarFunctionRunner. + + :param main_receivers: Receiver objects which is responsible for sending the execution + results back the the remote Java operator + """ + from apache_beam.runners.common import _OutputProcessor + self.output_processor = _OutputProcessor( + window_fn=None, + main_receivers=main_receivers, + tagged_receivers=None, + per_element_output_counter=None) + + def open(self): + for invoker in self.scalar_function_invokers: + invoker.invoke_open() + + def close(self): + for invoker in self.scalar_function_invokers: + invoker.invoke_close() + + def process(self, windowed_value): + results = [invoker.invoke_eval(windowed_value.value) for invoker in + self.scalar_function_invokers] + from pyflink.table import Row + result = Row(*results) + # send the execution results back + self.output_processor.process_outputs(windowed_value, [result]) + + +class ScalarFunctionOperation(Operation): + """ + An operation that will execute ScalarFunctions for each input element. + """ + + def __init__(self, name, spec, counter_factory, sampler, consumers): + super(ScalarFunctionOperation, self).__init__(name, spec, counter_factory, sampler) + for tag, op_consumers in consumers.items(): + for consumer in op_consumers: + self.add_receiver(consumer, 0) + + self.scalar_function_runner = ScalarFunctionRunner(self.spec.serialized_fn) + self.scalar_function_runner.open() + + def setup(self): + with self.scoped_start_state: + super(ScalarFunctionOperation, self).setup() + self.scalar_function_runner.setup(self.receivers[0]) + + def start(self): + with self.scoped_start_state: + super(ScalarFunctionOperation, self).start() + + def process(self, o): + with self.scoped_process_state: + self.scalar_function_runner.process(o) + + def finish(self): + with self.scoped_finish_state: + super(ScalarFunctionOperation, self).finish() + + def needs_finalization(self): + return False + + def reset(self): + super(ScalarFunctionOperation, self).reset() + + def teardown(self): + with self.scoped_finish_state: + self.scalar_function_runner.close() + + def progress_metrics(self): + metrics = super(ScalarFunctionOperation, self).progress_metrics() + metrics.processed_elements.measured.output_element_counts.clear() + tag = None + receiver = self.receivers[0] + metrics.processed_elements.measured.output_element_counts[ + str(tag)] = receiver.opcounter.element_counter.value() + return metrics + + +@bundle_processor.BeamTransformFactory.register_urn( + SCALAR_FUNCTION_URN, flink_fn_execution_pb2.UserDefinedFunctions) +def create(factory, transform_id, transform_proto, parameter, consumers): + return _create_user_defined_function_operation( + factory, transform_proto, consumers, parameter.udfs) + + +def _create_user_defined_function_operation(factory, transform_proto, consumers, udfs_proto, + operation_cls=ScalarFunctionOperation): + output_tags = list(transform_proto.outputs.keys()) + output_coders = factory.get_output_coders(transform_proto) + spec = operation_specs.WorkerDoFn( + serialized_fn=udfs_proto, + output_tags=output_tags, + input=None, + side_inputs=None, + output_coders=[output_coders[tag] for tag in output_tags]) + + return operation_cls( + transform_proto.unique_name, + spec, + factory.counter_factory, + factory.state_sampler, + consumers) diff --git a/flink-python/pyflink/fn_execution/sdk_worker_main.py b/flink-python/pyflink/fn_execution/sdk_worker_main.py new file mode 100644 index 0000000000000000000000000000000000000000..82d091c52953b7cc15eda0f8cdef90ee322911b1 --- /dev/null +++ b/flink-python/pyflink/fn_execution/sdk_worker_main.py @@ -0,0 +1,30 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +import sys + +# force to register the operations to SDK Harness +import pyflink.fn_execution.operations # noqa # pylint: disable=unused-import + +# force to register the coders to SDK Harness +import pyflink.fn_execution.coders # noqa # pylint: disable=unused-import + +import apache_beam.runners.worker.sdk_worker_main + +if __name__ == '__main__': + apache_beam.runners.worker.sdk_worker_main.main(sys.argv) diff --git a/flink-python/pyflink/fn_execution/tests/test_flink_fn_execution_pb2_synced.py b/flink-python/pyflink/fn_execution/tests/test_flink_fn_execution_pb2_synced.py new file mode 100644 index 0000000000000000000000000000000000000000..544e4f3a705c49fe2fbbdfa7930665885504aa30 --- /dev/null +++ b/flink-python/pyflink/fn_execution/tests/test_flink_fn_execution_pb2_synced.py @@ -0,0 +1,43 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +import filecmp +import os + +from pyflink.gen_protos import generate_proto_files +from pyflink.testing.test_case_utils import PyFlinkTestCase + + +class FlinkFnExecutionSyncTests(PyFlinkTestCase): + """ + Tests whether flink_fn_exeution_pb2.py is synced with flink-fn-execution.proto. + """ + + flink_fn_execution_pb2_file_name = "flink_fn_execution_pb2.py" + gen_protos_script = "gen_protos.py" + flink_fn_execution_proto_file_name = "flink-fn-execution.proto" + + def test_flink_fn_execution_pb2_synced(self): + generate_proto_files('True', self.tempdir) + expected = os.path.join(self.tempdir, self.flink_fn_execution_pb2_file_name) + actual = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', + self.flink_fn_execution_pb2_file_name) + self.assertTrue(filecmp.cmp(expected, actual), + 'File %s should be re-generated by executing %s as %s has changed.' + % (self.flink_fn_execution_pb2_file_name, + self.gen_protos_script, + self.flink_fn_execution_proto_file_name)) diff --git a/flink-python/pyflink/gen_protos.py b/flink-python/pyflink/gen_protos.py new file mode 100644 index 0000000000000000000000000000000000000000..b5908029a013a59998592e34590157b66ce8dc90 --- /dev/null +++ b/flink-python/pyflink/gen_protos.py @@ -0,0 +1,194 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +from __future__ import absolute_import +from __future__ import print_function + +import glob +import logging +import multiprocessing +import os +import platform +import shutil +import subprocess +import sys +import time +import warnings + +import pkg_resources + +# latest grpcio-tools incompatible with latest protobuf 3.6.1. +GRPC_TOOLS = 'grpcio-tools>=1.3.5,<=1.14.2' +PROTO_PATHS = ['proto'] +PYFLINK_ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_PYTHON_OUTPUT_PATH = os.path.join(PYFLINK_ROOT_PATH, 'fn_execution') + + +def generate_proto_files(force=True, output_dir=DEFAULT_PYTHON_OUTPUT_PATH): + try: + import grpc_tools # noqa # pylint: disable=unused-import + except ImportError: + warnings.warn('Installing grpcio-tools is recommended for development.') + + proto_dirs = [os.path.join(PYFLINK_ROOT_PATH, path) for path in PROTO_PATHS] + proto_files = sum( + [glob.glob(os.path.join(d, '*.proto')) for d in proto_dirs], []) + out_dir = os.path.join(PYFLINK_ROOT_PATH, output_dir) + out_files = [path for path in glob.glob(os.path.join(out_dir, '*_pb2.py'))] + + if out_files and not proto_files and not force: + # We have out_files but no protos; assume they're up to date. + # This is actually the common case (e.g. installation from an sdist). + logging.info('No proto files; using existing generated files.') + return + + elif not out_files and not proto_files: + raise RuntimeError( + 'No proto files found in %s.' % proto_dirs) + + # Regenerate iff the proto files or this file are newer. + elif force or not out_files or len(out_files) < len(proto_files) or ( + min(os.path.getmtime(path) for path in out_files) + <= max(os.path.getmtime(path) + for path in proto_files + [os.path.realpath(__file__)])): + try: + from grpc_tools import protoc + except ImportError: + if platform.system() == 'Windows': + # For Windows, grpcio-tools has to be installed manually. + raise RuntimeError( + 'Cannot generate protos for Windows since grpcio-tools package is ' + 'not installed. Please install this package manually ' + 'using \'pip install grpcio-tools\'.') + + # Use a subprocess to avoid messing with this process' path and imports. + # Note that this requires a separate module from setup.py for Windows: + # https://docs.python.org/2/library/multiprocessing.html#windows + p = multiprocessing.Process( + target=_install_grpcio_tools_and_generate_proto_files(force, output_dir)) + p.start() + p.join() + if p.exitcode: + raise ValueError("Proto generation failed (see log for details).") + else: + _check_grpcio_tools_version() + logging.info('Regenerating out-of-date Python proto definitions.') + builtin_protos = pkg_resources.resource_filename('grpc_tools', '_proto') + args = ( + [sys.executable] + # expecting to be called from command line + ['--proto_path=%s' % builtin_protos] + + ['--proto_path=%s' % d for d in proto_dirs] + + ['--python_out=%s' % out_dir] + + proto_files) + ret_code = protoc.main(args) + if ret_code: + raise RuntimeError( + 'Protoc returned non-zero status (see logs for details): ' + '%s' % ret_code) + + for output_file in os.listdir(output_dir): + if output_file.endswith('_pb2.py'): + _add_license_header(output_dir, output_file) + + +# Though wheels are available for grpcio-tools, setup_requires uses +# easy_install which doesn't understand them. This means that it is +# compiled from scratch (which is expensive as it compiles the full +# protoc compiler). Instead, we attempt to install a wheel in a temporary +# directory and add it to the path as needed. +# See https://github.com/pypa/setuptools/issues/377 +def _install_grpcio_tools_and_generate_proto_files(force, output_dir): + install_path = os.path.join(PYFLINK_ROOT_PATH, '..', '.eggs', 'grpcio-wheels') + build_path = install_path + '-build' + if os.path.exists(build_path): + shutil.rmtree(build_path) + logging.warning('Installing grpcio-tools into %s', install_path) + try: + start = time.time() + # since '--prefix' option is only supported for pip 8.0+, so here we fallback to + # use '--install-option' when the pip version is lower than 8.0.0. + pip_version = pkg_resources.get_distribution("pip").version + from pkg_resources import parse_version + if parse_version(pip_version) >= parse_version('8.0.0'): + subprocess.check_call( + [sys.executable, '-m', 'pip', 'install', + '--prefix', install_path, '--build', build_path, + '--upgrade', GRPC_TOOLS, "-I"]) + else: + subprocess.check_call( + [sys.executable, '-m', 'pip', 'install', + '--install-option', '--prefix=' + install_path, '--build', build_path, + '--upgrade', GRPC_TOOLS, "-I"]) + from distutils.dist import Distribution + install_obj = Distribution().get_command_obj('install', create=True) + install_obj.prefix = install_path + install_obj.finalize_options() + logging.warning( + 'Installing grpcio-tools took %0.2f seconds.', time.time() - start) + finally: + sys.stderr.flush() + shutil.rmtree(build_path, ignore_errors=True) + sys.path.append(install_obj.install_purelib) + pkg_resources.working_set.add_entry(install_obj.install_purelib) + if install_obj.install_purelib != install_obj.install_platlib: + sys.path.append(install_obj.install_platlib) + pkg_resources.working_set.add_entry(install_obj.install_platlib) + try: + generate_proto_files(force, output_dir) + finally: + sys.stderr.flush() + + +def _add_license_header(dir, file_name): + with open(os.path.join(dir, file_name), 'r') as original_file: + original_data = original_file.read() + tmp_file_name = file_name + '.tmp' + with open(os.path.join(dir, tmp_file_name), 'w') as tmp_file: + tmp_file.write( + '################################################################################\n' + '# Licensed to the Apache Software Foundation (ASF) under one\n' + '# or more contributor license agreements. See the NOTICE file\n' + '# distributed with this work for additional information\n' + '# regarding copyright ownership. The ASF licenses this file\n' + '# to you under the Apache License, Version 2.0 (the\n' + '# "License"); you may not use this file except in compliance\n' + '# with the License. You may obtain a copy of the License at\n' + '#\n' + '# http://www.apache.org/licenses/LICENSE-2.0\n' + '#\n' + '# Unless required by applicable law or agreed to in writing, software\n' + '# distributed under the License is distributed on an "AS IS" BASIS,\n' + '# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' + '# See the License for the specific language governing permissions and\n' + '# limitations under the License.\n' + '################################################################################\n' + ) + tmp_file.write(original_data) + os.rename(os.path.join(dir, tmp_file_name), os.path.join(dir, file_name)) + + +def _check_grpcio_tools_version(): + version = pkg_resources.get_distribution("grpcio-tools").parsed_version + from pkg_resources import parse_version + if version < parse_version('1.3.5') or version > parse_version('1.14.2'): + raise RuntimeError( + "Version of grpcio-tools must be between 1.3.5 and 1.14.2, got %s" % version) + + +if __name__ == '__main__': + generate_proto_files() diff --git a/flink-python/pyflink/proto/flink-fn-execution.proto b/flink-python/pyflink/proto/flink-fn-execution.proto index 190a362d928ea1645d9c47b91d816aa00ec73fa5..4ce9ed3ceb4c25b9d8850b13fe7d23f5f6ebe6e6 100644 --- a/flink-python/pyflink/proto/flink-fn-execution.proto +++ b/flink-python/pyflink/proto/flink-fn-execution.proto @@ -16,6 +16,8 @@ * limitations under the License. */ +// NOTE: File flink_fn_execution_pb2.py is generated from this file. Please re-generate it by calling +// gen_protos.py whenever this file is changed. syntax = "proto3"; package org.apache.flink.fn_execution.v1; @@ -30,6 +32,7 @@ message UserDefinedFunction { oneof input { UserDefinedFunction udf = 1; int32 inputOffset = 2; + bytes inputConstant = 3; } } @@ -39,6 +42,7 @@ message UserDefinedFunction { // The input arguments of the user-defined function, it could be one of the following: // 1. A column from the input row // 2. The result of another user-defined function + // 3. The constant value of the column repeated Input inputs = 2; } diff --git a/flink-python/pyflink/shell.py b/flink-python/pyflink/shell.py index 87eb21d967cab4bf3963a478f7324aeca163c619..8fabd496e50a7a25a51e304a1bbe7cb92a6eaf33 100644 --- a/flink-python/pyflink/shell.py +++ b/flink-python/pyflink/shell.py @@ -40,16 +40,11 @@ print("Using Python version %s (%s, %s)" % ( welcome_msg = u''' \u2592\u2593\u2588\u2588\u2593\u2588\u2588\u2592 - \u2593\u2588\u2588\u2588\u2588\u2592\u2592\u2588\u2593 \ - \u2592\u2593\u2588\u2588\u2588\u2593\u2592 - \u2593\u2588\u2588\u2588\u2593\u2591\u2591 \u2592 \ - \u2592\u2592\u2593\u2588\u2588\u2592 \u2592 - \u2591\u2588\u2588\u2592 \u2592\u2592\u2593\u2593\u2588 \ - \u2593\u2593\u2592\u2591 \u2592\u2588\u2588\u2588\u2588 - \u2588\u2588\u2592 \u2591\u2592\u2593\u2588\u2588 \ - \u2588\u2592 \u2592\u2588\u2592\u2588\u2592 - \u2591\u2593\u2588 \u2588\u2588\u2588 \u2593 \ - \u2591\u2592\u2588\u2588 + \u2593\u2588\u2588\u2588\u2588\u2592\u2592\u2588\u2593\u2592\u2593\u2588\u2588\u2588\u2593\u2592 + \u2593\u2588\u2588\u2588\u2593\u2591\u2591 \u2592\u2592\u2592\u2593\u2588\u2588\u2592 \u2592 + \u2591\u2588\u2588\u2592 \u2592\u2592\u2593\u2593\u2588\u2593\u2593\u2592\u2591 \u2592\u2588\u2588\u2588\u2588 + \u2588\u2588\u2592 \u2591\u2592\u2593\u2588\u2588\u2588\u2592 \u2592\u2588\u2592\u2588\u2592 + \u2591\u2593\u2588 \u2588\u2588\u2588 \u2593\u2591\u2592\u2588\u2588 \u2593\u2588 \u2592\u2592\u2592\u2592\u2592\u2593\u2588\u2588\u2593\u2591\u2592\u2591\u2593\u2593\u2588 \u2588\u2591 \u2588 \u2592\u2592\u2591 \u2588\u2588\u2588\u2593\u2593\u2588 \u2592\u2588\u2592\u2592\u2592 \u2588\u2588\u2588\u2588\u2591 \u2592\u2593\u2588\u2593 \u2588\u2588\u2592\u2592\u2592 \u2593\u2588\u2588\u2588\u2592 diff --git a/flink-python/pyflink/table/__init__.py b/flink-python/pyflink/table/__init__.py index e69a9b7ecb0cd8f182dd9634ef1ae1a6218ec56a..82ce28a34063c6d859a2c8b2105b19cb5102c933 100644 --- a/flink-python/pyflink/table/__init__.py +++ b/flink-python/pyflink/table/__init__.py @@ -52,6 +52,11 @@ Important classes of Flink Table API: from a registered :class:`pyflink.table.catalog.Catalog`. - :class:`pyflink.table.TableSchema` Represents a table's structure with field names and data types. + - :class:`pyflink.table.FunctionContext` + Used to obtain global runtime information about the context in which the + user-defined function is executed, such as the metric group, and global job parameters, etc. + - :class:`pyflink.table.ScalarFunction` + Base interface for user-defined scalar function. """ from __future__ import absolute_import @@ -65,6 +70,7 @@ from pyflink.table.sinks import TableSink, CsvTableSink, WriteMode from pyflink.table.sources import TableSource, CsvTableSource from pyflink.table.types import DataTypes, UserDefinedType, Row from pyflink.table.table_schema import TableSchema +from pyflink.table.udf import FunctionContext, ScalarFunction __all__ = [ 'TableEnvironment', @@ -85,5 +91,7 @@ __all__ = [ 'DataTypes', 'UserDefinedType', 'Row', - 'TableSchema' + 'TableSchema', + 'FunctionContext', + 'ScalarFunction' ] diff --git a/flink-python/pyflink/table/table_environment.py b/flink-python/pyflink/table/table_environment.py index 33160e0cdbef5d542d84affc6738686d141671ef..940aafcf62b12d58f99be3a590239d0795da3c30 100644 --- a/flink-python/pyflink/table/table_environment.py +++ b/flink-python/pyflink/table/table_environment.py @@ -542,6 +542,36 @@ class TableEnvironment(object): .loadClass(function_class_name).newInstance() self._j_tenv.registerFunction(name, java_function) + def register_function(self, name, function): + """ + Registers a python user-defined function under a unique name. Replaces already existing + user-defined function under this name. + + Example: + :: + + >>> table_env.register_function( + ... "add_one", udf(lambda i: i + 1, DataTypes.BIGINT(), DataTypes.BIGINT())) + + >>> @udf(input_types=[DataTypes.BIGINT(), DataTypes.BIGINT()], + ... result_type=DataTypes.BIGINT()) + ... def add(i, j): + ... return i + j + >>> table_env.register_function("add", add) + + >>> class SubtractOne(ScalarFunction): + ... def eval(self, i): + ... return i - 1 + >>> table_env.register_function( + ... "subtract_one", udf(SubtractOne(), DataTypes.BIGINT(), DataTypes.BIGINT())) + + :param name: The name under which the function is registered. + :type name: str + :param function: The python user-defined function to register. + :type function: UserDefinedFunctionWrapper + """ + self._j_tenv.registerFunction(name, function._judf) + def execute(self, job_name): """ Triggers the program execution. The environment will execute all parts of diff --git a/flink-python/pyflink/table/tests/test_environment_completeness.py b/flink-python/pyflink/table/tests/test_environment_completeness.py index 5f73c3c88b5670426b0a95ad43980521a6353039..89459891f8f3e5e9f38ee393eb8ed4d359396b33 100644 --- a/flink-python/pyflink/table/tests/test_environment_completeness.py +++ b/flink-python/pyflink/table/tests/test_environment_completeness.py @@ -41,8 +41,7 @@ class EnvironmentAPICompletenessTests(PythonAPICompletenessTestCase, unittest.Te # registerCatalog, getCatalog and listTables should be supported when catalog supported in # python. getCompletionHints has been deprecated. It will be removed in the next release. # TODO add TableEnvironment#create method with EnvironmentSettings as a parameter - return {'registerCatalog', 'getCatalog', 'registerFunction', 'listTables', - 'getCompletionHints', 'create'} + return {'registerCatalog', 'getCatalog', 'listTables', 'getCompletionHints', 'create'} if __name__ == '__main__': diff --git a/flink-python/pyflink/table/tests/test_udf.py b/flink-python/pyflink/table/tests/test_udf.py new file mode 100644 index 0000000000000000000000000000000000000000..321cd784f2874fd7b3fc2cadb2700989a2d8b717 --- /dev/null +++ b/flink-python/pyflink/table/tests/test_udf.py @@ -0,0 +1,367 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +from pyflink.table import DataTypes +from pyflink.table.udf import ScalarFunction, udf +from pyflink.testing import source_sink_utils +from pyflink.testing.test_case_utils import PyFlinkStreamTableTestCase + + +class UserDefinedFunctionTests(PyFlinkStreamTableTestCase): + + def test_scalar_function(self): + # test lambda function + self.t_env.register_function( + "add_one", udf(lambda i: i + 1, DataTypes.BIGINT(), DataTypes.BIGINT())) + + # test Python ScalarFunction + self.t_env.register_function( + "subtract_one", udf(SubtractOne(), DataTypes.BIGINT(), DataTypes.BIGINT())) + + # test Python function + self.t_env.register_function("add", add) + + # test callable function + self.t_env.register_function( + "add_one_callable", udf(CallablePlus(), DataTypes.BIGINT(), DataTypes.BIGINT())) + + def partial_func(col, param): + return col + param + + # test partial function + import functools + self.t_env.register_function( + "add_one_partial", + udf(functools.partial(partial_func, param=1), DataTypes.BIGINT(), DataTypes.BIGINT())) + + table_sink = source_sink_utils.TestAppendSink( + ['a', 'b', 'c', 'd', 'e'], + [DataTypes.BIGINT(), DataTypes.BIGINT(), DataTypes.BIGINT(), DataTypes.BIGINT(), + DataTypes.BIGINT()]) + self.t_env.register_table_sink("Results", table_sink) + + t = self.t_env.from_elements([(1, 2, 3), (2, 5, 6), (3, 1, 9)], ['a', 'b', 'c']) + t.where("add_one(b) <= 3") \ + .select("add_one(a), subtract_one(b), add(a, c), add_one_callable(a), " + "add_one_partial(a)") \ + .insert_into("Results") + self.t_env.execute("test") + actual = source_sink_utils.results() + self.assert_equals(actual, ["2,1,4,2,2", "4,0,12,4,4"]) + + def test_chaining_scalar_function(self): + self.t_env.register_function( + "add_one", udf(lambda i: i + 1, DataTypes.BIGINT(), DataTypes.BIGINT())) + self.t_env.register_function( + "subtract_one", udf(SubtractOne(), DataTypes.BIGINT(), DataTypes.BIGINT())) + self.t_env.register_function("add", add) + + table_sink = source_sink_utils.TestAppendSink(['a'], [DataTypes.BIGINT()]) + self.t_env.register_table_sink("Results", table_sink) + + t = self.t_env.from_elements([(1, 2), (2, 5), (3, 1)], ['a', 'b']) + t.select("add(add_one(a), subtract_one(b))") \ + .insert_into("Results") + self.t_env.execute("test") + actual = source_sink_utils.results() + self.assert_equals(actual, ["3", "7", "4"]) + + def test_udf_in_join_condition(self): + t1 = self.t_env.from_elements([(2, "Hi")], ['a', 'b']) + t2 = self.t_env.from_elements([(2, "Flink")], ['c', 'd']) + + self.t_env.register_function("f", udf(lambda i: i, DataTypes.BIGINT(), DataTypes.BIGINT())) + + table_sink = source_sink_utils.TestAppendSink( + ['a', 'b', 'c', 'd'], + [DataTypes.BIGINT(), DataTypes.STRING(), DataTypes.BIGINT(), DataTypes.STRING()]) + self.t_env.register_table_sink("Results", table_sink) + + t1.join(t2).where("f(a) = c").insert_into("Results") + self.t_env.execute("test") + actual = source_sink_utils.results() + self.assert_equals(actual, ["2,Hi,2,Flink"]) + + def test_udf_in_join_condition_2(self): + t1 = self.t_env.from_elements([(1, "Hi"), (2, "Hi")], ['a', 'b']) + t2 = self.t_env.from_elements([(2, "Flink")], ['c', 'd']) + + self.t_env.register_function("f", udf(lambda i: i, DataTypes.BIGINT(), DataTypes.BIGINT())) + + table_sink = source_sink_utils.TestAppendSink( + ['a', 'b', 'c', 'd'], + [DataTypes.BIGINT(), DataTypes.STRING(), DataTypes.BIGINT(), DataTypes.STRING()]) + self.t_env.register_table_sink("Results", table_sink) + + t1.join(t2).where("f(a) = f(c)").insert_into("Results") + self.t_env.execute("test") + actual = source_sink_utils.results() + self.assert_equals(actual, ["2,Hi,2,Flink"]) + + def test_udf_with_constant_params(self): + def udf_with_constant_params(p, null_param, tinyint_param, smallint_param, int_param, + bigint_param, decimal_param, float_param, double_param, + boolean_param, str_param, + date_param, time_param, timestamp_param): + # decide whether two floats are equal + def float_equal(a, b, rel_tol=1e-09, abs_tol=0.0): + return abs(a - b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol) + + from decimal import Decimal + import datetime + + assert null_param is None, 'null_param is wrong value %s' % null_param + + assert isinstance(tinyint_param, int), 'tinyint_param of wrong type %s !' \ + % type(tinyint_param) + p += tinyint_param + assert isinstance(smallint_param, int), 'smallint_param of wrong type %s !' \ + % type(smallint_param) + p += smallint_param + assert isinstance(int_param, int), 'int_param of wrong type %s !' \ + % type(int_param) + p += int_param + assert isinstance(bigint_param, int), 'bigint_param of wrong type %s !' \ + % type(bigint_param) + p += bigint_param + assert decimal_param == Decimal('1.05'), \ + 'decimal_param is wrong value %s ' % decimal_param + + p += int(decimal_param) + + assert isinstance(float_param, float) and float_equal(float_param, 1.23, 1e-06), \ + 'float_param is wrong value %s ' % float_param + + p += int(float_param) + assert isinstance(double_param, float) and float_equal(double_param, 1.98932, 1e-07), \ + 'double_param is wrong value %s ' % double_param + + p += int(double_param) + + assert boolean_param is True, 'boolean_param is wrong value %s' % boolean_param + + assert str_param == 'flink', 'str_param is wrong value %s' % str_param + + assert date_param == datetime.date(year=2014, month=9, day=13), \ + 'date_param is wrong value %s' % date_param + + assert time_param == datetime.time(hour=12, minute=0, second=0), \ + 'time_param is wrong value %s' % time_param + + assert timestamp_param == datetime.datetime(1999, 9, 10, 5, 20, 10), \ + 'timestamp_param is wrong value %s' % timestamp_param + + return p + + self.t_env.register_function("udf_with_constant_params", + udf(udf_with_constant_params, + input_types=[DataTypes.BIGINT(), + DataTypes.BIGINT(), + DataTypes.TINYINT(), + DataTypes.SMALLINT(), + DataTypes.INT(), + DataTypes.BIGINT(), + DataTypes.DECIMAL(20, 10), + DataTypes.FLOAT(), + DataTypes.DOUBLE(), + DataTypes.BOOLEAN(), + DataTypes.STRING(), + DataTypes.DATE(), + DataTypes.TIME(), + DataTypes.TIMESTAMP()], + result_type=DataTypes.BIGINT())) + + self.t_env.register_function( + "udf_with_all_constant_params", udf(lambda i, j: i + j, + [DataTypes.BIGINT(), DataTypes.BIGINT()], + DataTypes.BIGINT())) + + table_sink = source_sink_utils.TestAppendSink(['a', 'b'], + [DataTypes.BIGINT(), DataTypes.BIGINT()]) + self.t_env.register_table_sink("Results", table_sink) + + t = self.t_env.from_elements([(1, 2, 3), (2, 5, 6), (3, 1, 9)], ['a', 'b', 'c']) + self.t_env.register_table("test_table", t) + self.t_env.sql_query("select udf_with_all_constant_params(" + "cast (1 as BIGINT)," + "cast (2 as BIGINT)), " + "udf_with_constant_params(a, " + "cast (null as BIGINT)," + "cast (1 as TINYINT)," + "cast (1 as SMALLINT)," + "cast (1 as INT)," + "cast (1 as BIGINT)," + "cast (1.05 as DECIMAL)," + "cast (1.23 as FLOAT)," + "cast (1.98932 as DOUBLE)," + "true," + "'flink'," + "cast ('2014-09-13' as DATE)," + "cast ('12:00:00' as TIME)," + "cast ('1999-9-10 05:20:10' as TIMESTAMP))" + " from test_table").insert_into("Results") + self.t_env.execute("test") + actual = source_sink_utils.results() + self.assert_equals(actual, ["3,8", "3,9", "3,10"]) + + def test_overwrite_builtin_function(self): + self.t_env.register_function( + "plus", udf(lambda i, j: i + j - 1, + [DataTypes.BIGINT(), DataTypes.BIGINT()], DataTypes.BIGINT())) + + table_sink = source_sink_utils.TestAppendSink(['a'], [DataTypes.BIGINT()]) + self.t_env.register_table_sink("Results", table_sink) + + t = self.t_env.from_elements([(1, 2, 3), (2, 5, 6), (3, 1, 9)], ['a', 'b', 'c']) + t.select("plus(a, b)").insert_into("Results") + self.t_env.execute("test") + actual = source_sink_utils.results() + self.assert_equals(actual, ["2", "6", "3"]) + + def test_open(self): + self.t_env.register_function( + "subtract", udf(Subtract(), DataTypes.BIGINT(), DataTypes.BIGINT())) + table_sink = source_sink_utils.TestAppendSink( + ['a', 'b'], [DataTypes.BIGINT(), DataTypes.BIGINT()]) + self.t_env.register_table_sink("Results", table_sink) + + t = self.t_env.from_elements([(1, 2), (2, 5), (3, 4)], ['a', 'b']) + t.select("a, subtract(b)").insert_into("Results") + self.t_env.execute("test") + actual = source_sink_utils.results() + self.assert_equals(actual, ["1,1", "2,4", "3,3"]) + + def test_deterministic(self): + add_one = udf(lambda i: i + 1, DataTypes.BIGINT(), DataTypes.BIGINT()) + self.assertTrue(add_one._deterministic) + + add_one = udf(lambda i: i + 1, DataTypes.BIGINT(), DataTypes.BIGINT(), deterministic=False) + self.assertFalse(add_one._deterministic) + + subtract_one = udf(SubtractOne(), DataTypes.BIGINT(), DataTypes.BIGINT()) + self.assertTrue(subtract_one._deterministic) + + with self.assertRaises(ValueError, msg="Inconsistent deterministic: False and True"): + udf(SubtractOne(), DataTypes.BIGINT(), DataTypes.BIGINT(), deterministic=False) + + self.assertTrue(add._deterministic) + + @udf(input_types=DataTypes.BIGINT(), result_type=DataTypes.BIGINT(), deterministic=False) + def non_deterministic_udf(i): + return i + + self.assertFalse(non_deterministic_udf._deterministic) + + def test_name(self): + add_one = udf(lambda i: i + 1, DataTypes.BIGINT(), DataTypes.BIGINT()) + self.assertEqual("", add_one._name) + + add_one = udf(lambda i: i + 1, DataTypes.BIGINT(), DataTypes.BIGINT(), name="add_one") + self.assertEqual("add_one", add_one._name) + + subtract_one = udf(SubtractOne(), DataTypes.BIGINT(), DataTypes.BIGINT()) + self.assertEqual("SubtractOne", subtract_one._name) + + subtract_one = udf(SubtractOne(), DataTypes.BIGINT(), DataTypes.BIGINT(), + name="subtract_one") + self.assertEqual("subtract_one", subtract_one._name) + + self.assertEqual("add", add._name) + + @udf(input_types=DataTypes.BIGINT(), result_type=DataTypes.BIGINT(), name="named") + def named_udf(i): + return i + + self.assertEqual("named", named_udf._name) + + def test_abc(self): + class UdfWithoutEval(ScalarFunction): + def open(self, function_context): + pass + + with self.assertRaises( + TypeError, + msg="Can't instantiate abstract class UdfWithoutEval with abstract methods eval"): + UdfWithoutEval() + + def test_invalid_udf(self): + class Plus(object): + def eval(self, col): + return col + 1 + + with self.assertRaises( + TypeError, + msg="Invalid function: not a function or callable (__call__ is not defined)"): + # test non-callable function + self.t_env.register_function( + "non-callable-udf", udf(Plus(), DataTypes.BIGINT(), DataTypes.BIGINT())) + + def test_udf_without_arguments(self): + self.t_env.register_function("one", udf( + lambda: 1, input_types=[], result_type=DataTypes.BIGINT(), deterministic=True)) + self.t_env.register_function("two", udf( + lambda: 2, input_types=[], result_type=DataTypes.BIGINT(), deterministic=False)) + + table_sink = source_sink_utils.TestAppendSink(['a', 'b'], + [DataTypes.BIGINT(), DataTypes.BIGINT()]) + self.t_env.register_table_sink("Results", table_sink) + + t = self.t_env.from_elements([(1, 2), (2, 5), (3, 1)], ['a', 'b']) + t.select("one(), two()").insert_into("Results") + self.t_env.execute("test") + actual = source_sink_utils.results() + self.assert_equals(actual, ["1,2", "1,2", "1,2"]) + + +@udf(input_types=[DataTypes.BIGINT(), DataTypes.BIGINT()], result_type=DataTypes.BIGINT()) +def add(i, j): + return i + j + + +class SubtractOne(ScalarFunction): + + def eval(self, i): + return i - 1 + + +class Subtract(ScalarFunction): + + def __init__(self): + self.subtracted_value = 0 + + def open(self, function_context): + self.subtracted_value = 1 + + def eval(self, i): + return i - self.subtracted_value + + +class CallablePlus(object): + + def __call__(self, col): + return col + 1 + + +if __name__ == '__main__': + import unittest + + try: + import xmlrunner + testRunner = xmlrunner.XMLTestRunner(output='target/test-reports') + except ImportError: + testRunner = None + unittest.main(testRunner=testRunner, verbosity=2) diff --git a/flink-python/pyflink/table/types.py b/flink-python/pyflink/table/types.py index dbf92f88bb8c25defd6d909f6427df9f40d90d43..09186240a62904cbe4d4dfcff172d0bd861fa411 100644 --- a/flink-python/pyflink/table/types.py +++ b/flink-python/pyflink/table/types.py @@ -2005,7 +2005,7 @@ class Row(tuple): return "Row(%s)" % ", ".join("%s=%r" % (k, v) for k, v in zip(self._fields, tuple(self))) else: - return "" % ", ".join(self) + return "" % ", ".join("%r" % field for field in self) _acceptable_types = { diff --git a/flink-python/pyflink/table/udf.py b/flink-python/pyflink/table/udf.py new file mode 100644 index 0000000000000000000000000000000000000000..97ac3910708e687db5b69fe1267908f8c41bd884 --- /dev/null +++ b/flink-python/pyflink/table/udf.py @@ -0,0 +1,229 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ +import abc +import collections +import functools +import inspect +import sys + +from pyflink.java_gateway import get_gateway +from pyflink.table.types import DataType, _to_java_type +from pyflink.util import utils + +__all__ = ['FunctionContext', 'ScalarFunction', 'udf'] + + +if sys.version_info >= (3, 4): + ABC = abc.ABC +else: + ABC = abc.ABCMeta('ABC', (), {}) + + +class FunctionContext(object): + """ + Used to obtain global runtime information about the context in which the + user-defined function is executed. The information includes the metric group, + and global job parameters, etc. + """ + pass + + +class UserDefinedFunction(ABC): + """ + Base interface for user-defined function. + """ + + def open(self, function_context): + """ + Initialization method for the function. It is called before the actual working methods + and thus suitable for one time setup work. + + :param function_context: the context of the function + :type function_context: FunctionContext + """ + pass + + def close(self): + """ + Tear-down method for the user code. It is called after the last call to the main + working methods. + """ + pass + + def is_deterministic(self): + """ + Returns information about the determinism of the function's results. + It returns true if and only if a call to this function is guaranteed to + always return the same result given the same parameters. true is assumed by default. + If the function is not pure functional like random(), date(), now(), + this method must return false. + + :return: the determinism of the function's results. + :rtype: bool + """ + return True + + +class ScalarFunction(UserDefinedFunction): + """ + Base interface for user-defined scalar function. A user-defined scalar functions maps zero, one, + or multiple scalar values to a new scalar value. + """ + + @abc.abstractmethod + def eval(self, *args): + """ + Method which defines the logic of the scalar function. + """ + pass + + +class DelegatingScalarFunction(ScalarFunction): + """ + Helper scalar function implementation for lambda expression and python function. It's for + internal use only. + """ + + def __init__(self, func): + self.func = func + + def eval(self, *args): + return self.func(*args) + + +class UserDefinedFunctionWrapper(object): + """ + Wrapper for Python user-defined function. It handles things like converting lambda + functions to user-defined functions, creating the Java user-defined function representation, + etc. It's for internal use only. + """ + + def __init__(self, func, input_types, result_type, deterministic=None, name=None): + if inspect.isclass(func) or ( + not isinstance(func, UserDefinedFunction) and not callable(func)): + raise TypeError( + "Invalid function: not a function or callable (__call__ is not defined): {0}" + .format(type(func))) + + if not isinstance(input_types, collections.Iterable): + input_types = [input_types] + + for input_type in input_types: + if not isinstance(input_type, DataType): + raise TypeError( + "Invalid input_type: input_type should be DataType but contains {}".format( + input_type)) + + if not isinstance(result_type, DataType): + raise TypeError( + "Invalid returnType: returnType should be DataType but is {}".format(result_type)) + + self._func = func + self._input_types = input_types + self._result_type = result_type + self._judf_placeholder = None + self._name = name or ( + func.__name__ if hasattr(func, '__name__') else func.__class__.__name__) + + if deterministic is not None and isinstance(func, UserDefinedFunction) and deterministic \ + != func.is_deterministic(): + raise ValueError("Inconsistent deterministic: {} and {}".format( + deterministic, func.is_deterministic())) + + # default deterministic is True + self._deterministic = deterministic if deterministic is not None else ( + func.is_deterministic() if isinstance(func, UserDefinedFunction) else True) + + @property + def _judf(self): + if self._judf_placeholder is None: + self._judf_placeholder = self._create_judf() + return self._judf_placeholder + + def _create_judf(self): + func = self._func + if not isinstance(self._func, UserDefinedFunction): + func = DelegatingScalarFunction(self._func) + + import cloudpickle + serialized_func = cloudpickle.dumps(func) + + gateway = get_gateway() + j_input_types = utils.to_jarray(gateway.jvm.TypeInformation, + [_to_java_type(i) for i in self._input_types]) + j_result_type = _to_java_type(self._result_type) + return gateway.jvm.org.apache.flink.table.util.python.PythonTableUtils \ + .createPythonScalarFunction(self._name, + bytearray(serialized_func), + j_input_types, + j_result_type, + self._deterministic, + _get_python_env()) + + +# TODO: support to configure the python execution environment +def _get_python_env(): + gateway = get_gateway() + exec_type = gateway.jvm.org.apache.flink.table.functions.python.PythonEnv.ExecType.PROCESS + return gateway.jvm.org.apache.flink.table.functions.python.PythonEnv(exec_type) + + +def _create_udf(f, input_types, result_type, deterministic, name): + return UserDefinedFunctionWrapper(f, input_types, result_type, deterministic, name) + + +def udf(f=None, input_types=None, result_type=None, deterministic=None, name=None): + """ + Helper method for creating a user-defined function. + + Example: + :: + + >>> add_one = udf(lambda i: i + 1, DataTypes.BIGINT(), DataTypes.BIGINT()) + + >>> @udf(input_types=[DataTypes.BIGINT(), DataTypes.BIGINT()], + ... result_type=DataTypes.BIGINT()) + ... def add(i, j): + ... return i + j + + >>> class SubtractOne(ScalarFunction): + ... def eval(self, i): + ... return i - 1 + >>> subtract_one = udf(SubtractOne(), DataTypes.BIGINT(), DataTypes.BIGINT()) + + :param f: lambda function or user-defined function. + :type f: function or UserDefinedFunction or type + :param input_types: the input data types. + :type input_types: list[DataType] or DataType + :param result_type: the result data type. + :type result_type: DataType + :param name: the function name. + :type name: str + :param deterministic: the determinism of the function's results. True if and only if a call to + this function is guaranteed to always return the same result given the + same parameters. (default True) + :type deterministic: bool + :return: UserDefinedFunctionWrapper or function. + :rtype: UserDefinedFunctionWrapper or function + """ + # decorator + if f is None: + return functools.partial(_create_udf, input_types=input_types, result_type=result_type, + deterministic=deterministic, name=name) + else: + return _create_udf(f, input_types, result_type, deterministic, name) diff --git a/flink-python/pyflink/testing/test_case_utils.py b/flink-python/pyflink/testing/test_case_utils.py index c1d484ed01c041bdf09fd924377357568c8d9ffc..888824ab7cae8765a5bbc4ac3e8f6d67e090c4f9 100644 --- a/flink-python/pyflink/testing/test_case_utils.py +++ b/flink-python/pyflink/testing/test_case_utils.py @@ -28,10 +28,8 @@ from py4j.java_gateway import JavaObject from py4j.protocol import Py4JJavaError from pyflink.table.sources import CsvTableSource - from pyflink.dataset import ExecutionEnvironment from pyflink.datastream import StreamExecutionEnvironment - from pyflink.find_flink_home import _find_flink_home from pyflink.table import BatchTableEnvironment, StreamTableEnvironment from pyflink.java_gateway import get_gateway diff --git a/flink-python/setup.py b/flink-python/setup.py index 78a6edd6beb3207d9d612e84a6fa1fb4724154e8..11c1c1909e199e694663b65e35cd7bc8979e8ee1 100644 --- a/flink-python/setup.py +++ b/flink-python/setup.py @@ -184,7 +184,8 @@ run sdist. license='https://www.apache.org/licenses/LICENSE-2.0', author='Flink Developers', author_email='dev@flink.apache.org', - install_requires=['py4j==0.10.8.1', 'python-dateutil', 'apache-beam==2.15.0'], + install_requires=['py4j==0.10.8.1', 'python-dateutil==2.8.0', 'apache-beam==2.15.0', + 'cloudpickle==1.2.2'], tests_require=['pytest==4.4.1'], description='Apache Flink Python API', long_description=long_description, diff --git a/flink-python/src/main/java/org/apache/flink/api/common/python/PythonBridgeUtils.java b/flink-python/src/main/java/org/apache/flink/api/common/python/PythonBridgeUtils.java index 44a568b2f53eafea9715394cfa966b7ce6e3809c..31c18f22432abbeff24160c1f1c6b6da82fca15f 100644 --- a/flink-python/src/main/java/org/apache/flink/api/common/python/PythonBridgeUtils.java +++ b/flink-python/src/main/java/org/apache/flink/api/common/python/PythonBridgeUtils.java @@ -20,12 +20,16 @@ package org.apache.flink.api.common.python; import org.apache.flink.api.common.python.pickle.ArrayConstructor; import org.apache.flink.api.common.python.pickle.ByteArrayConstructor; +import net.razorvine.pickle.Pickler; import net.razorvine.pickle.Unpickler; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.sql.type.SqlTypeName; import java.io.DataInputStream; import java.io.EOFException; import java.io.FileInputStream; import java.io.IOException; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.LinkedList; import java.util.List; @@ -70,6 +74,77 @@ public final class PythonBridgeUtils { return unpickledData; } + public static byte[] convertLiteralToPython(RexLiteral o, SqlTypeName typeName) { + byte type; + Object value; + Pickler pickler = new Pickler(); + if (o.getValue3() == null) { + type = 0; + value = null; + } else { + switch (typeName) { + case TINYINT: + type = 0; + value = ((BigDecimal) o.getValue3()).byteValueExact(); + break; + case SMALLINT: + type = 0; + value = ((BigDecimal) o.getValue3()).shortValueExact(); + break; + case INTEGER: + type = 0; + value = ((BigDecimal) o.getValue3()).intValueExact(); + break; + case BIGINT: + type = 0; + value = ((BigDecimal) o.getValue3()).longValueExact(); + break; + case FLOAT: + type = 0; + value = ((BigDecimal) o.getValue3()).floatValue(); + break; + case DOUBLE: + type = 0; + value = ((BigDecimal) o.getValue3()).doubleValue(); + break; + case DECIMAL: + case BOOLEAN: + type = 0; + value = o.getValue3(); + break; + case CHAR: + case VARCHAR: + type = 0; + value = o.getValue3().toString(); + break; + case DATE: + type = 1; + value = o.getValue3(); + break; + case TIME: + type = 2; + value = o.getValue3(); + break; + case TIMESTAMP: + type = 3; + value = o.getValue3(); + break; + default: + throw new RuntimeException("Unsupported type " + typeName); + } + } + byte[] pickledData; + try { + pickledData = pickler.dumps(value); + } catch (IOException e) { + throw new RuntimeException("Pickle Java object failed", e); + } + byte[] typePickledData = new byte[pickledData.length + 1]; + typePickledData[0] = type; + System.arraycopy(pickledData, 0, typePickledData, 1, pickledData.length); + return typePickledData; + } + private static List readPickledBytes(final String fileName) throws IOException { List objs = new LinkedList<>(); try (DataInputStream din = new DataInputStream(new FileInputStream(fileName))) { diff --git a/flink-python/src/main/java/org/apache/flink/python/AbstractPythonFunctionRunner.java b/flink-python/src/main/java/org/apache/flink/python/AbstractPythonFunctionRunner.java index 498e090d1798b8334acee339887913631379d83f..53e6ef94fb16744113a454d9d953ec6af02c1356 100644 --- a/flink-python/src/main/java/org/apache/flink/python/AbstractPythonFunctionRunner.java +++ b/flink-python/src/main/java/org/apache/flink/python/AbstractPythonFunctionRunner.java @@ -20,6 +20,7 @@ package org.apache.flink.python; import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.core.memory.ByteArrayInputStreamWithPos; import org.apache.flink.core.memory.ByteArrayOutputStreamWithPos; import org.apache.flink.table.functions.python.PythonEnv; @@ -49,8 +50,7 @@ import java.io.DataOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; +import java.util.Arrays; import java.util.Random; /** @@ -92,9 +92,9 @@ public abstract class AbstractPythonFunctionRunner implements PythonFun private final StateRequestHandler stateRequestHandler; /** - * Temporary directory to store the retrieval token. + * Temporary directories to store the retrieval token. */ - private final String tempDir; + private final String[] tempDirs; /** * The file of the retrieval token representing the entirety of the staged artifacts. @@ -146,12 +146,12 @@ public abstract class AbstractPythonFunctionRunner implements PythonFun FnDataReceiver resultReceiver, PythonEnv pythonEnv, StateRequestHandler stateRequestHandler, - String tempDir) { + String[] tempDirs) { this.taskName = Preconditions.checkNotNull(taskName); this.resultReceiver = Preconditions.checkNotNull(resultReceiver); this.pythonEnv = Preconditions.checkNotNull(pythonEnv); this.stateRequestHandler = Preconditions.checkNotNull(stateRequestHandler); - this.tempDir = Preconditions.checkNotNull(tempDir); + this.tempDirs = Preconditions.checkNotNull(tempDirs); } @Override @@ -249,23 +249,22 @@ public abstract class AbstractPythonFunctionRunner implements PythonFun } private String createEmptyRetrievalToken() throws Exception { - // try to find a unique file name for the spilling channel + // try to find a unique file name for the retrieval token final Random rnd = new Random(); for (int attempt = 0; attempt < 10; attempt++) { - retrievalToken = new File(tempDir, randomString(rnd) + ".json"); - if (!retrievalToken.exists()) { - break; + String directory = tempDirs[rnd.nextInt(tempDirs.length)]; + retrievalToken = new File(directory, randomString(rnd) + ".json"); + if (retrievalToken.createNewFile()) { + final DataOutputStream dos = new DataOutputStream(new FileOutputStream(retrievalToken)); + dos.writeBytes("{\"manifest\": {}}"); + dos.flush(); + dos.close(); + return retrievalToken.getAbsolutePath(); } } - if (retrievalToken.exists()) { - throw new IOException( - "Could not find a unique file name in '" + tempDir + "' for retrieval token."); - } - final DataOutputStream dos = new DataOutputStream(new FileOutputStream(retrievalToken)); - dos.writeBytes("{\"manifest\": {}}"); - dos.flush(); - dos.close(); - return retrievalToken.getAbsolutePath(); + + throw new IOException( + "Could not find a unique file name in '" + Arrays.toString(tempDirs) + "' for retrieval token."); } private static String randomString(Random random) { @@ -280,14 +279,15 @@ public abstract class AbstractPythonFunctionRunner implements PythonFun */ protected RunnerApi.Environment createPythonExecutionEnvironment() { if (pythonEnv.getExecType() == PythonEnv.ExecType.PROCESS) { - final Map env = new HashMap<>(2); - env.put("python", pythonEnv.getPythonExec()); + String flinkHomePath = System.getenv(ConfigConstants.ENV_FLINK_HOME_DIR); + String pythonWorkerCommand = + flinkHomePath + File.separator + "bin" + File.separator + "pyflink-udf-runner.sh"; return Environments.createProcessEnvironment( "", "", - pythonEnv.getPythonWorkerCmd(), - env); + pythonWorkerCommand, + null); } else { throw new UnsupportedOperationException(String.format( "Execution type '%s' is not supported.", pythonEnv.getExecType())); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/CheckpointExceptionHandler.java b/flink-python/src/main/java/org/apache/flink/python/PythonOptions.java similarity index 44% rename from flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/CheckpointExceptionHandler.java rename to flink-python/src/main/java/org/apache/flink/python/PythonOptions.java index 01407952550fa8b498190462a45a59ddaf456139..6c0ee52d3e1e5fd7a06d610b1bad2e7c4284e3b6 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/CheckpointExceptionHandler.java +++ b/flink-python/src/main/java/org/apache/flink/python/PythonOptions.java @@ -16,21 +16,33 @@ * limitations under the License. */ -package org.apache.flink.streaming.runtime.tasks; +package org.apache.flink.python; -import org.apache.flink.runtime.checkpoint.CheckpointMetaData; +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; /** - * Handler for exceptions that happen on checkpointing. The handler can reject and rethrow the offered exceptions. + * Configuration options for the Python API. */ -public interface CheckpointExceptionHandler { +@PublicEvolving +public class PythonOptions { /** - * Offers the exception for handling. If the exception cannot be handled from this instance, it is rethrown. - * - * @param checkpointMetaData metadata for the checkpoint for which the exception occurred. - * @param exception the exception to handle. - * @throws Exception rethrows the exception if it cannot be handled. + * The maximum number of elements to include in a bundle. */ - void tryHandleCheckpointException(CheckpointMetaData checkpointMetaData, Exception exception) throws Exception; + public static final ConfigOption MAX_BUNDLE_SIZE = ConfigOptions + .key("python.fn-execution.bundle.size") + .defaultValue(1000) + .withDescription("The maximum number of elements to include in a bundle. The elements " + + "are processed asynchronously. One bundle of elements are processed before " + + "processing the next bundle of elements"); + + /** + * The maximum time to wait before finalising a bundle (in milliseconds). + */ + public static final ConfigOption MAX_BUNDLE_TIME_MILLS = ConfigOptions + .key("python.fn-execution.bundle.time") + .defaultValue(1000L) + .withDescription("The maximum time to wait before finishing a bundle (in milliseconds)."); } diff --git a/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/AbstractPythonFunctionOperator.java b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/AbstractPythonFunctionOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..1c869db2ca4af59d2961969cb61384e6407ec1d9 --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/streaming/api/operators/python/AbstractPythonFunctionOperator.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.operators.python; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.python.PythonFunctionRunner; +import org.apache.flink.python.PythonOptions; +import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.BoundedOneInput; +import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; + +import java.util.Map; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Base class for all stream operators to execute Python functions. + */ +@Internal +public abstract class AbstractPythonFunctionOperator + extends AbstractStreamOperator + implements OneInputStreamOperator, BoundedOneInput { + + private static final long serialVersionUID = 1L; + + /** + * The {@link PythonFunctionRunner} which is responsible for Python user-defined function execution. + */ + private transient PythonFunctionRunner pythonFunctionRunner; + + /** + * Use an AtomicBoolean because we start/stop bundles by a timer thread. + */ + private transient AtomicBoolean bundleStarted; + + /** + * Number of processed elements in the current bundle. + */ + private transient int elementCount; + + /** + * Max number of elements to include in a bundle. + */ + private transient int maxBundleSize; + + /** + * Max duration of a bundle. + */ + private transient long maxBundleTimeMills; + + /** + * Time that the last bundle was finished. + */ + private transient long lastFinishBundleTime; + + /** + * A timer that finishes the current bundle after a fixed amount of time. + */ + private transient ScheduledFuture checkFinishBundleTimer; + + /** + * Callback to be executed after the current bundle was finished. + */ + private transient Runnable bundleFinishedCallback; + + @Override + public void open() throws Exception { + try { + this.bundleStarted = new AtomicBoolean(false); + + Map jobParams = getExecutionConfig().getGlobalJobParameters().toMap(); + + this.maxBundleSize = Integer.valueOf(jobParams.getOrDefault( + PythonOptions.MAX_BUNDLE_SIZE.key(), + String.valueOf(PythonOptions.MAX_BUNDLE_SIZE.defaultValue()))); + if (this.maxBundleSize <= 0) { + this.maxBundleSize = PythonOptions.MAX_BUNDLE_SIZE.defaultValue(); + LOG.error("Invalid value for the maximum bundle size. Using default value of " + + this.maxBundleSize + '.'); + } else { + LOG.info("The maximum bundle size is configured to {}.", this.maxBundleSize); + } + + this.maxBundleTimeMills = Long.valueOf(jobParams.getOrDefault( + PythonOptions.MAX_BUNDLE_TIME_MILLS.key(), + String.valueOf(PythonOptions.MAX_BUNDLE_TIME_MILLS.defaultValue()))); + if (this.maxBundleTimeMills <= 0L) { + this.maxBundleTimeMills = PythonOptions.MAX_BUNDLE_TIME_MILLS.defaultValue(); + LOG.error("Invalid value for the maximum bundle time. Using default value of " + + this.maxBundleTimeMills + '.'); + } else { + LOG.info("The maximum bundle time is configured to {} milliseconds.", this.maxBundleTimeMills); + } + + this.pythonFunctionRunner = createPythonFunctionRunner(); + this.pythonFunctionRunner.open(); + + this.elementCount = 0; + this.lastFinishBundleTime = getProcessingTimeService().getCurrentProcessingTime(); + + // Schedule timer to check timeout of finish bundle. + long bundleCheckPeriod = Math.max(this.maxBundleTimeMills, 1); + this.checkFinishBundleTimer = + getProcessingTimeService() + .scheduleAtFixedRate( + // ProcessingTimeService callbacks are executed under the checkpointing lock + timestamp -> checkInvokeFinishBundleByTime(), bundleCheckPeriod, bundleCheckPeriod); + } finally { + super.open(); + } + } + + @Override + public void close() throws Exception { + try { + invokeFinishBundle(); + } finally { + super.close(); + } + } + + @Override + public void dispose() throws Exception { + try { + if (checkFinishBundleTimer != null) { + checkFinishBundleTimer.cancel(true); + checkFinishBundleTimer = null; + } + if (pythonFunctionRunner != null) { + pythonFunctionRunner.close(); + pythonFunctionRunner = null; + } + } finally { + super.dispose(); + } + } + + @Override + public void endInput() throws Exception { + invokeFinishBundle(); + } + + @Override + public void processElement(StreamRecord element) throws Exception { + checkInvokeStartBundle(); + pythonFunctionRunner.processElement(element.getValue()); + checkInvokeFinishBundleByCount(); + } + + @Override + public void prepareSnapshotPreBarrier(long checkpointId) throws Exception { + try { + // Ensures that no new bundle gets started + invokeFinishBundle(); + } finally { + super.prepareSnapshotPreBarrier(checkpointId); + } + } + + @Override + public void processWatermark(Watermark mark) throws Exception { + // Due to the asynchronous communication with the SDK harness, + // a bundle might still be in progress and not all items have + // yet been received from the SDK harness. If we just set this + // watermark as the new output watermark, we could violate the + // order of the records, i.e. pending items in the SDK harness + // could become "late" although they were "on time". + // + // We can solve this problem using one of the following options: + // + // 1) Finish the current bundle and emit this watermark as the + // new output watermark. Finishing the bundle ensures that + // all the items have been processed by the SDK harness and + // the execution results sent to the downstream operator. + // + // 2) Hold on the output watermark for as long as the current + // bundle has not been finished. We have to remember to manually + // finish the bundle in case we receive the final watermark. + // To avoid latency, we should process this watermark again as + // soon as the current bundle is finished. + // + // Approach 1) is the easiest and gives better latency, yet 2) + // gives better throughput due to the bundle not getting cut on + // every watermark. So we have implemented 2) below. + if (mark.getTimestamp() == Long.MAX_VALUE) { + invokeFinishBundle(); + super.processWatermark(mark); + } else if (!bundleStarted.get()) { + // forward the watermark immediately if the bundle is already finished. + super.processWatermark(mark); + } else { + // It is not safe to advance the output watermark yet, so add a hold on the current + // output watermark. + bundleFinishedCallback = + () -> { + try { + // at this point the bundle is finished, allow the watermark to pass + super.processWatermark(mark); + } catch (Exception e) { + throw new RuntimeException( + "Failed to process watermark after finished bundle.", e); + } + }; + } + } + + /** + * Creates the {@link PythonFunctionRunner} which is responsible for Python user-defined function execution. + */ + public abstract PythonFunctionRunner createPythonFunctionRunner(); + + /** + * Sends the execution results to the downstream operator. + */ + public abstract void emitResults(); + + /** + * Checks whether to invoke startBundle. + */ + private void checkInvokeStartBundle() throws Exception { + if (bundleStarted.compareAndSet(false, true)) { + pythonFunctionRunner.startBundle(); + } + } + + /** + * Checks whether to invoke finishBundle by elements count. Called in processElement. + */ + private void checkInvokeFinishBundleByCount() throws Exception { + elementCount++; + if (elementCount >= maxBundleSize) { + invokeFinishBundle(); + } + } + + /** + * Checks whether to invoke finishBundle by timeout. + */ + private void checkInvokeFinishBundleByTime() throws Exception { + long now = getProcessingTimeService().getCurrentProcessingTime(); + if (now - lastFinishBundleTime >= maxBundleTimeMills) { + invokeFinishBundle(); + } + } + + private void invokeFinishBundle() throws Exception { + if (bundleStarted.compareAndSet(true, false)) { + pythonFunctionRunner.finishBundle(); + + emitResults(); + elementCount = 0; + lastFinishBundleTime = getProcessingTimeService().getCurrentProcessingTime(); + // callback only after current bundle was fully finalized + if (bundleFinishedCallback != null) { + bundleFinishedCallback.run(); + bundleFinishedCallback = null; + } + } + } +} diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/AbstractPythonScalarFunctionOperator.java b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/AbstractPythonScalarFunctionOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..bbbc9be9643b95d307bec74dc2ae72fff18b01d8 --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/AbstractPythonScalarFunctionOperator.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.python.PythonFunctionRunner; +import org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.table.functions.ScalarFunction; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.Preconditions; + +import org.apache.beam.sdk.fn.data.FnDataReceiver; + +import java.util.Arrays; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.stream.Collectors; + +/** + * Base class for all stream operators to execute Python {@link ScalarFunction}s. It executes the Python + * {@link ScalarFunction}s in separate Python execution environment. + * + *

The inputs are assumed as the following format: + * {{{ + * +------------------+--------------+ + * | forwarded fields | extra fields | + * +------------------+--------------+ + * }}}. + * + *

The Python UDFs may take input columns directly from the input row or the execution result of Java UDFs: + * 1) The input columns from the input row can be referred from the 'forwarded fields'; + * 2) The Java UDFs will be computed and the execution results can be referred from the 'extra fields'. + * + *

The outputs will be as the following format: + * {{{ + * +------------------+-------------------------+ + * | forwarded fields | scalar function results | + * +------------------+-------------------------+ + * }}}. + * + * @param Type of the input elements. + * @param Type of the output elements. + * @param Type of the UDF input type. + * @param Type of the UDF input type. + */ +@Internal +public abstract class AbstractPythonScalarFunctionOperator + extends AbstractPythonFunctionOperator { + + private static final long serialVersionUID = 1L; + + /** + * The Python {@link ScalarFunction}s to be executed. + */ + protected final PythonFunctionInfo[] scalarFunctions; + + /** + * The input logical type. + */ + protected final RowType inputType; + + /** + * The output logical type. + */ + protected final RowType outputType; + + /** + * The offsets of udf inputs. + */ + protected final int[] udfInputOffsets; + + /** + * The offset of the fields which should be forwarded. + */ + protected final int[] forwardedFields; + + /** + * The udf input logical type. + */ + protected transient RowType udfInputType; + + /** + * The udf output logical type. + */ + protected transient RowType udfOutputType; + + /** + * The queue holding the input elements for which the execution results have not been received. + */ + protected transient LinkedBlockingQueue forwardedInputQueue; + + /** + * The queue holding the user-defined function execution results. The execution results are in + * the same order as the input elements. + */ + protected transient LinkedBlockingQueue udfResultQueue; + + AbstractPythonScalarFunctionOperator( + PythonFunctionInfo[] scalarFunctions, + RowType inputType, + RowType outputType, + int[] udfInputOffsets, + int[] forwardedFields) { + this.scalarFunctions = Preconditions.checkNotNull(scalarFunctions); + this.inputType = Preconditions.checkNotNull(inputType); + this.outputType = Preconditions.checkNotNull(outputType); + this.udfInputOffsets = Preconditions.checkNotNull(udfInputOffsets); + this.forwardedFields = Preconditions.checkNotNull(forwardedFields); + } + + @Override + public void open() throws Exception { + forwardedInputQueue = new LinkedBlockingQueue<>(); + udfResultQueue = new LinkedBlockingQueue<>(); + udfInputType = new RowType( + Arrays.stream(udfInputOffsets) + .mapToObj(i -> inputType.getFields().get(i)) + .collect(Collectors.toList())); + udfOutputType = new RowType(outputType.getFields().subList(forwardedFields.length, outputType.getFieldCount())); + super.open(); + } + + @Override + public void processElement(StreamRecord element) throws Exception { + bufferInput(element.getValue()); + super.processElement(element); + emitResults(); + } + + @Override + public PythonFunctionRunner createPythonFunctionRunner() { + final FnDataReceiver udfResultReceiver = input -> { + // handover to queue, do not block the result receiver thread + udfResultQueue.put(input); + }; + + return new ProjectUdfInputPythonScalarFunctionRunner(createPythonFunctionRunner(udfResultReceiver)); + } + + /** + * Buffers the specified input, it will be used to construct + * the operator result together with the udf execution result. + */ + public abstract void bufferInput(IN input); + + public abstract UDFIN getUdfInput(IN element); + + public abstract PythonFunctionRunner createPythonFunctionRunner(FnDataReceiver resultReceiver); + + private class ProjectUdfInputPythonScalarFunctionRunner implements PythonFunctionRunner { + + private final PythonFunctionRunner pythonFunctionRunner; + + ProjectUdfInputPythonScalarFunctionRunner(PythonFunctionRunner pythonFunctionRunner) { + this.pythonFunctionRunner = pythonFunctionRunner; + } + + @Override + public void open() throws Exception { + pythonFunctionRunner.open(); + } + + @Override + public void close() throws Exception { + pythonFunctionRunner.close(); + } + + @Override + public void startBundle() throws Exception { + pythonFunctionRunner.startBundle(); + } + + @Override + public void finishBundle() throws Exception { + pythonFunctionRunner.finishBundle(); + } + + @Override + public void processElement(IN element) throws Exception { + pythonFunctionRunner.processElement(getUdfInput(element)); + } + } +} diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/BaseRowPythonScalarFunctionOperator.java b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/BaseRowPythonScalarFunctionOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..4274ad550f4fb465af46907c689c1d71abbaf7d1 --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/BaseRowPythonScalarFunctionOperator.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.python.PythonFunctionRunner; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.table.api.TableConfig; +import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.BinaryRow; +import org.apache.flink.table.dataformat.JoinedRow; +import org.apache.flink.table.functions.ScalarFunction; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.planner.codegen.CodeGeneratorContext; +import org.apache.flink.table.planner.codegen.ProjectionCodeGenerator; +import org.apache.flink.table.runtime.generated.GeneratedProjection; +import org.apache.flink.table.runtime.generated.Projection; +import org.apache.flink.table.runtime.runners.python.BaseRowPythonScalarFunctionRunner; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.util.Collector; + +import org.apache.beam.sdk.fn.data.FnDataReceiver; + +import java.util.Arrays; +import java.util.stream.Collectors; + +/** + * The Python {@link ScalarFunction} operator for the blink planner. + */ +@Internal +public class BaseRowPythonScalarFunctionOperator + extends AbstractPythonScalarFunctionOperator { + + private static final long serialVersionUID = 1L; + + /** + * The collector used to collect records. + */ + private transient StreamRecordBaseRowWrappingCollector baseRowWrapper; + + /** + * The JoinedRow reused holding the execution result. + */ + private transient JoinedRow reuseJoinedRow; + + /** + * The Projection which projects the forwarded fields from the input row. + */ + private transient Projection forwardedFieldProjection; + + /** + * The Projection which projects the udf input fields from the input row. + */ + private transient Projection udfInputProjection; + + public BaseRowPythonScalarFunctionOperator( + PythonFunctionInfo[] scalarFunctions, + RowType inputType, + RowType outputType, + int[] udfInputOffsets, + int[] forwardedFields) { + super(scalarFunctions, inputType, outputType, udfInputOffsets, forwardedFields); + } + + @Override + public void open() throws Exception { + super.open(); + baseRowWrapper = new StreamRecordBaseRowWrappingCollector(output); + reuseJoinedRow = new JoinedRow(); + + udfInputProjection = createUdfInputProjection(); + forwardedFieldProjection = createForwardedFieldProjection(); + } + + @Override + public void bufferInput(BaseRow input) { + // always copy the projection result as the generated Projection reuses the projection result + BaseRow forwardedFields = forwardedFieldProjection.apply(input).copy(); + forwardedFields.setHeader(input.getHeader()); + forwardedInputQueue.add(forwardedFields); + } + + @Override + public BaseRow getUdfInput(BaseRow element) { + return udfInputProjection.apply(element); + } + + @Override + @SuppressWarnings("ConstantConditions") + public void emitResults() { + BaseRow udfResult; + while ((udfResult = udfResultQueue.poll()) != null) { + BaseRow input = forwardedInputQueue.poll(); + reuseJoinedRow.setHeader(input.getHeader()); + baseRowWrapper.collect(reuseJoinedRow.replace(input, udfResult)); + } + } + + @Override + public PythonFunctionRunner createPythonFunctionRunner(FnDataReceiver resultReceiver) { + return new BaseRowPythonScalarFunctionRunner( + getRuntimeContext().getTaskName(), + resultReceiver, + scalarFunctions, + scalarFunctions[0].getPythonFunction().getPythonEnv(), + udfInputType, + udfOutputType, + getContainingTask().getEnvironment().getTaskManagerInfo().getTmpDirectories()); + } + + private Projection createUdfInputProjection() { + final GeneratedProjection generatedProjection = ProjectionCodeGenerator.generateProjection( + CodeGeneratorContext.apply(new TableConfig()), + "UdfInputProjection", + inputType, + udfInputType, + udfInputOffsets); + // noinspection unchecked + return generatedProjection.newInstance(Thread.currentThread().getContextClassLoader()); + } + + private Projection createForwardedFieldProjection() { + final RowType forwardedFieldType = new RowType( + Arrays.stream(forwardedFields) + .mapToObj(i -> inputType.getFields().get(i)) + .collect(Collectors.toList())); + final GeneratedProjection generatedProjection = ProjectionCodeGenerator.generateProjection( + CodeGeneratorContext.apply(new TableConfig()), + "ForwardedFieldProjection", + inputType, + forwardedFieldType, + forwardedFields); + // noinspection unchecked + return generatedProjection.newInstance(Thread.currentThread().getContextClassLoader()); + } + + /** + * The collector is used to convert a {@link BaseRow} to a {@link StreamRecord}. + */ + private static class StreamRecordBaseRowWrappingCollector implements Collector { + + private final Collector> out; + + /** + * For Table API & SQL jobs, the timestamp field is not used. + */ + private final StreamRecord reuseStreamRecord = new StreamRecord<>(null); + + StreamRecordBaseRowWrappingCollector(Collector> out) { + this.out = out; + } + + @Override + public void collect(BaseRow record) { + out.collect(reuseStreamRecord.replace(record)); + } + + @Override + public void close() { + out.close(); + } + } +} diff --git a/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/PythonScalarFunctionOperator.java b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/PythonScalarFunctionOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..489c894476b7c76b99d1cd0eaca7ac56e2554991 --- /dev/null +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/operators/python/PythonScalarFunctionOperator.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.java.typeutils.RowTypeInfo; +import org.apache.flink.python.PythonFunctionRunner; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.table.functions.ScalarFunction; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.runtime.runners.python.PythonScalarFunctionRunner; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.runtime.types.CRowTypeInfo; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.utils.TypeConversions; +import org.apache.flink.types.Row; +import org.apache.flink.util.Collector; + +import org.apache.beam.sdk.fn.data.FnDataReceiver; + +import java.util.Arrays; + +/** + * The Python {@link ScalarFunction} operator for the legacy planner. + */ +@Internal +public class PythonScalarFunctionOperator extends AbstractPythonScalarFunctionOperator { + + private static final long serialVersionUID = 1L; + + /** + * The collector used to collect records. + */ + private transient StreamRecordCRowWrappingCollector cRowWrapper; + + /** + * The type serializer for the forwarded fields. + */ + private transient TypeSerializer forwardedInputSerializer; + + public PythonScalarFunctionOperator( + PythonFunctionInfo[] scalarFunctions, + RowType inputType, + RowType outputType, + int[] udfInputOffsets, + int[] forwardedFields) { + super(scalarFunctions, inputType, outputType, udfInputOffsets, forwardedFields); + } + + @Override + public void open() throws Exception { + super.open(); + this.cRowWrapper = new StreamRecordCRowWrappingCollector(output); + + CRowTypeInfo forwardedInputTypeInfo = new CRowTypeInfo(new RowTypeInfo( + Arrays.stream(forwardedFields) + .mapToObj(i -> inputType.getFields().get(i)) + .map(RowType.RowField::getType) + .map(TypeConversions::fromLogicalToDataType) + .map(TypeConversions::fromDataTypeToLegacyInfo) + .toArray(TypeInformation[]::new))); + forwardedInputSerializer = forwardedInputTypeInfo.createSerializer(getExecutionConfig()); + } + + @Override + public void bufferInput(CRow input) { + CRow forwardedFieldsRow = new CRow(Row.project(input.row(), forwardedFields), input.change()); + if (getExecutionConfig().isObjectReuseEnabled()) { + forwardedFieldsRow = forwardedInputSerializer.copy(forwardedFieldsRow); + } + forwardedInputQueue.add(forwardedFieldsRow); + } + + @Override + public Row getUdfInput(CRow element) { + return Row.project(element.row(), udfInputOffsets); + } + + @Override + @SuppressWarnings("ConstantConditions") + public void emitResults() { + Row udfResult; + while ((udfResult = udfResultQueue.poll()) != null) { + CRow input = forwardedInputQueue.poll(); + cRowWrapper.setChange(input.change()); + cRowWrapper.collect(Row.join(input.row(), udfResult)); + } + } + + @Override + public PythonFunctionRunner createPythonFunctionRunner(FnDataReceiver resultReceiver) { + return new PythonScalarFunctionRunner( + getRuntimeContext().getTaskName(), + resultReceiver, + scalarFunctions, + scalarFunctions[0].getPythonFunction().getPythonEnv(), + udfInputType, + udfOutputType, + getContainingTask().getEnvironment().getTaskManagerInfo().getTmpDirectories()); + } + + /** + * The collector is used to convert a {@link Row} to a {@link CRow}. + */ + private static class StreamRecordCRowWrappingCollector implements Collector { + + private final Collector> out; + private final CRow reuseCRow = new CRow(); + + /** + * For Table API & SQL jobs, the timestamp field is not used. + */ + private final StreamRecord reuseStreamRecord = new StreamRecord<>(reuseCRow); + + StreamRecordCRowWrappingCollector(Collector> out) { + this.out = out; + } + + public void setChange(boolean change) { + this.reuseCRow.change_$eq(change); + } + + @Override + public void collect(Row record) { + reuseCRow.row_$eq(record); + out.collect(reuseStreamRecord); + } + + @Override + public void close() { + out.close(); + } + } +} diff --git a/flink-python/src/main/java/org/apache/flink/table/functions/python/AbstractPythonScalarFunctionRunner.java b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/AbstractPythonScalarFunctionRunner.java similarity index 94% rename from flink-python/src/main/java/org/apache/flink/table/functions/python/AbstractPythonScalarFunctionRunner.java rename to flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/AbstractPythonScalarFunctionRunner.java index 104b0585057d097c492373acbb90325cd151d7a0..69c178b0ad5d91aaee6b197e49fb6f76872b5a58 100644 --- a/flink-python/src/main/java/org/apache/flink/table/functions/python/AbstractPythonScalarFunctionRunner.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/AbstractPythonScalarFunctionRunner.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python; +package org.apache.flink.table.runtime.runners.python; import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; @@ -24,6 +24,9 @@ import org.apache.flink.fnexecution.v1.FlinkFnApi; import org.apache.flink.python.AbstractPythonFunctionRunner; import org.apache.flink.python.PythonFunctionRunner; import org.apache.flink.table.functions.ScalarFunction; +import org.apache.flink.table.functions.python.PythonEnv; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.runtime.typeutils.BeamTypeUtils; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.util.Preconditions; @@ -78,8 +81,8 @@ public abstract class AbstractPythonScalarFunctionRunner extends Abstra PythonEnv pythonEnv, RowType inputType, RowType outputType, - String tempDir) { - super(taskName, resultReceiver, pythonEnv, StateRequestHandler.unsupported(), tempDir); + String[] tempDirs) { + super(taskName, resultReceiver, pythonEnv, StateRequestHandler.unsupported(), tempDirs); this.scalarFunctions = Preconditions.checkNotNull(scalarFunctions); this.inputType = Preconditions.checkNotNull(inputType); this.outputType = Preconditions.checkNotNull(outputType); @@ -164,7 +167,7 @@ public abstract class AbstractPythonScalarFunctionRunner extends Abstra * Gets the proto representation of the Python user-defined functions to be executed. */ @VisibleForTesting - FlinkFnApi.UserDefinedFunctions getUserDefinedFunctionsProto() { + public FlinkFnApi.UserDefinedFunctions getUserDefinedFunctionsProto() { FlinkFnApi.UserDefinedFunctions.Builder builder = FlinkFnApi.UserDefinedFunctions.newBuilder(); for (PythonFunctionInfo pythonFunctionInfo : scalarFunctions) { builder.addUdfs(getUserDefinedFunctionProto(pythonFunctionInfo)); @@ -180,8 +183,10 @@ public abstract class AbstractPythonScalarFunctionRunner extends Abstra FlinkFnApi.UserDefinedFunction.Input.newBuilder(); if (input instanceof PythonFunctionInfo) { inputProto.setUdf(getUserDefinedFunctionProto((PythonFunctionInfo) input)); - } else { + } else if (input instanceof Integer) { inputProto.setInputOffset((Integer) input); + } else { + inputProto.setInputConstant(ByteString.copyFrom((byte[]) input)); } builder.addInputs(inputProto); } diff --git a/flink-python/src/main/java/org/apache/flink/table/functions/python/BaseRowPythonScalarFunctionRunner.java b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/BaseRowPythonScalarFunctionRunner.java similarity index 87% rename from flink-python/src/main/java/org/apache/flink/table/functions/python/BaseRowPythonScalarFunctionRunner.java rename to flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/BaseRowPythonScalarFunctionRunner.java index 5f4cbce021481f1e6ee9e249447aec073e0dec10..4bb5fc806c03429d6d02941fd7eca37918501c57 100644 --- a/flink-python/src/main/java/org/apache/flink/table/functions/python/BaseRowPythonScalarFunctionRunner.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/BaseRowPythonScalarFunctionRunner.java @@ -16,12 +16,15 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python; +package org.apache.flink.table.runtime.runners.python; import org.apache.flink.annotation.Internal; import org.apache.flink.python.PythonFunctionRunner; import org.apache.flink.table.dataformat.BaseRow; import org.apache.flink.table.functions.ScalarFunction; +import org.apache.flink.table.functions.python.PythonEnv; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.runtime.typeutils.BeamTypeUtils; import org.apache.flink.table.types.logical.RowType; import org.apache.beam.sdk.coders.Coder; @@ -41,8 +44,8 @@ public class BaseRowPythonScalarFunctionRunner extends AbstractPythonScalarFunct PythonEnv pythonEnv, RowType inputType, RowType outputType, - String tempDir) { - super(taskName, resultReceiver, scalarFunctions, pythonEnv, inputType, outputType, tempDir); + String[] tempDirs) { + super(taskName, resultReceiver, scalarFunctions, pythonEnv, inputType, outputType, tempDirs); } @Override diff --git a/flink-python/src/main/java/org/apache/flink/table/functions/python/PythonScalarFunctionRunner.java b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/PythonScalarFunctionRunner.java similarity index 86% rename from flink-python/src/main/java/org/apache/flink/table/functions/python/PythonScalarFunctionRunner.java rename to flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/PythonScalarFunctionRunner.java index 92f2dd01edec71910d49c2c3847077fccc32c14d..8edba3a00f235393ce033947d2740b426d86ffdc 100644 --- a/flink-python/src/main/java/org/apache/flink/table/functions/python/PythonScalarFunctionRunner.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/runners/python/PythonScalarFunctionRunner.java @@ -16,11 +16,14 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python; +package org.apache.flink.table.runtime.runners.python; import org.apache.flink.annotation.Internal; import org.apache.flink.python.PythonFunctionRunner; import org.apache.flink.table.functions.ScalarFunction; +import org.apache.flink.table.functions.python.PythonEnv; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.runtime.typeutils.BeamTypeUtils; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.types.Row; @@ -41,8 +44,8 @@ public class PythonScalarFunctionRunner extends AbstractPythonScalarFunctionRunn PythonEnv pythonEnv, RowType inputType, RowType outputType, - String tempDir) { - super(taskName, resultReceiver, scalarFunctions, pythonEnv, inputType, outputType, tempDir); + String[] tempDirs) { + super(taskName, resultReceiver, scalarFunctions, pythonEnv, inputType, outputType, tempDirs); } @Override diff --git a/flink-python/src/main/java/org/apache/flink/table/functions/python/BeamTypeUtils.java b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/BeamTypeUtils.java similarity index 98% rename from flink-python/src/main/java/org/apache/flink/table/functions/python/BeamTypeUtils.java rename to flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/BeamTypeUtils.java index ac3e9893b94bdd2596e64074332071b9bf481211..710529cc775dc38d64716496c3081a733e671933 100644 --- a/flink-python/src/main/java/org/apache/flink/table/functions/python/BeamTypeUtils.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/BeamTypeUtils.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python; +package org.apache.flink.table.runtime.typeutils; import org.apache.flink.annotation.Internal; import org.apache.flink.fnexecution.v1.FlinkFnApi; -import org.apache.flink.table.functions.python.coders.BaseRowCoder; -import org.apache.flink.table.functions.python.coders.RowCoder; +import org.apache.flink.table.runtime.typeutils.coders.BaseRowCoder; +import org.apache.flink.table.runtime.typeutils.coders.RowCoder; import org.apache.flink.table.types.logical.AnyType; import org.apache.flink.table.types.logical.ArrayType; import org.apache.flink.table.types.logical.BigIntType; diff --git a/flink-python/src/main/java/org/apache/flink/table/functions/python/coders/BaseRowCoder.java b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/BaseRowCoder.java similarity index 98% rename from flink-python/src/main/java/org/apache/flink/table/functions/python/coders/BaseRowCoder.java rename to flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/BaseRowCoder.java index 8a07c2221fe9e2c5c0b4ecb176cf8614209e1fb3..9834c50b24c9348c052f9ab6ee77ce19465a3be7 100644 --- a/flink-python/src/main/java/org/apache/flink/table/functions/python/coders/BaseRowCoder.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/BaseRowCoder.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python.coders; +package org.apache.flink.table.runtime.typeutils.coders; import org.apache.flink.annotation.Internal; import org.apache.flink.core.memory.DataOutputView; diff --git a/flink-python/src/main/java/org/apache/flink/table/functions/python/coders/ReusableDataInputView.java b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/ReusableDataInputView.java similarity index 95% rename from flink-python/src/main/java/org/apache/flink/table/functions/python/coders/ReusableDataInputView.java rename to flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/ReusableDataInputView.java index 5beaa698f607c01807dcbf635ae6809d4c58c3ec..d71cf3bf8fcdd6ecb44e34ff889227de7ae3293e 100644 --- a/flink-python/src/main/java/org/apache/flink/table/functions/python/coders/ReusableDataInputView.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/ReusableDataInputView.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python.coders; +package org.apache.flink.table.runtime.typeutils.coders; import org.apache.flink.annotation.Internal; import org.apache.flink.core.memory.DataInputView; diff --git a/flink-python/src/main/java/org/apache/flink/table/functions/python/coders/ReusableDataOutputView.java b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/ReusableDataOutputView.java similarity index 95% rename from flink-python/src/main/java/org/apache/flink/table/functions/python/coders/ReusableDataOutputView.java rename to flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/ReusableDataOutputView.java index 9aabb4f8dafa20d05b6f00d073bffe627cbdc5f5..6128044648194ce7a99b09d9328b86af4c0482aa 100644 --- a/flink-python/src/main/java/org/apache/flink/table/functions/python/coders/ReusableDataOutputView.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/ReusableDataOutputView.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python.coders; +package org.apache.flink.table.runtime.typeutils.coders; import org.apache.flink.annotation.Internal; import org.apache.flink.core.memory.DataOutputView; diff --git a/flink-python/src/main/java/org/apache/flink/table/functions/python/coders/RowCoder.java b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/RowCoder.java similarity index 98% rename from flink-python/src/main/java/org/apache/flink/table/functions/python/coders/RowCoder.java rename to flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/RowCoder.java index 5b1e8e0e0ac7987fbeed3862bb4282d9277058db..b72d5f117fcea5baff8442e117bedb0bfd4399ae 100644 --- a/flink-python/src/main/java/org/apache/flink/table/functions/python/coders/RowCoder.java +++ b/flink-python/src/main/java/org/apache/flink/table/runtime/typeutils/coders/RowCoder.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python.coders; +package org.apache.flink.table.runtime.typeutils.coders; import org.apache.flink.annotation.Internal; import org.apache.flink.types.Row; diff --git a/flink-python/src/main/resources/META-INF/NOTICE b/flink-python/src/main/resources/META-INF/NOTICE index b02bb4d2bb5363133609b03ba6dcf31119dc459b..b2107533d628c8e6b0becbbbd7961e0e45021602 100644 --- a/flink-python/src/main/resources/META-INF/NOTICE +++ b/flink-python/src/main/resources/META-INF/NOTICE @@ -53,6 +53,7 @@ See bundled license files for details - com.google.protobuf:protobuf-java:3.7.1 - com.google.protobuf:protobuf-java-util:3.7.1 - com.google.auth:google-auth-library-credentials:0.13.0 +- cloudpickle:1.2.2 This project bundles the following dependencies under the MIT license. (https://opensource.org/licenses/MIT) See bundled license files for details. diff --git a/licenses-binary/LICENSE.split b/flink-python/src/main/resources/META-INF/licenses/LICENSE.cloudpickle similarity index 30% rename from licenses-binary/LICENSE.split rename to flink-python/src/main/resources/META-INF/licenses/LICENSE.cloudpickle index 19b305b00060a774a9180fb916c14b49edb2008f..d112c4806aa90b4c911b360fa0ce1d0ee7c031ce 100644 --- a/licenses-binary/LICENSE.split +++ b/flink-python/src/main/resources/META-INF/licenses/LICENSE.cloudpickle @@ -1,32 +1,32 @@ -Copyright 2008 Google Inc. All rights reserved. +This module was extracted from the `cloud` package, developed by +PiCloud, Inc. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +Copyright (c) 2015, Cloudpickle contributors. +Copyright (c) 2012, Regents of the University of California. +Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California, Berkeley nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/flink-python/src/test/java/org/apache/flink/python/PythonOptionsTest.java b/flink-python/src/test/java/org/apache/flink/python/PythonOptionsTest.java new file mode 100644 index 0000000000000000000000000000000000000000..74072dc2b965685529b4cd6ca242d5960ab650f0 --- /dev/null +++ b/flink-python/src/test/java/org/apache/flink/python/PythonOptionsTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.python; + +import org.apache.flink.configuration.Configuration; + +import org.junit.Test; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertThat; + +/** + * Test all configurations can be set using configuration. + */ +public class PythonOptionsTest { + + @Test + public void testBundleSize() { + final Configuration configuration = new Configuration(); + final int defaultBundleSize = configuration.getInteger(PythonOptions.MAX_BUNDLE_SIZE); + assertThat(defaultBundleSize, is(equalTo(PythonOptions.MAX_BUNDLE_SIZE.defaultValue()))); + + final int expectedBundleSize = 100; + configuration.setInteger(PythonOptions.MAX_BUNDLE_SIZE, expectedBundleSize); + + final int actualBundleSize = configuration.getInteger(PythonOptions.MAX_BUNDLE_SIZE); + assertThat(actualBundleSize, is(equalTo(expectedBundleSize))); + } + + @Test + public void testBundleTime() { + final Configuration configuration = new Configuration(); + final long defaultBundleTime = configuration.getLong(PythonOptions.MAX_BUNDLE_TIME_MILLS); + assertThat(defaultBundleTime, is(equalTo(PythonOptions.MAX_BUNDLE_TIME_MILLS.defaultValue()))); + + final long expectedBundleTime = 100; + configuration.setLong(PythonOptions.MAX_BUNDLE_TIME_MILLS, expectedBundleTime); + + final long actualBundleSize = configuration.getLong(PythonOptions.MAX_BUNDLE_TIME_MILLS); + assertThat(actualBundleSize, is(equalTo(expectedBundleTime))); + } +} diff --git a/flink-python/src/test/java/org/apache/flink/table/functions/python/AbstractPythonScalarFunctionRunnerTest.java b/flink-python/src/test/java/org/apache/flink/table/functions/python/AbstractPythonScalarFunctionRunnerTest.java index b811d78f5770b8e14e71947d3f2b5bb36ac63dd8..4c28d6a8515d53fec3ee82fc7307895f103869b9 100644 --- a/flink-python/src/test/java/org/apache/flink/table/functions/python/AbstractPythonScalarFunctionRunnerTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/functions/python/AbstractPythonScalarFunctionRunnerTest.java @@ -18,6 +18,7 @@ package org.apache.flink.table.functions.python; +import org.apache.flink.table.runtime.runners.python.AbstractPythonScalarFunctionRunner; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.RowType; @@ -120,7 +121,7 @@ public abstract class AbstractPythonScalarFunctionRunnerTest { @Override public PythonEnv getPythonEnv() { - return new PythonEnv("", "", PythonEnv.ExecType.PROCESS); + return new PythonEnv(PythonEnv.ExecType.PROCESS); } } } diff --git a/flink-python/src/test/java/org/apache/flink/table/functions/python/BaseRowPythonScalarFunctionRunnerTest.java b/flink-python/src/test/java/org/apache/flink/table/functions/python/BaseRowPythonScalarFunctionRunnerTest.java index 4e8e655a8d68c28f8195451647dc0a06fa98dab7..dacd5092b54f4727704dc6ad5f17e1b23ea05e0e 100644 --- a/flink-python/src/test/java/org/apache/flink/table/functions/python/BaseRowPythonScalarFunctionRunnerTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/functions/python/BaseRowPythonScalarFunctionRunnerTest.java @@ -19,7 +19,9 @@ package org.apache.flink.table.functions.python; import org.apache.flink.table.dataformat.BaseRow; -import org.apache.flink.table.functions.python.coders.BaseRowCoder; +import org.apache.flink.table.runtime.runners.python.AbstractPythonScalarFunctionRunner; +import org.apache.flink.table.runtime.runners.python.BaseRowPythonScalarFunctionRunner; +import org.apache.flink.table.runtime.typeutils.coders.BaseRowCoder; import org.apache.flink.table.types.logical.RowType; import org.apache.beam.sdk.coders.Coder; @@ -114,7 +116,7 @@ public class BaseRowPythonScalarFunctionRunnerTest extends AbstractPythonScalarF // ignore the execution results }; - final PythonEnv pythonEnv = new PythonEnv("", "", PythonEnv.ExecType.PROCESS); + final PythonEnv pythonEnv = new PythonEnv(PythonEnv.ExecType.PROCESS); return new BaseRowPythonScalarFunctionRunner( "testPythonRunner", @@ -123,6 +125,6 @@ public class BaseRowPythonScalarFunctionRunnerTest extends AbstractPythonScalarF pythonEnv, inputType, outputType, - System.getProperty("java.io.tmpdir")); + new String[] {System.getProperty("java.io.tmpdir")}); } } diff --git a/flink-python/src/test/java/org/apache/flink/table/functions/python/BeamTypeUtilsTest.java b/flink-python/src/test/java/org/apache/flink/table/functions/python/BeamTypeUtilsTest.java index 7acf7b3d7e5db4b02cf29e4031e860642c0433b3..0f3f3083e214a5e600bc7600284b1c9420bab293 100644 --- a/flink-python/src/test/java/org/apache/flink/table/functions/python/BeamTypeUtilsTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/functions/python/BeamTypeUtilsTest.java @@ -19,8 +19,9 @@ package org.apache.flink.table.functions.python; import org.apache.flink.fnexecution.v1.FlinkFnApi; -import org.apache.flink.table.functions.python.coders.BaseRowCoder; -import org.apache.flink.table.functions.python.coders.RowCoder; +import org.apache.flink.table.runtime.typeutils.BeamTypeUtils; +import org.apache.flink.table.runtime.typeutils.coders.BaseRowCoder; +import org.apache.flink.table.runtime.typeutils.coders.RowCoder; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.RowType; diff --git a/flink-python/src/test/java/org/apache/flink/table/functions/python/PythonScalarFunctionRunnerTest.java b/flink-python/src/test/java/org/apache/flink/table/functions/python/PythonScalarFunctionRunnerTest.java index c3b63bbb6e844fbcd7c8a98d5f35c5b03249947f..212bf2a6e8e964041111de18402377a9aed48f19 100644 --- a/flink-python/src/test/java/org/apache/flink/table/functions/python/PythonScalarFunctionRunnerTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/functions/python/PythonScalarFunctionRunnerTest.java @@ -19,7 +19,9 @@ package org.apache.flink.table.functions.python; import org.apache.flink.fnexecution.v1.FlinkFnApi; -import org.apache.flink.table.functions.python.coders.RowCoder; +import org.apache.flink.table.runtime.runners.python.AbstractPythonScalarFunctionRunner; +import org.apache.flink.table.runtime.runners.python.PythonScalarFunctionRunner; +import org.apache.flink.table.runtime.typeutils.coders.RowCoder; import org.apache.flink.table.types.logical.BigIntType; import org.apache.flink.table.types.logical.RowType; import org.apache.flink.types.Row; @@ -242,7 +244,7 @@ public class PythonScalarFunctionRunnerTest extends AbstractPythonScalarFunction // ignore the execution results }; - final PythonEnv pythonEnv = new PythonEnv("", "", PythonEnv.ExecType.PROCESS); + final PythonEnv pythonEnv = new PythonEnv(PythonEnv.ExecType.PROCESS); return new PythonScalarFunctionRunner( "testPythonRunner", @@ -251,7 +253,7 @@ public class PythonScalarFunctionRunnerTest extends AbstractPythonScalarFunction pythonEnv, inputType, outputType, - System.getProperty("java.io.tmpdir")); + new String[] {System.getProperty("java.io.tmpdir")}); } private AbstractPythonScalarFunctionRunner createUDFRunner( @@ -264,7 +266,7 @@ public class PythonScalarFunctionRunnerTest extends AbstractPythonScalarFunction RowType rowType = new RowType(Collections.singletonList(new RowType.RowField("f1", new BigIntType()))); - final PythonEnv pythonEnv = new PythonEnv("", "", PythonEnv.ExecType.PROCESS); + final PythonEnv pythonEnv = new PythonEnv(PythonEnv.ExecType.PROCESS); return new PythonScalarFunctionRunnerTestHarness( "testPythonRunner", @@ -274,7 +276,7 @@ public class PythonScalarFunctionRunnerTest extends AbstractPythonScalarFunction rowType, rowType, jobBundleFactory, - System.getProperty("java.io.tmpdir")); + new String[] {System.getProperty("java.io.tmpdir")}); } private static class PythonScalarFunctionRunnerTestHarness extends PythonScalarFunctionRunner { @@ -288,8 +290,8 @@ public class PythonScalarFunctionRunnerTest extends AbstractPythonScalarFunction PythonEnv pythonEnv, RowType inputType, RowType outputType, JobBundleFactory jobBundleFactory, - String tempDir) { - super(taskName, resultReceiver, scalarFunctions, pythonEnv, inputType, outputType, tempDir); + String[] tempDirs) { + super(taskName, resultReceiver, scalarFunctions, pythonEnv, inputType, outputType, tempDirs); this.jobBundleFactory = jobBundleFactory; } diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/BaseRowPythonScalarFunctionOperatorTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/BaseRowPythonScalarFunctionOperatorTest.java new file mode 100644 index 0000000000000000000000000000000000000000..93704d2820a8006303231cb01931e74d5ae84a38 --- /dev/null +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/BaseRowPythonScalarFunctionOperatorTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.python.PythonFunctionRunner; +import org.apache.flink.table.dataformat.BaseRow; +import org.apache.flink.table.dataformat.util.BaseRowUtil; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.runtime.util.BaseRowHarnessAssertor; +import org.apache.flink.table.types.logical.RowType; + +import org.apache.beam.sdk.fn.data.FnDataReceiver; + +import java.util.Collection; + +import static org.apache.flink.table.runtime.util.StreamRecordUtils.baserow; +import static org.apache.flink.table.runtime.util.StreamRecordUtils.binaryrow; + +/** + * Tests for {@link BaseRowPythonScalarFunctionOperator}. + */ +public class BaseRowPythonScalarFunctionOperatorTest + extends PythonScalarFunctionOperatorTestBase { + + private final BaseRowHarnessAssertor assertor = new BaseRowHarnessAssertor(new TypeInformation[]{ + Types.STRING, + Types.STRING, + Types.LONG + }); + + @Override + public AbstractPythonScalarFunctionOperator getTestOperator( + PythonFunctionInfo[] scalarFunctions, + RowType inputType, + RowType outputType, + int[] udfInputOffsets, + int[] forwardedFields) { + return new PassThroughPythonScalarFunctionOperator( + scalarFunctions, + inputType, + outputType, + udfInputOffsets, + forwardedFields + ); + } + + @Override + public BaseRow newRow(boolean accumulateMsg, Object... fields) { + if (accumulateMsg) { + return baserow(fields); + } else { + return BaseRowUtil.setRetract(baserow(fields)); + } + } + + @Override + public void assertOutputEquals(String message, Collection expected, Collection actual) { + assertor.assertOutputEquals(message, expected, actual); + } + + private static class PassThroughPythonScalarFunctionOperator extends BaseRowPythonScalarFunctionOperator { + + PassThroughPythonScalarFunctionOperator( + PythonFunctionInfo[] scalarFunctions, + RowType inputType, + RowType outputType, + int[] udfInputOffsets, + int[] forwardedFields) { + super(scalarFunctions, inputType, outputType, udfInputOffsets, forwardedFields); + } + + @Override + public PythonFunctionRunner createPythonFunctionRunner( + FnDataReceiver resultReceiver) { + return new PassThroughPythonFunctionRunner(resultReceiver) { + @Override + public BaseRow copy(BaseRow element) { + BaseRow row = binaryrow(element.getLong(0)); + row.setHeader(element.getHeader()); + return row; + } + }; + } + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/CheckpointExceptionHandlerFactory.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/PassThroughPythonFunctionRunner.java similarity index 40% rename from flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/CheckpointExceptionHandlerFactory.java rename to flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/PassThroughPythonFunctionRunner.java index 64ab71ce0022f5aafff0f542a3304c4be74519a2..9660ab0ffe1de3b0ac6a8e90aca97b7af4767414 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/CheckpointExceptionHandlerFactory.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/PassThroughPythonFunctionRunner.java @@ -16,44 +16,60 @@ * limitations under the License. */ -package org.apache.flink.streaming.runtime.tasks; +package org.apache.flink.table.runtime.operators.python; -import org.apache.flink.runtime.checkpoint.CheckpointMetaData; -import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.python.PythonFunctionRunner; import org.apache.flink.util.Preconditions; +import org.apache.beam.sdk.fn.data.FnDataReceiver; + +import java.util.ArrayList; +import java.util.List; + /** - * This factory produces {@link CheckpointExceptionHandler} instances that handle exceptions during checkpointing in a - * {@link StreamTask}. + * A {@link PythonFunctionRunner} that just emit each input element. + * + * @param Type of the input elements. */ -public class CheckpointExceptionHandlerFactory { +public abstract class PassThroughPythonFunctionRunner implements PythonFunctionRunner { - /** - * Returns a {@link CheckpointExceptionHandler} that just declines checkpoint on exception. - */ - public CheckpointExceptionHandler createCheckpointExceptionHandler( - Environment environment) { + private boolean bundleStarted; + private final List bufferedElements; + private final FnDataReceiver resultReceiver; - return new DecliningCheckpointExceptionHandler(environment); + PassThroughPythonFunctionRunner(FnDataReceiver resultReceiver) { + this.resultReceiver = Preconditions.checkNotNull(resultReceiver); + bundleStarted = false; + bufferedElements = new ArrayList<>(); } - /** - * This handler makes the task decline the checkpoint as reaction to the reported exception. The task is not failed. - */ - static final class DecliningCheckpointExceptionHandler implements CheckpointExceptionHandler { + @Override + public void open() {} - final Environment environment; + @Override + public void close() {} - DecliningCheckpointExceptionHandler(Environment environment) { - this.environment = Preconditions.checkNotNull(environment); - } + @Override + public void startBundle() { + Preconditions.checkState(!bundleStarted); + bundleStarted = true; + } - @Override - public void tryHandleCheckpointException( - CheckpointMetaData checkpointMetaData, - Exception exception) { + @Override + public void finishBundle() throws Exception { + Preconditions.checkState(bundleStarted); + bundleStarted = false; - environment.declineCheckpoint(checkpointMetaData.getCheckpointId(), exception); + for (IN element : bufferedElements) { + resultReceiver.accept(element); } + bufferedElements.clear(); + } + + @Override + public void processElement(IN element) { + bufferedElements.add(copy(element)); } + + public abstract IN copy(IN element); } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/CheckpointExceptionHandlerTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/PythonScalarFunctionOperatorTest.java similarity index 30% rename from flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/CheckpointExceptionHandlerTest.java rename to flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/PythonScalarFunctionOperatorTest.java index 2632c0102781f2b288f13d1cfccc90921485ee42..57e7497e05963f0237830fa94b2b85828b9963f7 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/CheckpointExceptionHandlerTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/PythonScalarFunctionOperatorTest.java @@ -16,62 +16,66 @@ * limitations under the License. */ -package org.apache.flink.streaming.runtime.tasks; +package org.apache.flink.table.runtime.operators.python; -import org.apache.flink.runtime.checkpoint.CheckpointMetaData; -import org.apache.flink.runtime.operators.testutils.DummyEnvironment; -import org.apache.flink.util.TestLogger; +import org.apache.flink.python.PythonFunctionRunner; +import org.apache.flink.streaming.util.TestHarnessUtil; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.runtime.types.CRow; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.types.Row; -import org.junit.Assert; -import org.junit.Test; +import org.apache.beam.sdk.fn.data.FnDataReceiver; + +import java.util.Collection; +import java.util.Queue; /** - * Test for the current implementations of {@link CheckpointExceptionHandler} and their factory. + * Tests for {@link PythonScalarFunctionOperator}. */ -public class CheckpointExceptionHandlerTest extends TestLogger { - - @Test - public void testDecliningHandler() { - DeclineDummyEnvironment environment = new DeclineDummyEnvironment(); - CheckpointExceptionHandlerFactory checkpointExceptionHandlerFactory = new CheckpointExceptionHandlerFactory(); - CheckpointExceptionHandler exceptionHandler = - checkpointExceptionHandlerFactory.createCheckpointExceptionHandler(environment); +public class PythonScalarFunctionOperatorTest extends PythonScalarFunctionOperatorTestBase { - CheckpointMetaData failedCheckpointMetaData = new CheckpointMetaData(42L, 4711L); - Exception testException = new Exception("test"); - try { - exceptionHandler.tryHandleCheckpointException(failedCheckpointMetaData, testException); - } catch (Exception e) { - Assert.fail("Exception not handled, but rethrown."); - } + @Override + public AbstractPythonScalarFunctionOperator getTestOperator( + PythonFunctionInfo[] scalarFunctions, + RowType inputType, + RowType outputType, + int[] udfInputOffsets, + int[] forwardedFields) { + return new PassThroughPythonScalarFunctionOperator( + scalarFunctions, inputType, outputType, udfInputOffsets, forwardedFields); + } - Assert.assertEquals(failedCheckpointMetaData.getCheckpointId(), environment.getLastDeclinedCheckpointId()); - Assert.assertEquals(testException, environment.getLastDeclinedCheckpointCause()); + @Override + public CRow newRow(boolean accumulateMsg, Object... fields) { + return new CRow(Row.of(fields), accumulateMsg); } - static final class DeclineDummyEnvironment extends DummyEnvironment { + @Override + public void assertOutputEquals(String message, Collection expected, Collection actual) { + TestHarnessUtil.assertOutputEquals(message, (Queue) expected, (Queue) actual); + } - private long lastDeclinedCheckpointId; - private Throwable lastDeclinedCheckpointCause; + private static class PassThroughPythonScalarFunctionOperator extends PythonScalarFunctionOperator { - DeclineDummyEnvironment() { - super("test", 1, 0); - this.lastDeclinedCheckpointId = Long.MIN_VALUE; - this.lastDeclinedCheckpointCause = null; + PassThroughPythonScalarFunctionOperator( + PythonFunctionInfo[] scalarFunctions, + RowType inputType, + RowType outputType, + int[] udfInputOffsets, + int[] forwardedFields) { + super(scalarFunctions, inputType, outputType, udfInputOffsets, forwardedFields); } @Override - public void declineCheckpoint(long checkpointId, Throwable cause) { - this.lastDeclinedCheckpointId = checkpointId; - this.lastDeclinedCheckpointCause = cause; - } - - long getLastDeclinedCheckpointId() { - return lastDeclinedCheckpointId; - } - - Throwable getLastDeclinedCheckpointCause() { - return lastDeclinedCheckpointCause; + public PythonFunctionRunner createPythonFunctionRunner( + FnDataReceiver resultReceiver) { + return new PassThroughPythonFunctionRunner(resultReceiver) { + @Override + public Row copy(Row element) { + return Row.copy(element); + } + }; } } } diff --git a/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/PythonScalarFunctionOperatorTestBase.java b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/PythonScalarFunctionOperatorTestBase.java new file mode 100644 index 0000000000000000000000000000000000000000..195fc77ca76468e4143f81d3683b45b870d54333 --- /dev/null +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/operators/python/PythonScalarFunctionOperatorTestBase.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.runtime.operators.python; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.python.PythonOptions; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; +import org.apache.flink.table.functions.python.AbstractPythonScalarFunctionRunnerTest; +import org.apache.flink.table.functions.python.PythonFunctionInfo; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.concurrent.ConcurrentLinkedQueue; + +/** + * Base class for Python scalar function operator test. These test that: + * + *
    + *
  • Retraction flag is correctly forwarded to the downstream
    • + *
  • FinishBundle is called when checkpoint is encountered
    • + *
  • Watermarks are buffered and only sent to downstream when finishedBundle is triggered
    • + *
+ * + * @param Type of the input elements. + * @param Type of the output elements. + * @param Type of the UDF input type. + * @param Type of the UDF input type. + */ +public abstract class PythonScalarFunctionOperatorTestBase { + + @Test + public void testRetractionFieldKept() throws Exception { + OneInputStreamOperatorTestHarness testHarness = getTestHarness(); + + long initialTime = 0L; + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + testHarness.open(); + + testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 0L), initialTime + 1)); + testHarness.processElement(new StreamRecord<>(newRow(false, "c3", "c4", 1L), initialTime + 2)); + testHarness.processElement(new StreamRecord<>(newRow(false, "c5", "c6", 2L), initialTime + 3)); + testHarness.close(); + + expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L))); + expectedOutput.add(new StreamRecord<>(newRow(false, "c3", "c4", 1L))); + expectedOutput.add(new StreamRecord<>(newRow(false, "c5", "c6", 2L))); + + assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + } + + @Test + public void testFinishBundleTriggeredOnCheckpoint() throws Exception { + OneInputStreamOperatorTestHarness testHarness = getTestHarness(); + + Configuration conf = new Configuration(); + conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 10); + testHarness.getExecutionConfig().setGlobalJobParameters(conf); + + long initialTime = 0L; + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + testHarness.open(); + + testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 0L), initialTime + 1)); + + // checkpoint trigger finishBundle + testHarness.prepareSnapshotPreBarrier(0L); + + expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L))); + + assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + @Test + public void testFinishBundleTriggeredByCount() throws Exception { + OneInputStreamOperatorTestHarness testHarness = getTestHarness(); + Configuration conf = new Configuration(); + conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 2); + testHarness.getExecutionConfig().setGlobalJobParameters(conf); + + long initialTime = 0L; + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + testHarness.open(); + + testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 0L), initialTime + 1)); + assertOutputEquals("FinishBundle should not be triggered.", expectedOutput, testHarness.getOutput()); + + testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 1L), initialTime + 2)); + expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L))); + expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 1L))); + + assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + @Test + public void testFinishBundleTriggeredByTime() throws Exception { + OneInputStreamOperatorTestHarness testHarness = getTestHarness(); + Configuration conf = new Configuration(); + conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 10); + conf.setLong(PythonOptions.MAX_BUNDLE_TIME_MILLS, 1000L); + testHarness.getExecutionConfig().setGlobalJobParameters(conf); + + long initialTime = 0L; + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + testHarness.open(); + + testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 0L), initialTime + 1)); + assertOutputEquals("FinishBundle should not be triggered.", expectedOutput, testHarness.getOutput()); + + testHarness.setProcessingTime(1000L); + expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L))); + assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + @Test + public void testFinishBundleTriggeredByClose() throws Exception { + OneInputStreamOperatorTestHarness testHarness = getTestHarness(); + Configuration conf = new Configuration(); + conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 10); + testHarness.getExecutionConfig().setGlobalJobParameters(conf); + + long initialTime = 0L; + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + testHarness.open(); + + testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 0L), initialTime + 1)); + assertOutputEquals("FinishBundle should not be triggered.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L))); + assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + } + + @Test + public void testWatermarkProcessedOnFinishBundle() throws Exception { + OneInputStreamOperatorTestHarness testHarness = getTestHarness(); + Configuration conf = new Configuration(); + conf.setInteger(PythonOptions.MAX_BUNDLE_SIZE, 10); + testHarness.getExecutionConfig().setGlobalJobParameters(conf); + long initialTime = 0L; + ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + + testHarness.open(); + + testHarness.processElement(new StreamRecord<>(newRow(true, "c1", "c2", 0L), initialTime + 1)); + testHarness.processWatermark(initialTime + 2); + assertOutputEquals("Watermark has been processed", expectedOutput, testHarness.getOutput()); + + // checkpoint trigger finishBundle + testHarness.prepareSnapshotPreBarrier(0L); + + expectedOutput.add(new StreamRecord<>(newRow(true, "c1", "c2", 0L))); + expectedOutput.add(new Watermark(initialTime + 2)); + + assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + + testHarness.close(); + } + + private OneInputStreamOperatorTestHarness getTestHarness() throws Exception { + RowType dataType = new RowType(Arrays.asList( + new RowType.RowField("f1", new VarCharType()), + new RowType.RowField("f2", new VarCharType()), + new RowType.RowField("f3", new BigIntType()))); + AbstractPythonScalarFunctionOperator operator = getTestOperator( + new PythonFunctionInfo[] { + new PythonFunctionInfo( + AbstractPythonScalarFunctionRunnerTest.DummyPythonFunction.INSTANCE, + new Integer[]{0}) + }, + dataType, + dataType, + new int[]{2}, + new int[]{0, 1} + ); + + return new OneInputStreamOperatorTestHarness<>(operator); + } + + public abstract AbstractPythonScalarFunctionOperator getTestOperator( + PythonFunctionInfo[] scalarFunctions, + RowType inputType, + RowType outputType, + int[] udfInputOffsets, + int[] forwardedFields); + + public abstract IN newRow(boolean accumulateMsg, Object... fields); + + public abstract void assertOutputEquals(String message, Collection expected, Collection actual); +} diff --git a/flink-python/src/test/java/org/apache/flink/table/functions/python/coders/BaseRowCoderTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/coders/BaseRowCoderTest.java similarity index 98% rename from flink-python/src/test/java/org/apache/flink/table/functions/python/coders/BaseRowCoderTest.java rename to flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/coders/BaseRowCoderTest.java index 57eb79b086c2ea74acbc59870d4fffbd79c57b08..7b5e281a3a5b140a862a4494865fd1421e5e0ac9 100644 --- a/flink-python/src/test/java/org/apache/flink/table/functions/python/coders/BaseRowCoderTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/coders/BaseRowCoderTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python.coders; +package org.apache.flink.table.runtime.typeutils.coders; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.table.dataformat.BaseRow; diff --git a/flink-python/src/test/java/org/apache/flink/table/functions/python/coders/CoderTestBase.java b/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/coders/CoderTestBase.java similarity index 98% rename from flink-python/src/test/java/org/apache/flink/table/functions/python/coders/CoderTestBase.java rename to flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/coders/CoderTestBase.java index 642a5447bbab204184494221942a2802fb76c0f6..308fe5dd7412db8c76a249e44fa3fab961f149ed 100644 --- a/flink-python/src/test/java/org/apache/flink/table/functions/python/coders/CoderTestBase.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/coders/CoderTestBase.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python.coders; +package org.apache.flink.table.runtime.typeutils.coders; import org.apache.flink.testutils.CustomEqualityMatcher; import org.apache.flink.testutils.DeeplyEqualsChecker; diff --git a/flink-python/src/test/java/org/apache/flink/table/functions/python/coders/RowCoderTest.java b/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/coders/RowCoderTest.java similarity index 96% rename from flink-python/src/test/java/org/apache/flink/table/functions/python/coders/RowCoderTest.java rename to flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/coders/RowCoderTest.java index 93b1c003e0d20fdf2a76fa277cc02727d13fb619..e8b64830b67ef3e13fdb1d2d93e3fd98c622c27c 100644 --- a/flink-python/src/test/java/org/apache/flink/table/functions/python/coders/RowCoderTest.java +++ b/flink-python/src/test/java/org/apache/flink/table/runtime/typeutils/coders/RowCoderTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.flink.table.functions.python.coders; +package org.apache.flink.table.runtime.typeutils.coders; import org.apache.flink.types.Row; diff --git a/flink-python/tox.ini b/flink-python/tox.ini index 4efde543a0205494f1893311280a2c28d317bdd1..873af202864577376e6b5d37ed843787bab8def4 100644 --- a/flink-python/tox.ini +++ b/flink-python/tox.ini @@ -38,4 +38,4 @@ commands = # up to 100 characters in length, not 79. ignore=E226,E241,E305,E402,E722,E731,E741,W503,W504 max-line-length=100 -exclude=.tox/*,dev/*,lib/*,target/*,build/*,dist/*,pyflink/shell.py,.eggs/*,pyflink/fn_execution/tests/process_mode_test_data.py +exclude=.tox/*,dev/*,lib/*,target/*,build/*,dist/*,pyflink/shell.py,.eggs/*,pyflink/fn_execution/tests/process_mode_test_data.py,pyflink/fn_execution/*_pb2.py diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java index 43fdf13a4bb2143032ae3b5849b26deb63703c15..de76d2bd059320730f779cd5851756d5adc7577f 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateFsBackendITCase.java @@ -96,7 +96,7 @@ public class HAQueryableStateFsBackendITCase extends AbstractQueryableStateTestB Configuration config = new Configuration(); config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "4m"); config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java index 8a622436977fb135e66673fe6ebc4558ee05b925..e170f0928bdb3e3ba674573cb90272448f1c4abe 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/HAQueryableStateRocksDBBackendITCase.java @@ -96,7 +96,7 @@ public class HAQueryableStateRocksDBBackendITCase extends AbstractQueryableState Configuration config = new Configuration(); config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "4m"); config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java index 183ad9e59b5c155a10c14ae359478d76944b38f0..89797d7418eac50f619c6851d26bd709b255ccfe 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateFsBackendITCase.java @@ -80,7 +80,7 @@ public class NonHAQueryableStateFsBackendITCase extends AbstractQueryableStateTe private static Configuration getConfig() { Configuration config = new Configuration(); config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "4m"); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1); diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java index 34713f38f0209bb1fe16301bf6f73c2c8cec5560..3880051b2e2b86394f2fbc27cec39d9deb8d4e20 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/itcases/NonHAQueryableStateRocksDBBackendITCase.java @@ -79,7 +79,7 @@ public class NonHAQueryableStateRocksDBBackendITCase extends AbstractQueryableSt private static Configuration getConfig() { Configuration config = new Configuration(); config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "4m"); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1); diff --git a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/AbstractServerTest.java b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/AbstractServerTest.java index 9e5720d15595e1ed8ab6e9cdc50e8a9dfe62d4b0..02bfa7cf090ca0fe2baba62ac00f3a70495e58d8 100644 --- a/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/AbstractServerTest.java +++ b/flink-queryable-state/flink-queryable-state-runtime/src/test/java/org/apache/flink/queryablestate/network/AbstractServerTest.java @@ -86,8 +86,8 @@ public class AbstractServerTest extends TestLogger { @Test public void testPortRangeSuccess() throws Throwable { - // this is shared between the two servers. - AtomicKvStateRequestStats serverStats = new AtomicKvStateRequestStats(); + AtomicKvStateRequestStats serverStats1 = new AtomicKvStateRequestStats(); + AtomicKvStateRequestStats serverStats2 = new AtomicKvStateRequestStats(); AtomicKvStateRequestStats clientStats = new AtomicKvStateRequestStats(); final int portRangeStart = 7777; @@ -95,8 +95,8 @@ public class AbstractServerTest extends TestLogger { List portList = IntStream.range(portRangeStart, portRangeEnd + 1).boxed().collect(Collectors.toList()); try ( - TestServer server1 = new TestServer("Test Server 1", serverStats, portList.iterator()); - TestServer server2 = new TestServer("Test Server 2", serverStats, portList.iterator()); + TestServer server1 = new TestServer("Test Server 1", serverStats1, portList.iterator()); + TestServer server2 = new TestServer("Test Server 2", serverStats2, portList.iterator()); TestClient client = new TestClient( "Test Client", 1, @@ -116,8 +116,8 @@ public class AbstractServerTest extends TestLogger { TestMessage response2 = client.sendRequest(server2.getServerAddress(), new TestMessage("pong")).join(); Assert.assertEquals(server2.getServerName() + "-pong", response2.getMessage()); - // the client connects to both servers and the stats object is shared. - Assert.assertEquals(2L, serverStats.getNumConnections()); + Assert.assertEquals(1L, serverStats1.getNumConnections()); + Assert.assertEquals(1L, serverStats2.getNumConnections()); Assert.assertEquals(2L, clientStats.getNumConnections()); Assert.assertEquals(0L, clientStats.getNumFailed()); @@ -125,6 +125,9 @@ public class AbstractServerTest extends TestLogger { Assert.assertEquals(2L, clientStats.getNumRequests()); } + Assert.assertEquals(0L, serverStats1.getNumConnections()); + Assert.assertEquals(0L, serverStats2.getNumConnections()); + Assert.assertEquals(0L, clientStats.getNumConnections()); Assert.assertEquals(0L, clientStats.getNumFailed()); Assert.assertEquals(2L, clientStats.getNumSuccessful()); @@ -184,10 +187,6 @@ public class AbstractServerTest extends TestLogger { @Override public void close() throws Exception { shutdownServer().get(); - if (requestStats instanceof AtomicKvStateRequestStats) { - AtomicKvStateRequestStats stats = (AtomicKvStateRequestStats) requestStats; - Assert.assertEquals(0L, stats.getNumConnections()); - } Assert.assertTrue(getQueryExecutor().isTerminated()); Assert.assertTrue(isEventGroupShutdown()); } diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml index e0f50f1e1c274eb7267ae3e39ffd847f40a50807..5fa6184da7fa0c076c54512d54fe75a4c1577845 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml @@ -171,6 +171,11 @@ under the License. + + + -nobootcp + + diff --git a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcher.java b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcher.java index fed220fe9528c7bfc7c3d55b26d39b7b26af8bc7..79add41fba5193542e2c7816262792e77d9e1e44 100644 --- a/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcher.java +++ b/flink-runtime-web/src/main/java/org/apache/flink/runtime/webmonitor/history/HistoryServerArchiveFetcher.java @@ -152,7 +152,6 @@ class HistoryServerArchiveFetcher { if (jobArchives == null) { continue; } - boolean updateOverview = false; int numFetchedArchives = 0; for (FileStatus jobArchive : jobArchives) { Path jobArchivePath = jobArchive.getPath(); @@ -164,7 +163,7 @@ class HistoryServerArchiveFetcher { refreshDir, jobID, iae); continue; } - if (cachedArchives.add(jobID)) { + if (!cachedArchives.contains(jobID)) { try { for (ArchivedJson archive : FsJobArchivist.getArchivedJsons(jobArchive.getPath())) { String path = archive.getPath(); @@ -177,6 +176,7 @@ class HistoryServerArchiveFetcher { json = convertLegacyJobOverview(json); target = new File(webOverviewDir, jobID + JSON_FILE_ENDING); } else { + // this implicitly writes into webJobDir target = new File(webDir, path + JSON_FILE_ENDING); } @@ -200,12 +200,10 @@ class HistoryServerArchiveFetcher { fw.flush(); } } - updateOverview = true; + cachedArchives.add(jobID); numFetchedArchives++; } catch (IOException e) { LOG.error("Failure while fetching/processing job archive for job {}.", jobID, e); - // Make sure we attempt to fetch the archive again - cachedArchives.remove(jobID); // Make sure we do not include this job in the overview try { Files.delete(new File(webOverviewDir, jobID + JSON_FILE_ENDING).toPath()); @@ -223,7 +221,7 @@ class HistoryServerArchiveFetcher { } } } - if (updateOverview) { + if (numFetchedArchives > 0) { updateJobOverview(webOverviewDir, webDir); for (int x = 0; x < numFetchedArchives; x++) { numArchivedJobs.countDown(); diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/LeaderRetrievalHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/LeaderRetrievalHandlerTest.java index d4ea516f4e4b609294dd8f48e9a99b6c4c85352a..67acfe4112a3a2cf9e6c105972f1a9478bc29d1a 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/LeaderRetrievalHandlerTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/LeaderRetrievalHandlerTest.java @@ -66,7 +66,7 @@ public class LeaderRetrievalHandlerTest extends TestLogger { final Time timeout = Time.seconds(10L); final CompletableFuture gatewayFuture = new CompletableFuture<>(); final GatewayRetriever gatewayRetriever = () -> gatewayFuture; - final RestfulGateway gateway = TestingRestfulGateway.newBuilder().build(); + final RestfulGateway gateway = new TestingRestfulGateway.Builder().build(); final TestingHandler testingHandler = new TestingHandler( gatewayRetriever, @@ -84,18 +84,18 @@ public class LeaderRetrievalHandlerTest extends TestLogger { try (HttpTestClient httpClient = new HttpTestClient("localhost", bootstrap.getServerPort())) { // 1. no leader gateway available --> Service unavailable - httpClient.sendGetRequest(restPath, FutureUtils.toFiniteDuration(timeout)); + httpClient.sendGetRequest(restPath, FutureUtils.toDuration(timeout)); - HttpTestClient.SimpleHttpResponse response = httpClient.getNextResponse(FutureUtils.toFiniteDuration(timeout)); + HttpTestClient.SimpleHttpResponse response = httpClient.getNextResponse(FutureUtils.toDuration(timeout)); Assert.assertEquals(HttpResponseStatus.SERVICE_UNAVAILABLE, response.getStatus()); // 2. with leader gatewayFuture.complete(gateway); - httpClient.sendGetRequest(restPath, FutureUtils.toFiniteDuration(timeout)); + httpClient.sendGetRequest(restPath, FutureUtils.toDuration(timeout)); - response = httpClient.getNextResponse(FutureUtils.toFiniteDuration(timeout)); + response = httpClient.getNextResponse(FutureUtils.toDuration(timeout)); Assert.assertEquals(HttpResponseStatus.OK, response.getStatus()); Assert.assertEquals(RESPONSE_MESSAGE, response.getContent()); diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebFrontendITCase.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebFrontendITCase.java index abeb9ea31a328cfdf449c542760363cf74ebeb35..1202499309c8ba9f83bfcf611be89ef7f931d096 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebFrontendITCase.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/WebFrontendITCase.java @@ -33,6 +33,7 @@ import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.runtime.webmonitor.testutils.HttpTestClient; import org.apache.flink.test.util.MiniClusterWithClientResource; import org.apache.flink.test.util.TestBaseUtils; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.TestLogger; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode; @@ -46,22 +47,21 @@ import org.junit.After; import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.io.File; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.nio.file.Files; +import java.time.Duration; +import java.time.LocalTime; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import scala.concurrent.duration.Deadline; -import scala.concurrent.duration.FiniteDuration; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -70,6 +70,7 @@ import static org.junit.Assert.fail; /** * Tests for the WebFrontend. */ +@Category(AlsoRunWithSchedulerNG.class) public class WebFrontendITCase extends TestLogger { private static final int NUM_TASK_MANAGERS = 2; @@ -104,7 +105,7 @@ public class WebFrontendITCase extends TestLogger { } // !!DO NOT REMOVE!! next line is required for tests - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "12m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "12m"); return config; } @@ -226,8 +227,8 @@ public class WebFrontendITCase extends TestLogger { String config = TestBaseUtils.getFromHTTP("http://localhost:" + getRestPort() + "/jobmanager/config"); Map conf = WebMonitorUtils.fromKeyValueJsonArray(config); - String expected = CLUSTER_CONFIGURATION.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE); - String actual = conf.get(TaskManagerOptions.MANAGED_MEMORY_SIZE.key()); + String expected = CLUSTER_CONFIGURATION.getString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE); + String actual = conf.get(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.key()); assertEquals(expected, actual); } @@ -257,13 +258,13 @@ public class WebFrontendITCase extends TestLogger { // wait for tasks to be properly running BlockingInvokable.latch.await(); - final FiniteDuration testTimeout = new FiniteDuration(2, TimeUnit.MINUTES); - final Deadline deadline = testTimeout.fromNow(); + final Duration testTimeout = Duration.ofMinutes(2); + final LocalTime deadline = LocalTime.now().plus(testTimeout); try (HttpTestClient client = new HttpTestClient("localhost", getRestPort())) { // cancel the job - client.sendPatchRequest("/jobs/" + jid + "/", deadline.timeLeft()); - HttpTestClient.SimpleHttpResponse response = client.getNextResponse(deadline.timeLeft()); + client.sendPatchRequest("/jobs/" + jid + "/", getTimeLeft(deadline)); + HttpTestClient.SimpleHttpResponse response = client.getNextResponse(getTimeLeft(deadline)); assertEquals(HttpResponseStatus.ACCEPTED, response.getStatus()); assertEquals("application/json; charset=UTF-8", response.getType()); @@ -277,7 +278,7 @@ public class WebFrontendITCase extends TestLogger { // ensure we can access job details when its finished (FLINK-4011) try (HttpTestClient client = new HttpTestClient("localhost", getRestPort())) { - FiniteDuration timeout = new FiniteDuration(30, TimeUnit.SECONDS); + Duration timeout = Duration.ofSeconds(30); client.sendGetRequest("/jobs/" + jid + "/config", timeout); HttpTestClient.SimpleHttpResponse response = client.getNextResponse(timeout); @@ -316,15 +317,14 @@ public class WebFrontendITCase extends TestLogger { // wait for tasks to be properly running BlockingInvokable.latch.await(); - final FiniteDuration testTimeout = new FiniteDuration(2, TimeUnit.MINUTES); - final Deadline deadline = testTimeout.fromNow(); + final Duration testTimeout = Duration.ofMinutes(2); + final LocalTime deadline = LocalTime.now().plus(testTimeout); try (HttpTestClient client = new HttpTestClient("localhost", getRestPort())) { // Request the file from the web server - client.sendGetRequest("/jobs/" + jid + "/yarn-cancel", deadline.timeLeft()); + client.sendGetRequest("/jobs/" + jid + "/yarn-cancel", getTimeLeft(deadline)); - HttpTestClient.SimpleHttpResponse response = client - .getNextResponse(deadline.timeLeft()); + HttpTestClient.SimpleHttpResponse response = client.getNextResponse(getTimeLeft(deadline)); assertEquals(HttpResponseStatus.ACCEPTED, response.getStatus()); assertEquals("application/json; charset=UTF-8", response.getType()); @@ -347,6 +347,10 @@ public class WebFrontendITCase extends TestLogger { .collect(Collectors.toList()); } + private static Duration getTimeLeft(LocalTime deadline) { + return Duration.between(LocalTime.now(), deadline); + } + /** * Test invokable that allows waiting for all subtasks to be running. */ diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandlerTest.java index fced9010ee5805903b4c46da101fa7e147a4966b..edbeca4bbe357d73d501f274012d8bc8a4e5f452 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandlerTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarDeleteHandlerTest.java @@ -70,7 +70,7 @@ public class JarDeleteHandlerTest extends TestLogger { @Before public void setUp() throws Exception { jarDir = temporaryFolder.newFolder().toPath(); - restfulGateway = TestingRestfulGateway.newBuilder().build(); + restfulGateway = new TestingRestfulGateway.Builder().build(); jarDeleteHandler = new JarDeleteHandler( () -> CompletableFuture.completedFuture(restfulGateway), Time.seconds(10), diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java index b4182c735c9b35b6380d7847bff2e9529849ccc8..1b306300c6c29f4c2da37a6f222d7d02c8f29a8e 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/handlers/JarRunHandlerTest.java @@ -28,11 +28,13 @@ import org.apache.flink.runtime.rest.util.RestClientException; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.runtime.testutils.MiniClusterResource; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.TestLogger; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.junit.rules.TemporaryFolder; import java.nio.file.Files; @@ -46,6 +48,7 @@ import static org.junit.Assert.assertTrue; /** * Tests for the {@link JarRunHandler}. */ +@Category(AlsoRunWithSchedulerNG.class) public class JarRunHandlerTest extends TestLogger { @ClassRule diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTest.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTest.java index fa932dc266ce32ed4e58c71a7f072acdfc194a9f..75113000379f5727d1dbabf3554821a3a5ecb51d 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTest.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/history/HistoryServerTest.java @@ -30,6 +30,7 @@ import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.DiscardingSink; import org.apache.flink.test.util.MiniClusterWithClientResource; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.TestLogger; import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonFactory; @@ -42,6 +43,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -63,6 +65,7 @@ import java.util.concurrent.TimeUnit; * Tests for the HistoryServer. */ @RunWith(Parameterized.class) +@Category(AlsoRunWithSchedulerNG.class) public class HistoryServerTest extends TestLogger { @ClassRule diff --git a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/testutils/HttpTestClient.java b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/testutils/HttpTestClient.java index d94f7a265e5d6332a63de499cba06c554ede2a6a..0d51cc4b08476584aefb9dd69e5890bf0a3be4a0 100644 --- a/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/testutils/HttpTestClient.java +++ b/flink-runtime-web/src/test/java/org/apache/flink/runtime/webmonitor/testutils/HttpTestClient.java @@ -47,13 +47,12 @@ import org.apache.flink.shaded.netty4.io.netty.util.CharsetUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import scala.concurrent.duration.FiniteDuration; - /** * A simple HTTP client. * @@ -130,7 +129,7 @@ public class HttpTestClient implements AutoCloseable { * * @param request The {@link HttpRequest} to send to the server */ - public void sendRequest(HttpRequest request, FiniteDuration timeout) throws InterruptedException, TimeoutException { + public void sendRequest(HttpRequest request, Duration timeout) throws InterruptedException, TimeoutException { LOG.debug("Writing {}.", request); // Make the connection attempt. @@ -153,7 +152,7 @@ public class HttpTestClient implements AutoCloseable { * * @param path The $path to GET (http://$host:$host/$path) */ - public void sendGetRequest(String path, FiniteDuration timeout) throws TimeoutException, InterruptedException { + public void sendGetRequest(String path, Duration timeout) throws TimeoutException, InterruptedException { if (!path.startsWith("/")) { path = "/" + path; } @@ -172,7 +171,7 @@ public class HttpTestClient implements AutoCloseable { * * @param path The $path to DELETE (http://$host:$host/$path) */ - public void sendDeleteRequest(String path, FiniteDuration timeout) throws TimeoutException, InterruptedException { + public void sendDeleteRequest(String path, Duration timeout) throws TimeoutException, InterruptedException { if (!path.startsWith("/")) { path = "/" + path; } @@ -191,7 +190,7 @@ public class HttpTestClient implements AutoCloseable { * * @param path The $path to PATCH (http://$host:$host/$path) */ - public void sendPatchRequest(String path, FiniteDuration timeout) throws TimeoutException, InterruptedException { + public void sendPatchRequest(String path, Duration timeout) throws TimeoutException, InterruptedException { if (!path.startsWith("/")) { path = "/" + path; } @@ -221,7 +220,7 @@ public class HttpTestClient implements AutoCloseable { * @param timeout Timeout in milliseconds for the next response to become available * @return The next available {@link SimpleHttpResponse} */ - public SimpleHttpResponse getNextResponse(FiniteDuration timeout) throws InterruptedException, + public SimpleHttpResponse getNextResponse(Duration timeout) throws InterruptedException, TimeoutException { SimpleHttpResponse response = responses.poll(timeout.toMillis(), TimeUnit.MILLISECONDS); diff --git a/flink-runtime-web/src/test/resources/rest_api_v1.snapshot b/flink-runtime-web/src/test/resources/rest_api_v1.snapshot index 629835d52797ecef2e469d852c4c7d7636cd0c35..65345ba20ded52ae4c73ea75828d61c4bf42b2d5 100644 --- a/flink-runtime-web/src/test/resources/rest_api_v1.snapshot +++ b/flink-runtime-web/src/test/resources/rest_api_v1.snapshot @@ -1637,7 +1637,7 @@ "type" : "array", "items" : { "type" : "object", - "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:JobVertexDetailsInfo:SubtaskExecutionAttemptDetailsInfo", + "id" : "urn:jsonschema:org:apache:flink:runtime:rest:messages:job:SubtaskExecutionAttemptDetailsInfo", "properties" : { "subtask" : { "type" : "integer" @@ -2317,6 +2317,9 @@ "additionalProperties" : { "type" : "integer" } + }, + "taskmanager-id" : { + "type" : "string" } } } diff --git a/flink-runtime-web/web-dashboard/package-lock.json b/flink-runtime-web/web-dashboard/package-lock.json index 42e95ae53744e769ec3f2a9af6f9093e85a00f0a..b9d70d0f6e6c3f0f9fc30c477c8f4e1ab65ea3e4 100644 --- a/flink-runtime-web/web-dashboard/package-lock.json +++ b/flink-runtime-web/web-dashboard/package-lock.json @@ -1572,6 +1572,7 @@ "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", "dev": true, + "optional": true, "requires": { "delegates": "^1.0.0", "readable-stream": "^2.0.6" @@ -2806,7 +2807,8 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", - "dev": true + "dev": true, + "optional": true }, "constants-browserify": { "version": "1.0.0", @@ -3508,7 +3510,8 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", - "dev": true + "dev": true, + "optional": true }, "depd": { "version": "1.1.2", @@ -4505,7 +4508,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -4526,12 +4530,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -4546,17 +4552,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -4673,7 +4682,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -4685,6 +4695,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -4699,6 +4710,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -4706,12 +4718,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.3.5", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -4730,6 +4744,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -4810,7 +4825,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -4822,6 +4838,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -4907,7 +4924,8 @@ "safe-buffer": { "version": "5.1.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -4943,6 +4961,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -4962,6 +4981,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5005,12 +5025,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, @@ -5019,6 +5041,7 @@ "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.11.tgz", "integrity": "sha1-XB+x8RdHcRTwYyoOtLcbPLD9MXE=", "dev": true, + "optional": true, "requires": { "graceful-fs": "^4.1.2", "inherits": "~2.0.0", @@ -5047,6 +5070,7 @@ "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", "dev": true, + "optional": true, "requires": { "aproba": "^1.0.3", "console-control-strings": "^1.0.0", @@ -5090,7 +5114,8 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", - "dev": true + "dev": true, + "optional": true }, "get-stream": { "version": "3.0.0", @@ -5300,7 +5325,8 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", - "dev": true + "dev": true, + "optional": true }, "has-value": { "version": "1.0.0", @@ -6252,7 +6278,8 @@ "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=", - "dev": true + "dev": true, + "optional": true }, "is-windows": { "version": "1.0.2", @@ -7135,6 +7162,7 @@ "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "dev": true, + "optional": true, "requires": { "graceful-fs": "^4.1.2", "parse-json": "^2.2.0", @@ -7147,7 +7175,8 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", - "dev": true + "dev": true, + "optional": true } } }, @@ -7520,7 +7549,8 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", - "dev": true + "dev": true, + "optional": true }, "map-visit": { "version": "1.0.0", @@ -8170,6 +8200,7 @@ "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", "dev": true, + "optional": true, "requires": { "are-we-there-yet": "~1.1.2", "console-control-strings": "~1.1.0", @@ -9217,6 +9248,7 @@ "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", "dev": true, + "optional": true, "requires": { "load-json-file": "^1.0.0", "normalize-package-data": "^2.3.2", @@ -9228,6 +9260,7 @@ "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", "dev": true, + "optional": true, "requires": { "graceful-fs": "^4.1.2", "pify": "^2.0.0", @@ -9238,7 +9271,8 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", - "dev": true + "dev": true, + "optional": true } } }, @@ -9247,6 +9281,7 @@ "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=", "dev": true, + "optional": true, "requires": { "find-up": "^1.0.0", "read-pkg": "^1.0.0" @@ -9257,6 +9292,7 @@ "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", "dev": true, + "optional": true, "requires": { "path-exists": "^2.0.0", "pinkie-promise": "^2.0.0" @@ -9267,6 +9303,7 @@ "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", "dev": true, + "optional": true, "requires": { "pinkie-promise": "^2.0.0" } @@ -10719,6 +10756,7 @@ "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", "dev": true, + "optional": true, "requires": { "is-utf8": "^0.2.0" } @@ -12139,6 +12177,7 @@ "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", "dev": true, + "optional": true, "requires": { "string-width": "^1.0.2 || 2" } diff --git a/flink-runtime-web/web-dashboard/src/app/interfaces/job-detail.ts b/flink-runtime-web/web-dashboard/src/app/interfaces/job-detail.ts index 3bf9c13e9b96f642cc603a8c6ca2e3160a78bb07..14291049b5cc830c456fa2c21d4c579bb186c23a 100644 --- a/flink-runtime-web/web-dashboard/src/app/interfaces/job-detail.ts +++ b/flink-runtime-web/web-dashboard/src/app/interfaces/job-detail.ts @@ -129,6 +129,7 @@ export interface NodesItemInterface { export interface NodesItemCorrectInterface extends NodesItemInterface { detail: VerticesItemInterface | undefined; + lowWatermark?: number; } export interface NodesItemLinkInterface { diff --git a/flink-runtime-web/web-dashboard/src/app/interfaces/job-vertex-task-manager.ts b/flink-runtime-web/web-dashboard/src/app/interfaces/job-vertex-task-manager.ts index 822e8373a602413aa17434e498b7a89b207b62f6..91f7e98615abe780484c4ebc7ac8a5cab84bc10c 100644 --- a/flink-runtime-web/web-dashboard/src/app/interfaces/job-vertex-task-manager.ts +++ b/flink-runtime-web/web-dashboard/src/app/interfaces/job-vertex-task-manager.ts @@ -50,4 +50,5 @@ export interface VertexTaskManagerDetailInterface { RUNNING: number; SCHEDULED: number; }; + 'taskmanager-id': string; } diff --git a/flink-runtime-web/web-dashboard/src/app/pages/job-manager/configuration/job-manager-configuration.component.ts b/flink-runtime-web/web-dashboard/src/app/pages/job-manager/configuration/job-manager-configuration.component.ts index e130ed2bddc3792a2e30b3fd3202a35d7f359a31..1dc88ee0aa790bc35ffb5c6d2e72544e4af0904a 100644 --- a/flink-runtime-web/web-dashboard/src/app/pages/job-manager/configuration/job-manager-configuration.component.ts +++ b/flink-runtime-web/web-dashboard/src/app/pages/job-manager/configuration/job-manager-configuration.component.ts @@ -32,7 +32,7 @@ export class JobManagerConfigurationComponent implements OnInit { ngOnInit() { this.jobManagerService.loadConfig().subscribe(data => { - this.listOfConfig = data; + this.listOfConfig = data.sort((pre, next) => (pre.key > next.key ? 1 : -1)); this.cdr.markForCheck(); }); } diff --git a/flink-runtime-web/web-dashboard/src/app/pages/job/overview/job-overview.component.ts b/flink-runtime-web/web-dashboard/src/app/pages/job/overview/job-overview.component.ts index 5174653f4499ea6f1d86912eaba7adef5a524f8d..a0765c43df6a80ff95bd8db659da8e83d660946d 100644 --- a/flink-runtime-web/web-dashboard/src/app/pages/job/overview/job-overview.component.ts +++ b/flink-runtime-web/web-dashboard/src/app/pages/job/overview/job-overview.component.ts @@ -26,10 +26,11 @@ import { ViewChild } from '@angular/core'; import { ActivatedRoute, Router } from '@angular/router'; -import { Subject } from 'rxjs'; -import { filter, takeUntil } from 'rxjs/operators'; +import { LONG_MIN_VALUE } from 'config'; +import { forkJoin, Observable, of, Subject } from 'rxjs'; +import { catchError, filter, map, takeUntil } from 'rxjs/operators'; import { NodesItemCorrectInterface, NodesItemLinkInterface } from 'interfaces'; -import { JobService } from 'services'; +import { JobService, MetricsService } from 'services'; import { DagreComponent } from 'share/common/dagre/dagre.component'; @Component({ @@ -62,11 +63,52 @@ export class JobOverviewComponent implements OnInit, OnDestroy { } } + mergeWithWatermarks(nodes: NodesItemCorrectInterface[]): Observable { + return forkJoin( + nodes.map(node => { + const listOfMetricId = []; + let lowWatermark = NaN; + for (let i = 0; i < node.parallelism; i++) { + listOfMetricId.push(`${i}.currentInputWatermark`); + } + return this.metricService.getMetrics(this.jobId, node.id, listOfMetricId).pipe( + map(metrics => { + let minValue = NaN; + const watermarks: { [index: string]: number } = {}; + for (const key in metrics.values) { + const value = metrics.values[key]; + const subtaskIndex = key.replace('.currentInputWatermark', ''); + watermarks[subtaskIndex] = value; + if (isNaN(minValue) || value < minValue) { + minValue = value; + } + } + if (!isNaN(minValue) && minValue > LONG_MIN_VALUE) { + lowWatermark = minValue; + } else { + lowWatermark = NaN; + } + return { ...node, lowWatermark }; + }) + ); + }) + ).pipe(catchError(() => of(nodes))); + } + + refreshNodesWithWatermarks() { + this.mergeWithWatermarks(this.nodes).subscribe(nodes => { + nodes.forEach(node => { + this.dagreComponent.updateNode(node.id, node); + }); + }); + } + constructor( private jobService: JobService, private router: Router, private activatedRoute: ActivatedRoute, public elementRef: ElementRef, + private metricService: MetricsService, private cdr: ChangeDetectorRef ) {} @@ -82,11 +124,10 @@ export class JobOverviewComponent implements OnInit, OnDestroy { this.links = data.plan.links; this.jobId = data.plan.jid; this.dagreComponent.flush(this.nodes, this.links, true).then(); + this.refreshNodesWithWatermarks(); } else { this.nodes = data.plan.nodes; - this.nodes.forEach(node => { - this.dagreComponent.updateNode(node.id, node); - }); + this.refreshNodesWithWatermarks(); } this.cdr.markForCheck(); }); diff --git a/flink-runtime-web/web-dashboard/src/app/pages/job/overview/list/job-overview-list.component.html b/flink-runtime-web/web-dashboard/src/app/pages/job/overview/list/job-overview-list.component.html index 8e2debe32a7f5dd985a1523becb01338d654b829..f6d73990585d00634514f3c5bfb029ca39b90bbf 100644 --- a/flink-runtime-web/web-dashboard/src/app/pages/job/overview/list/job-overview-list.component.html +++ b/flink-runtime-web/web-dashboard/src/app/pages/job/overview/list/job-overview-list.component.html @@ -20,7 +20,7 @@ class="no-border small" [nzSize]="'small'" [nzData]="nodes" - [nzScroll]="{x:((1310+left)+'px')}" + [nzScroll]="{x:((1360+left)+'px')}" [nzFrontPagination]="false" [nzShowPagination]="false"> @@ -33,7 +33,7 @@ Records Sent Parallelism Start Time - Duration + Duration End Time Tasks diff --git a/flink-runtime-web/web-dashboard/src/app/pages/job/overview/list/job-overview-list.component.ts b/flink-runtime-web/web-dashboard/src/app/pages/job/overview/list/job-overview-list.component.ts index e7cf66bfe6c9bec220ea9a0c6b3d25a2698ee4cc..c1ceaef58393dd897100e0f84a86f02ed70996e1 100644 --- a/flink-runtime-web/web-dashboard/src/app/pages/job/overview/list/job-overview-list.component.ts +++ b/flink-runtime-web/web-dashboard/src/app/pages/job/overview/list/job-overview-list.component.ts @@ -28,8 +28,8 @@ import { NodesItemCorrectInterface } from 'interfaces'; }) export class JobOverviewListComponent { innerNodes: NodesItemCorrectInterface[] = []; - sortName = 'detail.topology-id'; - sortValue = 'ascend'; + sortName: string; + sortValue: string; left = 390; @Output() nodeClick = new EventEmitter(); @Input() selectedNode: NodesItemCorrectInterface; diff --git a/flink-runtime-web/web-dashboard/src/app/pages/job/overview/taskmanagers/job-overview-drawer-taskmanagers.component.html b/flink-runtime-web/web-dashboard/src/app/pages/job/overview/taskmanagers/job-overview-drawer-taskmanagers.component.html index a3fc5bbc86b09aa55915312fff2c9ad2bce60a61..2345c85fd6561c9c6e7759458036e9a9cc31ada9 100644 --- a/flink-runtime-web/web-dashboard/src/app/pages/job/overview/taskmanagers/job-overview-drawer-taskmanagers.component.html +++ b/flink-runtime-web/web-dashboard/src/app/pages/job/overview/taskmanagers/job-overview-drawer-taskmanagers.component.html @@ -21,12 +21,13 @@ [nzSize]="'small'" [nzLoading]="isLoading" [nzData]="listOfTaskManager" - [nzScroll]="{x:'1450px',y:'calc( 100% - 35px )'}" + [nzScroll]="{x:'1500px',y:'calc( 100% - 35px )'}" [nzFrontPagination]="false" [nzShowPagination]="false"> Host + LOG Bytes received Records received Bytes sent @@ -41,6 +42,12 @@ {{ taskManager.host }} + + - + + LOG + + {{ taskManager.metrics['read-bytes'] | humanizeBytes }} diff --git a/flink-runtime-web/web-dashboard/src/app/services/job.service.ts b/flink-runtime-web/web-dashboard/src/app/services/job.service.ts index 4da2b703428ace94813f2bf997e73f188c3bde20..4abaa3f04dbc29cebba2519c28007b306fa3a9d6 100644 --- a/flink-runtime-web/web-dashboard/src/app/services/job.service.ts +++ b/flink-runtime-web/web-dashboard/src/app/services/job.service.ts @@ -264,6 +264,8 @@ export class JobService { }); } }); + const listOfVerticesId = job.vertices.map(item => item.id); + nodes.sort((pre, next) => listOfVerticesId.indexOf(pre.id) - listOfVerticesId.indexOf(next.id)); } return { ...job, diff --git a/flink-runtime-web/web-dashboard/src/app/services/metrics.service.ts b/flink-runtime-web/web-dashboard/src/app/services/metrics.service.ts index 081aa4efa298ed609133566a44731ed0b0091b3d..d13b3a674a425e36bfb795fc96e93e71d0906a96 100644 --- a/flink-runtime-web/web-dashboard/src/app/services/metrics.service.ts +++ b/flink-runtime-web/web-dashboard/src/app/services/metrics.service.ts @@ -33,9 +33,23 @@ export class MetricsService { * @param vertexId */ getAllAvailableMetrics(jobId: string, vertexId: string) { - return this.httpClient.get>( - `${BASE_URL}/jobs/${jobId}/vertices/${vertexId}/metrics` - ); + return this.httpClient + .get>(`${BASE_URL}/jobs/${jobId}/vertices/${vertexId}/metrics`) + .pipe( + map(item => + item.sort((pre, next) => { + const preId = pre.id.toLowerCase(); + const nextId = next.id.toLowerCase(); + if (preId < nextId) { + return -1; + } else if (preId > nextId) { + return 1; + } else { + return 0; + } + }) + ) + ); } /** diff --git a/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.html b/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.html index 799ad2b0dcc6ae1d5846283493d4518d38aa55e5..3b57b46bb88165c96b4ad4c10503b89c130e4f8a 100644 --- a/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.html +++ b/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.html @@ -23,6 +23,7 @@ {{operator}} {{description}} Parallelism: {{parallelism}} + Low Watermark {{lowWatermark}} Operation: {{operatorStrategy}} diff --git a/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.less b/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.less index 0f10e2d7dacd3b67d91cd21b1434735d51b01ebc..0533e053ac33286ebd75ae4154eb7bbd9da23e20 100644 --- a/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.less +++ b/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.less @@ -50,12 +50,17 @@ &:first-child { margin-top: 24px; } + + &.watermark { + font-weight: normal;; + font-weight: 12px; + color: @text-color-secondary; + } } .detail { margin-bottom: 12px; color: @text-color; - &.description { color: @heading-color; } diff --git a/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.ts b/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.ts index 2f96b1215106376ec2722f19db569016a2cb7fd4..f24ef2411b46d710689b6654e235afe790d4a733 100644 --- a/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.ts +++ b/flink-runtime-web/web-dashboard/src/app/share/common/dagre/node.component.ts @@ -30,6 +30,7 @@ export class NodeComponent { operator: string | null; operatorStrategy: string | null; parallelism: number | null; + lowWatermark: number | null | undefined; height = 0; id: string; @@ -45,6 +46,7 @@ export class NodeComponent { this.operator = this.decodeHTML(value.operator); this.operatorStrategy = this.decodeHTML(value.operator_strategy); this.parallelism = value.parallelism; + this.lowWatermark = value.lowWatermark; this.height = value.height || 0; this.id = value.id; if (description && description.length > 300) { diff --git a/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.html b/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.html index f20258eed224bc573acada4c80d908ef32bd4587..4ab72d6ba244ba1c84626174875628f02deab0d3 100644 --- a/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.html +++ b/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.html @@ -21,13 +21,20 @@
{{title}}
- - + +
-
+
+
{{latestValue | humanizeChartNumeric:title}}
+
+ + + + +
diff --git a/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.less b/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.less index 4cfe665b43afad0f2e6707b04f6daa7cb610a90f..100027c336f6091d6cbd6e5ab0ce880c34525929 100644 --- a/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.less +++ b/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.less @@ -59,6 +59,12 @@ .content { padding: 12px; + + .numeric { + line-height: 150px; + font-size: 32px; + text-align: center; + } } button { diff --git a/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.ts b/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.ts index 2e33c0d69e110ffa3adff950610753db287188ab..e5b716c3f8a8abf11631904f81455d19ebbf8387 100644 --- a/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.ts +++ b/flink-runtime-web/web-dashboard/src/app/share/customize/job-chart/job-chart.component.ts @@ -45,8 +45,10 @@ export class JobChartComponent implements AfterViewInit, OnDestroy { @Output() closed = new EventEmitter(); @ViewChild('chart') chart: ElementRef; size = 'small'; + displayMode: 'chart' | 'numeric' = 'chart'; chartInstance: Chart; data: Array<{ time: number; value: number; type: string }> = []; + latestValue: number; @HostBinding('class.big') get isBig() { @@ -54,9 +56,13 @@ export class JobChartComponent implements AfterViewInit, OnDestroy { } refresh(res: { timestamp: number; values: { [id: string]: number } }) { + this.latestValue = res.values[this.title]; + if (this.displayMode === 'numeric') { + this.cdr.detectChanges(); + } this.data.push({ time: res.timestamp, - value: res.values[this.title], + value: this.latestValue, type: this.title }); @@ -68,6 +74,11 @@ export class JobChartComponent implements AfterViewInit, OnDestroy { } } + setMode(mode: 'chart' | 'numeric') { + this.displayMode = mode; + this.cdr.detectChanges(); + } + resize(size: string) { this.size = size; this.cdr.detectChanges(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ServicesThreadFactory.java b/flink-runtime-web/web-dashboard/src/app/share/pipes/humanize-chart-numeric.pipe.ts similarity index 46% rename from flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ServicesThreadFactory.java rename to flink-runtime-web/web-dashboard/src/app/share/pipes/humanize-chart-numeric.pipe.ts index 24667e42527de4e06da6273eae7ff17abe379496..4ad8b9249600d20cea1ffa3e07edfe21107f2863 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/ServicesThreadFactory.java +++ b/flink-runtime-web/web-dashboard/src/app/share/pipes/humanize-chart-numeric.pipe.ts @@ -16,25 +16,31 @@ * limitations under the License. */ -package org.apache.flink.runtime.highavailability; +import { Pipe, PipeTransform } from '@angular/core'; +import { isNil } from 'utils'; +import { HumanizeBytesPipe } from './humanize-bytes.pipe'; +import { HumanizeDurationPipe } from './humanize-duration.pipe'; -import javax.annotation.Nonnull; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; - -public class ServicesThreadFactory implements ThreadFactory { - - private AtomicInteger enumerator = new AtomicInteger(); - - @Override - public Thread newThread(@Nonnull Runnable r) { - Thread thread = new Thread(r, "Flink HA Services Thread #" + enumerator.incrementAndGet()); - - // HA threads should have a very high priority, but not - // keep the JVM running by themselves - thread.setPriority(Thread.MAX_PRIORITY); - thread.setDaemon(true); - - return thread; - } +@Pipe({ + name: 'humanizeChartNumeric' +}) +export class HumanizeChartNumericPipe implements PipeTransform { + transform(value: number, id: string): string { + if (isNil(value)) { + return '-'; + } + let returnVal = ''; + if (/bytes/i.test(id) && /persecond/i.test(id)) { + returnVal = new HumanizeBytesPipe().transform(value) + ' / s'; + } else if (/bytes/i.test(id)) { + returnVal = new HumanizeBytesPipe().transform(value); + } else if (/persecond/i.test(id)) { + returnVal = value + ' / s'; + } else if (/time/i.test(id) || /latency/i.test(id)) { + returnVal = new HumanizeDurationPipe().transform(value, true); + } else { + returnVal = `${value}`; + } + return returnVal; + } } diff --git a/flink-runtime-web/web-dashboard/src/app/share/pipes/humanize-duration.pipe.ts b/flink-runtime-web/web-dashboard/src/app/share/pipes/humanize-duration.pipe.ts index 7440ddf1e65951fa87df55d79b063aff13e18858..3cf8043055c65815b7f402333d6c7c16892e0ab6 100644 --- a/flink-runtime-web/web-dashboard/src/app/share/pipes/humanize-duration.pipe.ts +++ b/flink-runtime-web/web-dashboard/src/app/share/pipes/humanize-duration.pipe.ts @@ -23,7 +23,7 @@ import { isNil } from 'utils'; name: 'humanizeDuration' }) export class HumanizeDurationPipe implements PipeTransform { - transform(value: number): any { + transform(value: number, short: boolean = false): any { if (isNil(value)) { return '-'; } else if (value < 0) { @@ -50,10 +50,18 @@ export class HumanizeDurationPipe implements PipeTransform { return `${minutes}m ${seconds}s`; } } else { - return `${hours}h ${minutes}m`; + if (short) { + return `${hours}h ${minutes}m`; + } else { + return `${hours}h ${minutes}m ${seconds}s`; + } } } else { - return `${days}d ${hours}h ${minutes}m`; + if (short) { + return `${days}d ${hours}h`; + } else { + return `${days}d ${hours}h ${minutes}m ${seconds}s`; + } } } } diff --git a/flink-runtime-web/web-dashboard/src/app/share/pipes/pipe.module.ts b/flink-runtime-web/web-dashboard/src/app/share/pipes/pipe.module.ts index 185c8205532aec9e4694224a02a3f32a6c86f5a5..8afe7cc66795818bf69ebb51eb17f83a1cefd8ff 100644 --- a/flink-runtime-web/web-dashboard/src/app/share/pipes/pipe.module.ts +++ b/flink-runtime-web/web-dashboard/src/app/share/pipes/pipe.module.ts @@ -22,10 +22,17 @@ import { HumanizeBytesPipe } from 'share/pipes/humanize-bytes.pipe'; import { HumanizeWatermarkPipe } from 'share/pipes/humanize-watermark.pipe'; import { HumanizeDurationPipe } from './humanize-duration.pipe'; import { HumanizeDatePipe } from './humanize-date.pipe'; +import { HumanizeChartNumericPipe } from './humanize-chart-numeric.pipe'; @NgModule({ imports: [CommonModule], - declarations: [HumanizeDurationPipe, HumanizeDatePipe, HumanizeBytesPipe, HumanizeWatermarkPipe], - exports: [HumanizeDurationPipe, HumanizeDatePipe, HumanizeBytesPipe, HumanizeWatermarkPipe] + declarations: [ + HumanizeDurationPipe, + HumanizeDatePipe, + HumanizeBytesPipe, + HumanizeWatermarkPipe, + HumanizeChartNumericPipe + ], + exports: [HumanizeDurationPipe, HumanizeDatePipe, HumanizeBytesPipe, HumanizeWatermarkPipe, HumanizeChartNumericPipe] }) export class PipeModule {} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointException.java index c0bc2d1573555fd0c2c7ffe1eb6f9db6a07865f3..7c8ab49bfd169afcb9472c6f8ccf96a81e8cdefb 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointException.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CheckpointException.java @@ -19,6 +19,7 @@ package org.apache.flink.runtime.checkpoint; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.SerializedThrowable; /** * Base class for checkpoint related exceptions. @@ -40,12 +41,20 @@ public class CheckpointException extends Exception { } public CheckpointException(CheckpointFailureReason failureReason, Throwable cause) { - super(failureReason.message(), cause); + // Defensively replace the cause with a SerializedThrowable in case it's a user-defined exception + // that doesn't exist on the JobManager's default classpath. + super( + failureReason.message(), + cause == null ? null : new SerializedThrowable(cause)); this.checkpointFailureReason = Preconditions.checkNotNull(failureReason); } public CheckpointException(String message, CheckpointFailureReason failureReason, Throwable cause) { - super(message + " Failure reason: " + failureReason.message(), cause); + // Defensively replace the cause with a SerializedThrowable in case it's a user-defined exception + // that doesn't exist on the JobManager's default classpath. + super( + message + " Failure reason: " + failureReason.message(), + cause == null ? null : new SerializedThrowable(cause)); this.checkpointFailureReason = Preconditions.checkNotNull(failureReason); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java index 1cda1313708a2f355715a06e26ea9158cf419f88..9ed3151742debf7b6590ca6d76f32b3b379a3c84 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/checkpoint/CompletedCheckpointStore.java @@ -19,7 +19,6 @@ package org.apache.flink.runtime.checkpoint; import org.apache.flink.runtime.jobgraph.JobStatus; -import org.apache.flink.util.FlinkRuntimeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,31 +54,26 @@ public interface CompletedCheckpointStore { * added. */ default CompletedCheckpoint getLatestCheckpoint(boolean isPreferCheckpointForRecovery) throws Exception { - if (getAllCheckpoints().isEmpty()) { + List allCheckpoints = getAllCheckpoints(); + if (allCheckpoints.isEmpty()) { return null; } - CompletedCheckpoint candidate = getAllCheckpoints().get(getAllCheckpoints().size() - 1); - if (isPreferCheckpointForRecovery && getAllCheckpoints().size() > 1) { - List allCheckpoints; - try { - allCheckpoints = getAllCheckpoints(); - ListIterator listIterator = allCheckpoints.listIterator(allCheckpoints.size() - 1); - while (listIterator.hasPrevious()) { - CompletedCheckpoint prev = listIterator.previous(); - if (!prev.getProperties().isSavepoint()) { - candidate = prev; - LOG.info("Found a completed checkpoint before the latest savepoint, will use it to recover!"); - break; - } + CompletedCheckpoint lastCompleted = allCheckpoints.get(allCheckpoints.size() - 1); + + if (isPreferCheckpointForRecovery && allCheckpoints.size() > 1 && lastCompleted.getProperties().isSavepoint()) { + ListIterator listIterator = allCheckpoints.listIterator(allCheckpoints.size() - 1); + while (listIterator.hasPrevious()) { + CompletedCheckpoint prev = listIterator.previous(); + if (!prev.getProperties().isSavepoint()) { + LOG.info("Found a completed checkpoint ({}) before the latest savepoint, will use it to recover!", prev); + return prev; } - } catch (Exception e) { - LOG.error("Method getAllCheckpoints caused exception : ", e); - throw new FlinkRuntimeException(e); } + LOG.info("Did not find earlier checkpoint, using latest savepoint to recover."); } - return candidate; + return lastCompleted; } /** diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlBytesType.java b/flink-runtime/src/main/java/org/apache/flink/runtime/client/DuplicateJobSubmissionException.java similarity index 58% rename from flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlBytesType.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/client/DuplicateJobSubmissionException.java index dfc2d1fdee5dfc16a2644479b26ef661ac7cdd1d..6399aebabb2bf97f783ca31939ad6fe7e3942201 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlBytesType.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/client/DuplicateJobSubmissionException.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,23 +16,17 @@ * limitations under the License. */ -package org.apache.flink.sql.parser.type; +package org.apache.flink.runtime.client; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlWriter; -import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.flink.api.common.JobID; /** - * Parse type "BYTES" which is a synonym of VARBINARY(INT_MAX). + * Exception which is returned upon job submission if the submitted job + * is currently being executed. */ -public class SqlBytesType extends SqlIdentifier implements ExtendedSqlType { +public class DuplicateJobSubmissionException extends JobSubmissionException { - public SqlBytesType(SqlParserPos pos) { - super("BYTES", pos); - } - - @Override - public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - writer.keyword("BYTES"); + public DuplicateJobSubmissionException(JobID jobID) { + super(jobID, "Job has already been submitted."); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java index da226718f4c88ca9d222aad9d74ebeaa3fef2365..75de58178d837ef60af80f55eb55221bdabee050 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/BootstrapTools.java @@ -23,8 +23,6 @@ import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.CoreOptions; -import org.apache.flink.configuration.JobManagerOptions; -import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.runtime.akka.AkkaUtils; import org.apache.flink.util.NetUtils; @@ -53,7 +51,6 @@ import java.util.Map; import scala.Some; import scala.Tuple2; -import scala.concurrent.duration.FiniteDuration; import static org.apache.flink.configuration.ConfigOptions.key; @@ -268,40 +265,6 @@ public class BootstrapTools { } } - /** - * Generate a task manager configuration. - * @param baseConfig Config to start from. - * @param jobManagerHostname Job manager host name. - * @param jobManagerPort Port of the job manager. - * @param numSlots Number of slots to configure. - * @param registrationTimeout Timeout for registration - * @return TaskManager configuration - */ - public static Configuration generateTaskManagerConfiguration( - Configuration baseConfig, - String jobManagerHostname, - int jobManagerPort, - int numSlots, - FiniteDuration registrationTimeout) { - - Configuration cfg = cloneConfiguration(baseConfig); - - if (jobManagerHostname != null && !jobManagerHostname.isEmpty()) { - cfg.setString(JobManagerOptions.ADDRESS, jobManagerHostname); - } - - if (jobManagerPort > 0) { - cfg.setInteger(JobManagerOptions.PORT, jobManagerPort); - } - - cfg.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, registrationTimeout.toString()); - if (numSlots != -1){ - cfg.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlots); - } - - return cfg; - } - /** * Writes a Flink YAML config file from a Flink Configuration object. * @param cfg The Flink config diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/TaskExecutorResourceSpec.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/TaskExecutorResourceSpec.java new file mode 100644 index 0000000000000000000000000000000000000000..0813aead4cb1cdf39682a2bf3d55e5d8dd9a3823 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/TaskExecutorResourceSpec.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.clusterframework; + +import org.apache.flink.configuration.MemorySize; + +/** + * Describe the specifics of different resource dimensions of the TaskExecutor. + * + *

A TaskExecutor's memory consists of the following components. + *

    + *
  • Framework Heap Memory
    • + *
  • Task Heap Memory
    • + *
  • Task Off-Heap Memory
    • + *
  • Shuffle Memory
    • + *
  • Managed Memory
    • + *
      + *
    • On-Heap Managed Memory
      • + *
    • Off-Heap Managed Memory
      • + *
    + *
  • JVM Metaspace
    • + *
  • JVM Overhead
    • + *
+ * Among all the components, Framework Heap Memory, Task Heap Memory and On-Heap Managed Memory use on heap memory, + * while the rest use off heap memory. We use Total Process Memory to refer to all the memory components, while Total + * Flink Memory refering to all the components except JVM Metaspace and JVM Overhead. + * + *

The relationships of TaskExecutor memory components are shown below. + * 

+ *               ┌ ─ ─ Total Process Memory  ─ ─ ┐
+ *                ┌ ─ ─ Total Flink Memory  ─ ─ ┐
+ *               │ ┌───────────────────────────┐ │
+ *                ││   Framework Heap Memory   ││  ─┐
+ *               │ └───────────────────────────┘ │  │
+ *                │┌───────────────────────────┐│   │
+ *               │ │     Task Heap Memory      │ │ ─┤
+ *                │└───────────────────────────┘│   │
+ *               │ ┌───────────────────────────┐ │  │
+ *            ┌─  ││   Task Off-Heap Memory    ││   │
+ *            │  │ └───────────────────────────┘ │  ├─ On-Heap
+ *            │   │┌───────────────────────────┐│   │
+ *            ├─ │ │      Shuffle Memory       │ │  │
+ *            │   │└───────────────────────────┘│   │
+ *            │  │ ┌───── Managed Memory ──────┐ │  │
+ *            │   ││┌─────────────────────────┐││   │
+ *            │  │ ││ On-Heap Managed Memory  ││ │ ─┘
+ *            │   ││├─────────────────────────┤││
+ *  Off-Heap ─┼─ │ ││ Off-Heap Managed Memory ││ │
+ *            │   ││└─────────────────────────┘││
+ *            │  │ └───────────────────────────┘ │
+ *            │   └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘
+ *            │  │┌─────────────────────────────┐│
+ *            ├─  │        JVM Metaspace        │
+ *            │  │└─────────────────────────────┘│
+ *            │   ┌─────────────────────────────┐
+ *            └─ ││        JVM Overhead         ││
+ *                └─────────────────────────────┘
+ *               └ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘
+ *
+ */ +public class TaskExecutorResourceSpec { + + private final MemorySize frameworkHeapSize; + + private final MemorySize taskHeapSize; + + private final MemorySize taskOffHeapSize; + + private final MemorySize shuffleMemSize; + + private final MemorySize onHeapManagedMemorySize; + + private final MemorySize offHeapManagedMemorySize; + + private final MemorySize jvmMetaspaceSize; + + private final MemorySize jvmOverheadSize; + + public TaskExecutorResourceSpec( + MemorySize frameworkHeapSize, + MemorySize taskHeapSize, + MemorySize taskOffHeapSize, + MemorySize shuffleMemSize, + MemorySize onHeapManagedMemorySize, + MemorySize offHeapManagedMemorySize, + MemorySize jvmMetaspaceSize, + MemorySize jvmOverheadSize) { + + this.frameworkHeapSize = frameworkHeapSize; + this.taskHeapSize = taskHeapSize; + this.taskOffHeapSize = taskOffHeapSize; + this.shuffleMemSize = shuffleMemSize; + this.onHeapManagedMemorySize = onHeapManagedMemorySize; + this.offHeapManagedMemorySize = offHeapManagedMemorySize; + this.jvmMetaspaceSize = jvmMetaspaceSize; + this.jvmOverheadSize = jvmOverheadSize; + } + + public MemorySize getFrameworkHeapSize() { + return frameworkHeapSize; + } + + public MemorySize getTaskHeapSize() { + return taskHeapSize; + } + + public MemorySize getTaskOffHeapSize() { + return taskOffHeapSize; + } + + public MemorySize getShuffleMemSize() { + return shuffleMemSize; + } + + public MemorySize getOnHeapManagedMemorySize() { + return onHeapManagedMemorySize; + } + + public MemorySize getOffHeapManagedMemorySize() { + return offHeapManagedMemorySize; + } + + public MemorySize getManagedMemorySize() { + return onHeapManagedMemorySize.add(offHeapManagedMemorySize); + } + + public MemorySize getJvmMetaspaceSize() { + return jvmMetaspaceSize; + } + + public MemorySize getJvmOverheadSize() { + return jvmOverheadSize; + } + + public MemorySize getTotalFlinkMemorySize() { + return frameworkHeapSize.add(taskHeapSize).add(taskOffHeapSize).add(shuffleMemSize).add(getManagedMemorySize()); + } + + public MemorySize getTotalProcessMemorySize() { + return getTotalFlinkMemorySize().add(jvmMetaspaceSize).add(jvmOverheadSize); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/TaskExecutorResourceUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/TaskExecutorResourceUtils.java new file mode 100644 index 0000000000000000000000000000000000000000..ee13943a78a1f8834e7d7f87cdb2895c67b39a0f --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/clusterframework/TaskExecutorResourceUtils.java @@ -0,0 +1,610 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.clusterframework; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.IllegalConfigurationException; +import org.apache.flink.configuration.MemorySize; +import org.apache.flink.configuration.NettyShuffleEnvironmentOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.runtime.util.ConfigurationParserUtils; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Utility class for TaskExecutor memory configurations. + * + *

See {@link TaskExecutorResourceSpec} for details about memory components of TaskExecutor and their relationships. + */ +public class TaskExecutorResourceUtils { + + private TaskExecutorResourceUtils() {} + + // ------------------------------------------------------------------------ + // Generating JVM Parameters + // ------------------------------------------------------------------------ + + public static String generateJvmParametersStr(final TaskExecutorResourceSpec taskExecutorResourceSpec) { + final MemorySize jvmHeapSize = taskExecutorResourceSpec.getFrameworkHeapSize() + .add(taskExecutorResourceSpec.getTaskHeapSize()) + .add(taskExecutorResourceSpec.getOnHeapManagedMemorySize()); + final MemorySize jvmDirectSize = taskExecutorResourceSpec.getTaskOffHeapSize() + .add(taskExecutorResourceSpec.getShuffleMemSize()); + final MemorySize jvmMetaspaceSize = taskExecutorResourceSpec.getJvmMetaspaceSize(); + + return "-Xmx" + jvmHeapSize.getBytes() + + " -Xms" + jvmHeapSize.getBytes() + + " -XX:MaxDirectMemorySize=" + jvmDirectSize.getBytes() + + " -XX:MaxMetaspaceSize=" + jvmMetaspaceSize.getBytes(); + } + + // ------------------------------------------------------------------------ + // Generating Dynamic Config Options + // ------------------------------------------------------------------------ + + public static String generateDynamicConfigsStr(final TaskExecutorResourceSpec taskExecutorResourceSpec) { + final Map configs = new HashMap<>(); + configs.put(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY.key(), taskExecutorResourceSpec.getFrameworkHeapSize().getBytes() + "b"); + configs.put(TaskManagerOptions.TASK_HEAP_MEMORY.key(), taskExecutorResourceSpec.getTaskHeapSize().getBytes() + "b"); + configs.put(TaskManagerOptions.TASK_OFF_HEAP_MEMORY.key(), taskExecutorResourceSpec.getTaskOffHeapSize().getBytes() + "b"); + configs.put(TaskManagerOptions.SHUFFLE_MEMORY_MIN.key(), taskExecutorResourceSpec.getShuffleMemSize().getBytes() + "b"); + configs.put(TaskManagerOptions.SHUFFLE_MEMORY_MAX.key(), taskExecutorResourceSpec.getShuffleMemSize().getBytes() + "b"); + configs.put(TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), taskExecutorResourceSpec.getManagedMemorySize().getBytes() + "b"); + configs.put(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_SIZE.key(), taskExecutorResourceSpec.getOffHeapManagedMemorySize().getBytes() + "b"); + return assembleDynamicConfigsStr(configs); + } + + private static String assembleDynamicConfigsStr(final Map configs) { + final StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : configs.entrySet()) { + sb.append("-D ").append(entry.getKey()).append("=").append(entry.getValue()).append(" "); + } + return sb.toString(); + } + + // ------------------------------------------------------------------------ + // Memory Configuration Calculations + // ------------------------------------------------------------------------ + + public static TaskExecutorResourceSpec resourceSpecFromConfig(final Configuration config) { + if (isTaskHeapMemorySizeExplicitlyConfigured(config) && isManagedMemorySizeExplicitlyConfigured(config)) { + // both task heap memory and managed memory are configured, use these to derive total flink memory + return deriveResourceSpecWithExplicitTaskAndManagedMemory(config); + } else if (isTotalFlinkMemorySizeExplicitlyConfigured(config)) { + // either of task heap memory and managed memory is not configured, total flink memory is configured, + // derive from total flink memory + return deriveResourceSpecWithTotalFlinkMemory(config); + } else if (isTotalProcessMemorySizeExplicitlyConfigured(config)) { + // total flink memory is not configured, total process memory is configured, + // derive from total process memory + return deriveResourceSpecWithTotalProcessMemory(config); + } else { + throw new IllegalConfigurationException("Either Task Heap Memory size and Managed Memory size, or Total Flink" + + " Memory size, or Total Process Memory size need to be configured explicitly."); + } + } + + private static TaskExecutorResourceSpec deriveResourceSpecWithExplicitTaskAndManagedMemory(final Configuration config) { + // derive flink internal memory from explicitly configure task heap memory size and managed memory size + + final MemorySize taskHeapMemorySize = getTaskHeapMemorySize(config); + final MemorySize managedMemorySize = getManagedMemorySize(config); + + final MemorySize frameworkHeapMemorySize = getFrameworkHeapMemorySize(config); + final MemorySize taskOffHeapMemorySize = getTaskOffHeapMemorySize(config); + + final OnHeapAndOffHeapManagedMemory onHeapAndOffHeapManagedMemory = deriveOnHeapAndOffHeapMemoryFromManagedMemory(config, managedMemorySize); + + final MemorySize shuffleMemorySize; + final MemorySize totalFlinkExcludeShuffleMemorySize = + frameworkHeapMemorySize.add(taskHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize); + + if (isTotalFlinkMemorySizeExplicitlyConfigured(config)) { + // derive shuffle memory from total flink memory, and check against shuffle min/max + final MemorySize totalFlinkMemorySize = getTotalFlinkMemorySize(config); + if (totalFlinkExcludeShuffleMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) { + throw new IllegalConfigurationException( + "Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toString() + + "), Task Heap Memory (" + taskHeapMemorySize.toString() + + "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toString() + + ") and Managed Memory (" + managedMemorySize.toString() + + ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toString() + ")."); + } + shuffleMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeShuffleMemorySize); + sanityCheckShuffleMemory(config, shuffleMemorySize, totalFlinkMemorySize); + } else { + // derive shuffle memory from shuffle configs + if (isUsingLegacyShuffleConfigs(config)) { + shuffleMemorySize = getShuffleMemorySizeWithLegacyConfig(config); + } else { + shuffleMemorySize = deriveShuffleMemoryWithInverseFraction(config, totalFlinkExcludeShuffleMemorySize); + } + } + + final FlinkInternalMemory flinkInternalMemory = new FlinkInternalMemory( + frameworkHeapMemorySize, + taskHeapMemorySize, + taskOffHeapMemorySize, + shuffleMemorySize, + onHeapAndOffHeapManagedMemory.onHeap, + onHeapAndOffHeapManagedMemory.offHeap); + sanityCheckTotalFlinkMemory(config, flinkInternalMemory); + + // derive jvm metaspace and overhead + + final JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead = deriveJvmMetaspaceAndOverheadFromTotalFlinkMemory(config, flinkInternalMemory.getTotalFlinkMemorySize()); + sanityCheckTotalProcessMemory(config, flinkInternalMemory.getTotalFlinkMemorySize(), jvmMetaspaceAndOverhead); + + return createTaskExecutorResourceSpec(flinkInternalMemory, jvmMetaspaceAndOverhead); + } + + private static TaskExecutorResourceSpec deriveResourceSpecWithTotalFlinkMemory(final Configuration config) { + // derive flink internal memory from explicitly configured total flink memory + + final MemorySize totalFlinkMemorySize = getTotalFlinkMemorySize(config); + final FlinkInternalMemory flinkInternalMemory = deriveInternalMemoryFromTotalFlinkMemory(config, totalFlinkMemorySize); + + // derive jvm metaspace and overhead + + final JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead = deriveJvmMetaspaceAndOverheadFromTotalFlinkMemory(config, totalFlinkMemorySize); + sanityCheckTotalProcessMemory(config, totalFlinkMemorySize, jvmMetaspaceAndOverhead); + + return createTaskExecutorResourceSpec(flinkInternalMemory, jvmMetaspaceAndOverhead); + } + + private static TaskExecutorResourceSpec deriveResourceSpecWithTotalProcessMemory(final Configuration config) { + // derive total flink memory from explicitly configured total process memory size + + final MemorySize totalProcessMemorySize = getTotalProcessMemorySize(config); + final MemorySize jvmMetaspaceSize = getJvmMetaspaceSize(config); + final MemorySize jvmOverheadSize = deriveJvmOverheadWithFraction(config, totalProcessMemorySize); + final JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead = new JvmMetaspaceAndOverhead(jvmMetaspaceSize, jvmOverheadSize); + + if (jvmMetaspaceAndOverhead.getTotalJvmMetaspaceAndOverheadSize().getBytes() > totalProcessMemorySize.getBytes()) { + throw new IllegalConfigurationException( + "Sum of configured JVM Metaspace (" + jvmMetaspaceAndOverhead.metaspace.toString() + + ") and JVM Overhead (" + jvmMetaspaceAndOverhead.overhead.toString() + + ") exceed configured Total Process Memory (" + totalProcessMemorySize.toString() + ")."); + } + final MemorySize totalFlinkMemorySize = totalProcessMemorySize.subtract(jvmMetaspaceAndOverhead.getTotalJvmMetaspaceAndOverheadSize()); + + // derive flink internal memory + + final FlinkInternalMemory flinkInternalMemory = deriveInternalMemoryFromTotalFlinkMemory(config, totalFlinkMemorySize); + + return createTaskExecutorResourceSpec(flinkInternalMemory, jvmMetaspaceAndOverhead); + } + + private static JvmMetaspaceAndOverhead deriveJvmMetaspaceAndOverheadFromTotalFlinkMemory( + final Configuration config, final MemorySize totalFlinkMemorySize) { + final MemorySize jvmMetaspaceSize = getJvmMetaspaceSize(config); + final MemorySize jvmOverheadSize = deriveJvmOverheadWithInverseFraction(config, + totalFlinkMemorySize.add(jvmMetaspaceSize)); + return new JvmMetaspaceAndOverhead(jvmMetaspaceSize, jvmOverheadSize); + } + + private static FlinkInternalMemory deriveInternalMemoryFromTotalFlinkMemory( + final Configuration config, final MemorySize totalFlinkMemorySize) { + final MemorySize frameworkHeapMemorySize = getFrameworkHeapMemorySize(config); + final MemorySize taskOffHeapMemorySize = getTaskOffHeapMemorySize(config); + + final MemorySize taskHeapMemorySize; + final MemorySize shuffleMemorySize; + final MemorySize managedMemorySize; + + if (isTaskHeapMemorySizeExplicitlyConfigured(config)) { + // task heap memory is configured, + // derive managed memory first, leave the remaining to shuffle memory and check against shuffle min/max + taskHeapMemorySize = getTaskHeapMemorySize(config); + managedMemorySize = deriveManagedMemoryAbsoluteOrWithFraction(config, totalFlinkMemorySize); + final MemorySize totalFlinkExcludeShuffleMemorySize = + frameworkHeapMemorySize.add(taskHeapMemorySize).add(taskOffHeapMemorySize).add(managedMemorySize); + if (totalFlinkExcludeShuffleMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) { + throw new IllegalConfigurationException( + "Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toString() + + "), Task Heap Memory (" + taskHeapMemorySize.toString() + + "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toString() + + ") and Managed Memory (" + managedMemorySize.toString() + + ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toString() + ")."); + } + shuffleMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeShuffleMemorySize); + sanityCheckShuffleMemory(config, shuffleMemorySize, totalFlinkMemorySize); + } else { + // task heap memory is not configured + // derive managed memory and shuffle memory, leave the remaining to task heap memory + if (isManagedMemorySizeExplicitlyConfigured(config)) { + managedMemorySize = getManagedMemorySize(config); + } else { + managedMemorySize = deriveManagedMemoryAbsoluteOrWithFraction(config, totalFlinkMemorySize); + } + shuffleMemorySize = deriveShuffleMemoryWithFraction(config, totalFlinkMemorySize); + final MemorySize totalFlinkExcludeTaskHeapMemorySize = + frameworkHeapMemorySize.add(taskOffHeapMemorySize).add(managedMemorySize).add(shuffleMemorySize); + if (totalFlinkExcludeTaskHeapMemorySize.getBytes() > totalFlinkMemorySize.getBytes()) { + throw new IllegalConfigurationException( + "Sum of configured Framework Heap Memory (" + frameworkHeapMemorySize.toString() + + "), Task Off-Heap Memory (" + taskOffHeapMemorySize.toString() + + "), Managed Memory (" + managedMemorySize.toString() + + ") and Shuffle Memory (" + shuffleMemorySize.toString() + + ") exceed configured Total Flink Memory (" + totalFlinkMemorySize.toString() + ")."); + } + taskHeapMemorySize = totalFlinkMemorySize.subtract(totalFlinkExcludeTaskHeapMemorySize); + } + + final OnHeapAndOffHeapManagedMemory onHeapAndOffHeapManagedMemory = deriveOnHeapAndOffHeapMemoryFromManagedMemory(config, managedMemorySize); + final FlinkInternalMemory flinkInternalMemory = new FlinkInternalMemory( + frameworkHeapMemorySize, + taskHeapMemorySize, + taskOffHeapMemorySize, + shuffleMemorySize, + onHeapAndOffHeapManagedMemory.onHeap, + onHeapAndOffHeapManagedMemory.offHeap); + sanityCheckTotalFlinkMemory(config, flinkInternalMemory); + + return flinkInternalMemory; + } + + private static OnHeapAndOffHeapManagedMemory deriveOnHeapAndOffHeapMemoryFromManagedMemory( + final Configuration config, final MemorySize managedMemorySize) { + + final MemorySize offHeapSize; + + if (isManagedMemoryOffHeapSizeExplicitlyConfigured(config)) { + offHeapSize = getManagedMemoryOffHeapSize(config); + // sanity check + if (offHeapSize.getBytes() > managedMemorySize.getBytes()) { + throw new IllegalConfigurationException("Configured Off-Heap Managed Memory size (" + offHeapSize.toString() + + ") is larger than configured/derived total Managed Memory size (" + managedMemorySize.toString() + ")."); + } + } else { + final double offHeapFraction; + if (isManagedMemoryOffHeapFractionExplicitlyConfigured(config)) { + offHeapFraction = getManagedMemoryOffHeapFraction(config); + } else { + @SuppressWarnings("deprecation") + final boolean legacyManagedMemoryOffHeap = config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP); + offHeapFraction = legacyManagedMemoryOffHeap ? 1.0 : 0.0; + } + offHeapSize = managedMemorySize.multiply(offHeapFraction); + } + + final MemorySize onHeapSize = managedMemorySize.subtract(offHeapSize); + return new OnHeapAndOffHeapManagedMemory(onHeapSize, offHeapSize); + } + + private static MemorySize deriveManagedMemoryAbsoluteOrWithFraction(final Configuration config, final MemorySize base) { + if (isManagedMemorySizeExplicitlyConfigured(config)) { + return getManagedMemorySize(config); + } else { + return deriveWithFraction(base, getManagedMemoryRangeFraction(config)); + } + } + + private static MemorySize deriveShuffleMemoryWithFraction(final Configuration config, final MemorySize base) { + return deriveWithFraction(base, getShuffleMemoryRangeFraction(config)); + } + + private static MemorySize deriveShuffleMemoryWithInverseFraction(final Configuration config, final MemorySize base) { + return deriveWithInverseFraction(base, getShuffleMemoryRangeFraction(config)); + } + + private static MemorySize deriveJvmOverheadWithFraction(final Configuration config, final MemorySize base) { + return deriveWithFraction(base, getJvmOverheadRangeFraction(config)); + } + + private static MemorySize deriveJvmOverheadWithInverseFraction(final Configuration config, final MemorySize base) { + return deriveWithInverseFraction(base, getJvmOverheadRangeFraction(config)); + } + + private static MemorySize deriveWithFraction(final MemorySize base, final RangeFraction rangeFraction) { + final long relative = (long) (rangeFraction.fraction * base.getBytes()); + return new MemorySize(Math.max(rangeFraction.minSize.getBytes(), Math.min(rangeFraction.maxSize.getBytes(), relative))); + } + + private static MemorySize deriveWithInverseFraction(final MemorySize base, final RangeFraction rangeFraction) { + checkArgument(rangeFraction.fraction < 1); + final long relative = (long) (rangeFraction.fraction / (1 - rangeFraction.fraction) * base.getBytes()); + return new MemorySize(Math.max(rangeFraction.minSize.getBytes(), Math.min(rangeFraction.maxSize.getBytes(), relative))); + } + + private static MemorySize getFrameworkHeapMemorySize(final Configuration config) { + return MemorySize.parse(config.getString(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY)); + } + + private static MemorySize getTaskHeapMemorySize(final Configuration config) { + checkArgument(isTaskHeapMemorySizeExplicitlyConfigured(config)); + return MemorySize.parse(config.getString(TaskManagerOptions.TASK_HEAP_MEMORY)); + } + + private static MemorySize getTaskOffHeapMemorySize(final Configuration config) { + return MemorySize.parse(config.getString(TaskManagerOptions.TASK_OFF_HEAP_MEMORY)); + } + + private static MemorySize getManagedMemorySize(final Configuration config) { + checkArgument(isManagedMemorySizeExplicitlyConfigured(config)); + return MemorySize.parse(config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE)); + } + + private static RangeFraction getManagedMemoryRangeFraction(final Configuration config) { + final MemorySize minSize = new MemorySize(0); + final MemorySize maxSize = new MemorySize(Long.MAX_VALUE); + final double fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION); + if (fraction >= 1 || fraction < 0) { + throw new IllegalConfigurationException("Configured Managed Memory fraction (" + + fraction + ") must be in [0, 1)."); + } + return new RangeFraction(minSize, maxSize, fraction); + } + + private static double getManagedMemoryOffHeapFraction(final Configuration config) { + checkArgument(isManagedMemoryOffHeapFractionExplicitlyConfigured(config)); + final double offHeapFraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_FRACTION); + if (offHeapFraction > 1 || offHeapFraction < 0) { + throw new IllegalConfigurationException("Configured Off-Heap Managed Memory fraction (" + + offHeapFraction + ") must be in [0, 1]."); + } + return offHeapFraction; + } + + private static MemorySize getManagedMemoryOffHeapSize(final Configuration config) { + checkArgument(isManagedMemoryOffHeapSizeExplicitlyConfigured(config)); + return MemorySize.parse(config.getString(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_SIZE)); + } + + private static MemorySize getShuffleMemorySizeWithLegacyConfig(final Configuration config) { + checkArgument(isUsingLegacyShuffleConfigs(config)); + @SuppressWarnings("deprecation") + final long numOfBuffers = config.getInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS); + final long pageSize = ConfigurationParserUtils.getPageSize(config); + return new MemorySize(numOfBuffers * pageSize); + } + + private static RangeFraction getShuffleMemoryRangeFraction(final Configuration config) { + final MemorySize minSize = MemorySize.parse(config.getString(TaskManagerOptions.SHUFFLE_MEMORY_MIN)); + final MemorySize maxSize = MemorySize.parse(config.getString(TaskManagerOptions.SHUFFLE_MEMORY_MAX)); + final double fraction = config.getFloat(TaskManagerOptions.SHUFFLE_MEMORY_FRACTION); + if (fraction >= 1 || fraction < 0) { + throw new IllegalConfigurationException("Configured Shuffle Memory fraction (" + + fraction + ") must be in [0, 1)."); + } + return new RangeFraction(minSize, maxSize, fraction); + } + + private static MemorySize getJvmMetaspaceSize(final Configuration config) { + return MemorySize.parse(config.getString(TaskManagerOptions.JVM_METASPACE)); + } + + private static RangeFraction getJvmOverheadRangeFraction(final Configuration config) { + final MemorySize minSize = MemorySize.parse(config.getString(TaskManagerOptions.JVM_OVERHEAD_MIN)); + final MemorySize maxSize = MemorySize.parse(config.getString(TaskManagerOptions.JVM_OVERHEAD_MAX)); + final double fraction = config.getFloat(TaskManagerOptions.JVM_OVERHEAD_FRACTION); + if (fraction >= 1 || fraction < 0) { + throw new IllegalConfigurationException("Configured JVM Overhead fraction (" + + fraction + ") must be in [0, 1)."); + } + return new RangeFraction(minSize, maxSize, fraction); + } + + private static MemorySize getTotalFlinkMemorySize(final Configuration config) { + checkArgument(isTotalFlinkMemorySizeExplicitlyConfigured(config)); + if (config.contains(TaskManagerOptions.TOTAL_FLINK_MEMORY)) { + return MemorySize.parse(config.getString(TaskManagerOptions.TOTAL_FLINK_MEMORY)); + } else { + @SuppressWarnings("deprecation") + final long legacyHeapMemoryMB = config.getInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB); + return new MemorySize(legacyHeapMemoryMB << 20); // megabytes to bytes + } + } + + private static MemorySize getTotalProcessMemorySize(final Configuration config) { + checkArgument(isTotalProcessMemorySizeExplicitlyConfigured(config)); + return MemorySize.parse(config.getString(TaskManagerOptions.TOTAL_PROCESS_MEMORY)); + } + + private static boolean isTaskHeapMemorySizeExplicitlyConfigured(final Configuration config) { + return config.contains(TaskManagerOptions.TASK_HEAP_MEMORY); + } + + private static boolean isManagedMemorySizeExplicitlyConfigured(final Configuration config) { + return config.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE); + } + + private static boolean isManagedMemoryOffHeapFractionExplicitlyConfigured(final Configuration config) { + return config.getFloat(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_FRACTION) >= 0; + } + + private static boolean isManagedMemoryOffHeapSizeExplicitlyConfigured(final Configuration config) { + return config.contains(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_SIZE); + } + + private static boolean isUsingLegacyShuffleConfigs(final Configuration config) { + // use the legacy number-of-buffer config option only when it is explicitly configured and + // none of new config options is explicitly configured + @SuppressWarnings("deprecation") + final boolean legacyConfigured = config.contains(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS); + return !config.contains(TaskManagerOptions.SHUFFLE_MEMORY_MIN) && + !config.contains(TaskManagerOptions.SHUFFLE_MEMORY_MAX) && + !config.contains(TaskManagerOptions.SHUFFLE_MEMORY_FRACTION) && + legacyConfigured; + } + + private static boolean isShuffleMemoryFractionExplicitlyConfigured(final Configuration config) { + return config.contains(TaskManagerOptions.SHUFFLE_MEMORY_FRACTION); + } + + private static boolean isTotalFlinkMemorySizeExplicitlyConfigured(final Configuration config) { + // backward compatible with the deprecated config option TASK_MANAGER_HEAP_MEMORY_MB only when it's explicitly + // configured by the user + @SuppressWarnings("deprecation") + final boolean legacyConfigured = config.contains(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB); + return config.contains(TaskManagerOptions.TOTAL_FLINK_MEMORY) || legacyConfigured; + } + + private static boolean isTotalProcessMemorySizeExplicitlyConfigured(final Configuration config) { + return config.contains(TaskManagerOptions.TOTAL_PROCESS_MEMORY); + } + + private static void sanityCheckTotalFlinkMemory(final Configuration config, final FlinkInternalMemory flinkInternalMemory) { + if (isTotalFlinkMemorySizeExplicitlyConfigured(config)) { + final MemorySize configuredTotalFlinkMemorySize = getTotalFlinkMemorySize(config); + if (!configuredTotalFlinkMemorySize.equals(flinkInternalMemory.getTotalFlinkMemorySize())) { + throw new IllegalConfigurationException( + "Configured/Derived Flink internal memory sizes (total " + flinkInternalMemory.getTotalFlinkMemorySize().toString() + + ") do not add up to the configured Total Flink Memory size (" + configuredTotalFlinkMemorySize.toString() + + "). Configured/Derived Flink internal memory sizes are: " + + "Framework Heap Memory (" + flinkInternalMemory.frameworkHeap.toString() + + "), Task Heap Memory (" + flinkInternalMemory.taskHeap.toString() + + "), Task Off-Heap Memory (" + flinkInternalMemory.taskOffHeap.toString() + + "), Shuffle Memory (" + flinkInternalMemory.shuffle.toString() + + "), Managed Memory (" + flinkInternalMemory.getManagedMemorySize().toString() + ")."); + } + } + } + + private static void sanityCheckTotalProcessMemory(final Configuration config, final MemorySize totalFlinkMemory, final JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead) { + final MemorySize derivedTotalProcessMemorySize = + totalFlinkMemory.add(jvmMetaspaceAndOverhead.metaspace).add(jvmMetaspaceAndOverhead.overhead); + if (isTotalProcessMemorySizeExplicitlyConfigured(config)) { + final MemorySize configuredTotalProcessMemorySize = getTotalProcessMemorySize(config); + if (!configuredTotalProcessMemorySize.equals(derivedTotalProcessMemorySize)) { + throw new IllegalConfigurationException( + "Configured/Derived memory sizes (total " + derivedTotalProcessMemorySize.toString() + + ") do not add up to the configured Total Process Memory size (" + configuredTotalProcessMemorySize.toString() + + "). Configured/Derived memory sizes are: " + + "Total Flink Memory (" + totalFlinkMemory.toString() + + "), JVM Metaspace (" + jvmMetaspaceAndOverhead.metaspace.toString() + + "), JVM Overhead (" + jvmMetaspaceAndOverhead.overhead.toString() + ")."); + } + } + } + + private static void sanityCheckShuffleMemory(final Configuration config, final MemorySize derivedShuffleMemorySize, final MemorySize totalFlinkMemorySize) { + if (isUsingLegacyShuffleConfigs(config)) { + final MemorySize configuredShuffleMemorySize = getShuffleMemorySizeWithLegacyConfig(config); + if (!configuredShuffleMemorySize.equals(derivedShuffleMemorySize)) { + throw new IllegalConfigurationException( + "Derived Shuffle Memory size (" + derivedShuffleMemorySize.toString() + + ") does not match configured Shuffle Memory size (" + configuredShuffleMemorySize.toString() + ")."); + } + } else { + final RangeFraction shuffleRangeFraction = getShuffleMemoryRangeFraction(config); + if (derivedShuffleMemorySize.getBytes() > shuffleRangeFraction.maxSize.getBytes() || + derivedShuffleMemorySize.getBytes() < shuffleRangeFraction.minSize.getBytes()) { + throw new IllegalConfigurationException("Derived Shuffle Memory size (" + + derivedShuffleMemorySize.toString() + ") is not in configured Shuffle Memory range [" + + shuffleRangeFraction.minSize.toString() + ", " + + shuffleRangeFraction.maxSize.toString() + "]."); + } + if (isShuffleMemoryFractionExplicitlyConfigured(config) && + !derivedShuffleMemorySize.equals(totalFlinkMemorySize.multiply(shuffleRangeFraction.fraction))) { + throw new IllegalConfigurationException("Derived Shuffle Memory size(" + + derivedShuffleMemorySize.toString() + ") does not match configured Shuffle Memory fraction (" + + shuffleRangeFraction.fraction + ")."); + } + } + } + + private static TaskExecutorResourceSpec createTaskExecutorResourceSpec( + final FlinkInternalMemory flinkInternalMemory, final JvmMetaspaceAndOverhead jvmMetaspaceAndOverhead) { + return new TaskExecutorResourceSpec( + flinkInternalMemory.frameworkHeap, + flinkInternalMemory.taskHeap, + flinkInternalMemory.taskOffHeap, + flinkInternalMemory.shuffle, + flinkInternalMemory.onHeapManaged, + flinkInternalMemory.offHeapManaged, + jvmMetaspaceAndOverhead.metaspace, + jvmMetaspaceAndOverhead.overhead); + } + + private static class RangeFraction { + final MemorySize minSize; + final MemorySize maxSize; + final double fraction; + + RangeFraction(final MemorySize minSize, final MemorySize maxSize, final double fraction) { + this.minSize = minSize; + this.maxSize = maxSize; + this.fraction = fraction; + checkArgument(minSize.getBytes() <= maxSize.getBytes()); + checkArgument(fraction >= 0 && fraction <= 1); + } + } + + private static class FlinkInternalMemory { + final MemorySize frameworkHeap; + final MemorySize taskHeap; + final MemorySize taskOffHeap; + final MemorySize shuffle; + final MemorySize onHeapManaged; + final MemorySize offHeapManaged; + + FlinkInternalMemory( + final MemorySize frameworkHeap, + final MemorySize taskHeap, + final MemorySize taskOffHeap, + final MemorySize shuffle, + final MemorySize onHeapManaged, + final MemorySize offHeapManaged) { + + this.frameworkHeap = checkNotNull(frameworkHeap); + this.taskHeap = checkNotNull(taskHeap); + this.taskOffHeap = checkNotNull(taskOffHeap); + this.shuffle = checkNotNull(shuffle); + this.onHeapManaged = checkNotNull(onHeapManaged); + this.offHeapManaged = checkNotNull(offHeapManaged); + } + + MemorySize getTotalFlinkMemorySize() { + return frameworkHeap.add(taskHeap).add(taskOffHeap).add(shuffle).add(getManagedMemorySize()); + } + + MemorySize getManagedMemorySize() { + return onHeapManaged.add(offHeapManaged); + } + } + + private static class OnHeapAndOffHeapManagedMemory { + final MemorySize onHeap; + final MemorySize offHeap; + + OnHeapAndOffHeapManagedMemory(final MemorySize onHeap, final MemorySize offHeap) { + this.onHeap = onHeap; + this.offHeap = offHeap; + } + } + + private static class JvmMetaspaceAndOverhead { + final MemorySize metaspace; + final MemorySize overhead; + + JvmMetaspaceAndOverhead(final MemorySize jvmMetaspace, final MemorySize jvmOverhead) { + this.metaspace = checkNotNull(jvmMetaspace); + this.overhead = checkNotNull(jvmOverhead); + } + + MemorySize getTotalJvmMetaspaceAndOverheadSize() { + return metaspace.add(overhead); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java index 5b958b74507cde35de85d8a1114a91ba89b6952c..b60ee3cff18fffc1e2e672c6182a5a78dc96c404 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/concurrent/FutureUtils.java @@ -30,6 +30,7 @@ import akka.dispatch.OnComplete; import javax.annotation.Nonnull; +import java.time.Duration; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -55,7 +56,6 @@ import java.util.function.Predicate; import java.util.function.Supplier; import scala.concurrent.Future; -import scala.concurrent.duration.FiniteDuration; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -64,6 +64,17 @@ import static org.apache.flink.util.Preconditions.checkNotNull; */ public class FutureUtils { + private static final CompletableFuture COMPLETED_VOID_FUTURE = CompletableFuture.completedFuture(null); + + /** + * Returns a completed future of type {@link Void}. + * + * @return a completed future of type {@link Void} + */ + public static CompletableFuture completedVoidFuture() { + return COMPLETED_VOID_FUTURE; + } + // ------------------------------------------------------------------------ // retrying operations // ------------------------------------------------------------------------ @@ -833,23 +844,13 @@ public class FutureUtils { } /** - * Converts Flink time into a {@link FiniteDuration}. - * - * @param time to convert into a FiniteDuration - * @return FiniteDuration with the length of the given time - */ - public static FiniteDuration toFiniteDuration(Time time) { - return new FiniteDuration(time.toMilliseconds(), TimeUnit.MILLISECONDS); - } - - /** - * Converts {@link FiniteDuration} into Flink time. + * Converts Flink time into a {@link Duration}. * - * @param finiteDuration to convert into Flink time - * @return Flink time with the length of the given finite duration + * @param time to convert into a Duration + * @return Duration with the length of the given time */ - public static Time toTime(FiniteDuration finiteDuration) { - return Time.milliseconds(finiteDuration.toMillis()); + public static Duration toDuration(Time time) { + return Duration.ofMillis(time.toMilliseconds()); } // ------------------------------------------------------------------------ @@ -1076,4 +1077,21 @@ public class FutureUtils { throw suppressedExceptions; } } + + /** + * Forwards the value from the source future to the target future. + * + * @param source future to forward the value from + * @param target future to forward the value to + * @param type of the value + */ + public static void forward(CompletableFuture source, CompletableFuture target) { + source.whenComplete((value, throwable) -> { + if (throwable != null) { + target.completeExceptionally(throwable); + } else { + target.complete(value); + } + }); + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DefaultJobManagerRunnerFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DefaultJobManagerRunnerFactory.java index cab264d671258ac54be183625114a266c7ec89af..e42643c13a4f7ee65ff333ab5ba802749c5f2f9c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DefaultJobManagerRunnerFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DefaultJobManagerRunnerFactory.java @@ -23,6 +23,7 @@ import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobmaster.JobManagerRunner; +import org.apache.flink.runtime.jobmaster.JobManagerRunnerImpl; import org.apache.flink.runtime.jobmaster.JobManagerSharedServices; import org.apache.flink.runtime.jobmaster.JobMasterConfiguration; import org.apache.flink.runtime.jobmaster.factories.DefaultJobMasterServiceFactory; @@ -39,7 +40,7 @@ import org.apache.flink.runtime.shuffle.ShuffleMaster; import org.apache.flink.runtime.shuffle.ShuffleServiceLoader; /** - * Singleton default factory for {@link JobManagerRunner}. + * Singleton default factory for {@link JobManagerRunnerImpl}. */ public enum DefaultJobManagerRunnerFactory implements JobManagerRunnerFactory { INSTANCE; @@ -75,7 +76,7 @@ public enum DefaultJobManagerRunnerFactory implements JobManagerRunnerFactory { schedulerNGFactory, shuffleMaster); - return new JobManagerRunner( + return new JobManagerRunnerImpl( jobGraph, jobMasterFactory, highAvailabilityServices, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java index c4beb01528f1db827ebe2bda01ba4f39848b8b42..1c7fb6a414142d7b6f633c75fdfa1f83683c65b3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/Dispatcher.java @@ -27,7 +27,9 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.MetricGroup; import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.checkpoint.Checkpoints; +import org.apache.flink.runtime.client.DuplicateJobSubmissionException; import org.apache.flink.runtime.client.JobSubmissionException; +import org.apache.flink.runtime.clusterframework.ApplicationStatus; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; @@ -40,6 +42,7 @@ import org.apache.flink.runtime.jobgraph.JobVertex; import org.apache.flink.runtime.jobgraph.JobVertexID; import org.apache.flink.runtime.jobmanager.JobGraphStore; import org.apache.flink.runtime.jobmaster.JobManagerRunner; +import org.apache.flink.runtime.jobmaster.JobManagerRunnerImpl; import org.apache.flink.runtime.jobmaster.JobManagerSharedServices; import org.apache.flink.runtime.jobmaster.JobMasterGateway; import org.apache.flink.runtime.jobmaster.JobNotFinishedException; @@ -132,36 +135,29 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme private CompletableFuture recoveryOperation = CompletableFuture.completedFuture(null); + protected final CompletableFuture shutDownFuture; + public Dispatcher( RpcService rpcService, String endpointId, - Configuration configuration, - HighAvailabilityServices highAvailabilityServices, - JobGraphStore jobGraphStore, - GatewayRetriever resourceManagerGatewayRetriever, - BlobServer blobServer, - HeartbeatServices heartbeatServices, - JobManagerMetricGroup jobManagerMetricGroup, - @Nullable String metricServiceQueryAddress, - ArchivedExecutionGraphStore archivedExecutionGraphStore, - JobManagerRunnerFactory jobManagerRunnerFactory, - FatalErrorHandler fatalErrorHandler, - HistoryServerArchivist historyServerArchivist) throws Exception { - super(rpcService, endpointId); - - this.configuration = Preconditions.checkNotNull(configuration); - this.highAvailabilityServices = Preconditions.checkNotNull(highAvailabilityServices); - this.resourceManagerGatewayRetriever = Preconditions.checkNotNull(resourceManagerGatewayRetriever); - this.heartbeatServices = Preconditions.checkNotNull(heartbeatServices); - this.blobServer = Preconditions.checkNotNull(blobServer); - this.fatalErrorHandler = Preconditions.checkNotNull(fatalErrorHandler); + DispatcherServices dispatcherServices, + JobGraphStore jobGraphStore) throws Exception { + super(rpcService, endpointId, null); + Preconditions.checkNotNull(dispatcherServices); + + this.configuration = dispatcherServices.getConfiguration(); + this.highAvailabilityServices = dispatcherServices.getHighAvailabilityServices(); + this.resourceManagerGatewayRetriever = dispatcherServices.getResourceManagerGatewayRetriever(); + this.heartbeatServices = dispatcherServices.getHeartbeatServices(); + this.blobServer = dispatcherServices.getBlobServer(); + this.fatalErrorHandler = dispatcherServices.getFatalErrorHandler(); this.jobGraphStore = Preconditions.checkNotNull(jobGraphStore); - this.jobManagerMetricGroup = Preconditions.checkNotNull(jobManagerMetricGroup); - this.metricServiceQueryAddress = metricServiceQueryAddress; + this.jobManagerMetricGroup = dispatcherServices.getJobManagerMetricGroup(); + this.metricServiceQueryAddress = dispatcherServices.getMetricQueryServiceAddress(); this.jobManagerSharedServices = JobManagerSharedServices.fromConfiguration( configuration, - this.blobServer); + blobServer); this.runningJobsRegistry = highAvailabilityServices.getRunningJobsRegistry(); @@ -169,13 +165,23 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme leaderElectionService = highAvailabilityServices.getDispatcherLeaderElectionService(); - this.historyServerArchivist = Preconditions.checkNotNull(historyServerArchivist); + this.historyServerArchivist = dispatcherServices.getHistoryServerArchivist(); - this.archivedExecutionGraphStore = Preconditions.checkNotNull(archivedExecutionGraphStore); + this.archivedExecutionGraphStore = dispatcherServices.getArchivedExecutionGraphStore(); - this.jobManagerRunnerFactory = Preconditions.checkNotNull(jobManagerRunnerFactory); + this.jobManagerRunnerFactory = dispatcherServices.getJobManagerRunnerFactory(); this.jobManagerTerminationFutures = new HashMap<>(2); + + this.shutDownFuture = new CompletableFuture<>(); + } + + //------------------------------------------------------ + // Getters + //------------------------------------------------------ + + public CompletableFuture getShutDownFuture() { + return shutDownFuture; } //------------------------------------------------------ @@ -265,7 +271,7 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme try { if (isDuplicateJob(jobGraph.getJobID())) { return FutureUtils.completedExceptionally( - new JobSubmissionException(jobGraph.getJobID(), "Job has already been submitted.")); + new DuplicateJobSubmissionException(jobGraph.getJobID())); } else if (isPartialResourceConfigured(jobGraph)) { return FutureUtils.completedExceptionally( new JobSubmissionException(jobGraph.getJobID(), "Currently jobs is not supported if parts of the vertices have " + @@ -386,14 +392,15 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme } private JobManagerRunner startJobManagerRunner(JobManagerRunner jobManagerRunner) throws Exception { - final JobID jobId = jobManagerRunner.getJobGraph().getJobID(); + final JobID jobId = jobManagerRunner.getJobID(); FutureUtils.assertNoException( jobManagerRunner.getResultFuture().handleAsync( (ArchivedExecutionGraph archivedExecutionGraph, Throwable throwable) -> { // check if we are still the active JobManagerRunner by checking the identity - final CompletableFuture jobManagerRunnerFuture = jobManagerRunnerFutures.get(jobId); - final JobManagerRunner currentJobManagerRunner = jobManagerRunnerFuture != null ? jobManagerRunnerFuture.getNow(null) : null; + final JobManagerRunner currentJobManagerRunner = Optional.ofNullable(jobManagerRunnerFutures.get(jobId)) + .map(future -> future.getNow(null)) + .orElse(null); //noinspection ObjectEquality if (jobManagerRunner == currentJobManagerRunner) { if (archivedExecutionGraph != null) { @@ -609,7 +616,7 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme @Override public CompletableFuture shutDownCluster() { - closeAsync(); + shutDownFuture.complete(ApplicationStatus.SUCCEEDED); return CompletableFuture.completedFuture(Acknowledge.get()); } @@ -690,7 +697,7 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme } /** - * Terminate all currently running {@link JobManagerRunner}. + * Terminate all currently running {@link JobManagerRunnerImpl}. */ private void terminateJobManagerRunners() { log.info("Stopping all currently running jobs of dispatcher {}.", getAddress()); @@ -817,7 +824,7 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme if (jobManagerRunnerFuture == null) { return FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId)); } else { - final CompletableFuture leaderGatewayFuture = jobManagerRunnerFuture.thenCompose(JobManagerRunner::getLeaderGatewayFuture); + final CompletableFuture leaderGatewayFuture = jobManagerRunnerFuture.thenCompose(JobManagerRunner::getJobMasterGateway); return leaderGatewayFuture.thenApplyAsync( (JobMasterGateway jobMasterGateway) -> { // check whether the retrieved JobMasterGateway belongs still to a running JobMaster @@ -889,7 +896,7 @@ public abstract class Dispatcher extends FencedRpcEndpoint impleme recoveredJobsFuture, BiFunctionWithException.unchecked((Boolean confirmLeadership, Collection recoveredJobs) -> { if (confirmLeadership) { - leaderElectionService.confirmLeaderSessionID(newLeaderSessionID); + leaderElectionService.confirmLeadership(newLeaderSessionID, getAddress()); } else { for (JobGraph recoveredJob : recoveredJobs) { jobGraphStore.releaseJobGraph(recoveredJob.getJobID()); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherFactory.java index 4c421a09d0842e74bd2480a9029d340f12a40168..4e7ea67ebc1c179819ee8365fee72af16b19501d 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherFactory.java @@ -18,40 +18,23 @@ package org.apache.flink.runtime.dispatcher; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.blob.BlobServer; -import org.apache.flink.runtime.heartbeat.HeartbeatServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; -import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; -import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; import java.util.UUID; /** * {@link Dispatcher} factory interface. */ -public interface DispatcherFactory { +public interface DispatcherFactory { /** - * Create a {@link Dispatcher} of the given type {@link T}. + * Create a {@link Dispatcher}. */ - T createDispatcher( - Configuration configuration, - RpcService rpcService, - HighAvailabilityServices highAvailabilityServices, - GatewayRetriever resourceManagerGatewayRetriever, - BlobServer blobServer, - HeartbeatServices heartbeatServices, - JobManagerMetricGroup jobManagerMetricGroup, - @Nullable String metricQueryServiceAddress, - ArchivedExecutionGraphStore archivedExecutionGraphStore, - FatalErrorHandler fatalErrorHandler, - HistoryServerArchivist historyServerArchivist) throws Exception; + Dispatcher createDispatcher( + @Nonnull RpcService rpcService, + @Nonnull PartialDispatcherServices partialDispatcherServices) throws Exception; default String generateEndpointIdWithUUID() { return getEndpointId() + UUID.randomUUID(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherServices.java new file mode 100644 index 0000000000000000000000000000000000000000..bdb64638a59645622c2c67165e2261b2d319e76c --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/DispatcherServices.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.dispatcher; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.blob.BlobServer; +import org.apache.flink.runtime.heartbeat.HeartbeatServices; +import org.apache.flink.runtime.highavailability.HighAvailabilityServices; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; +import org.apache.flink.runtime.rpc.FatalErrorHandler; +import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * {@link Dispatcher} services container. + */ +public class DispatcherServices { + + @Nonnull + private final Configuration configuration; + + @Nonnull + private final HighAvailabilityServices highAvailabilityServices; + + @Nonnull + private final GatewayRetriever resourceManagerGatewayRetriever; + + @Nonnull + private final BlobServer blobServer; + + @Nonnull + private final HeartbeatServices heartbeatServices; + + @Nonnull + private final JobManagerMetricGroup jobManagerMetricGroup; + + @Nonnull + private final ArchivedExecutionGraphStore archivedExecutionGraphStore; + + @Nonnull + private final FatalErrorHandler fatalErrorHandler; + + @Nonnull + private final HistoryServerArchivist historyServerArchivist; + + @Nullable + private final String metricQueryServiceAddress; + + @Nonnull + private final JobManagerRunnerFactory jobManagerRunnerFactory; + + public DispatcherServices( + @Nonnull Configuration configuration, + @Nonnull HighAvailabilityServices highAvailabilityServices, + @Nonnull GatewayRetriever resourceManagerGatewayRetriever, + @Nonnull BlobServer blobServer, + @Nonnull HeartbeatServices heartbeatServices, + @Nonnull JobManagerMetricGroup jobManagerMetricGroup, + @Nonnull ArchivedExecutionGraphStore archivedExecutionGraphStore, + @Nonnull FatalErrorHandler fatalErrorHandler, + @Nonnull HistoryServerArchivist historyServerArchivist, + @Nullable String metricQueryServiceAddress, + @Nonnull JobManagerRunnerFactory jobManagerRunnerFactory) { + this.configuration = configuration; + this.highAvailabilityServices = highAvailabilityServices; + this.resourceManagerGatewayRetriever = resourceManagerGatewayRetriever; + this.blobServer = blobServer; + this.heartbeatServices = heartbeatServices; + this.jobManagerMetricGroup = jobManagerMetricGroup; + this.archivedExecutionGraphStore = archivedExecutionGraphStore; + this.fatalErrorHandler = fatalErrorHandler; + this.historyServerArchivist = historyServerArchivist; + this.metricQueryServiceAddress = metricQueryServiceAddress; + this.jobManagerRunnerFactory = jobManagerRunnerFactory; + } + + @Nonnull + public Configuration getConfiguration() { + return configuration; + } + + @Nonnull + public HighAvailabilityServices getHighAvailabilityServices() { + return highAvailabilityServices; + } + + @Nonnull + public GatewayRetriever getResourceManagerGatewayRetriever() { + return resourceManagerGatewayRetriever; + } + + @Nonnull + public BlobServer getBlobServer() { + return blobServer; + } + + @Nonnull + public HeartbeatServices getHeartbeatServices() { + return heartbeatServices; + } + + @Nonnull + public JobManagerMetricGroup getJobManagerMetricGroup() { + return jobManagerMetricGroup; + } + + @Nonnull + public ArchivedExecutionGraphStore getArchivedExecutionGraphStore() { + return archivedExecutionGraphStore; + } + + @Nonnull + public FatalErrorHandler getFatalErrorHandler() { + return fatalErrorHandler; + } + + @Nonnull + public HistoryServerArchivist getHistoryServerArchivist() { + return historyServerArchivist; + } + + @Nullable + public String getMetricQueryServiceAddress() { + return metricQueryServiceAddress; + } + + @Nonnull + JobManagerRunnerFactory getJobManagerRunnerFactory() { + return jobManagerRunnerFactory; + } + + public static DispatcherServices from(@Nonnull PartialDispatcherServices partialDispatcherServices, @Nonnull JobManagerRunnerFactory jobManagerRunnerFactory) { + return new DispatcherServices( + partialDispatcherServices.getConfiguration(), + partialDispatcherServices.getHighAvailabilityServices(), + partialDispatcherServices.getResourceManagerGatewayRetriever(), + partialDispatcherServices.getBlobServer(), + partialDispatcherServices.getHeartbeatServices(), + partialDispatcherServices.getJobManagerMetricGroup(), + partialDispatcherServices.getArchivedExecutionGraphStore(), + partialDispatcherServices.getFatalErrorHandler(), + partialDispatcherServices.getHistoryServerArchivist(), + partialDispatcherServices.getMetricQueryServiceAddress(), + jobManagerRunnerFactory); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStore.java index 652607241456cd75b0d26086316480cc6f8f7f9c..bd607e3fee83a871cb9e20e448c324763c3982d5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStore.java @@ -83,6 +83,7 @@ public class FileArchivedExecutionGraphStore implements ArchivedExecutionGraphSt public FileArchivedExecutionGraphStore( File rootDir, Time expirationTime, + int maximumCapacity, long maximumCacheSizeBytes, ScheduledExecutor scheduledExecutor, Ticker ticker) throws IOException { @@ -102,6 +103,7 @@ public class FileArchivedExecutionGraphStore implements ArchivedExecutionGraphSt "The storage directory must exist and be a directory."); this.jobDetailsCache = CacheBuilder.newBuilder() .expireAfterWrite(expirationTime.toMilliseconds(), TimeUnit.MILLISECONDS) + .maximumSize(maximumCapacity) .removalListener( (RemovalListener) notification -> deleteExecutionGraphFile(notification.getKey())) .ticker(ticker) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/JobDispatcherFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/JobDispatcherFactory.java index f6c22c2fed221740a283a2a684fb5168395a0195..6ca6157351da4f91c695fd255b4d389ad13d223f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/JobDispatcherFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/JobDispatcherFactory.java @@ -19,26 +19,19 @@ package org.apache.flink.runtime.dispatcher; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; import org.apache.flink.runtime.entrypoint.component.JobGraphRetriever; -import org.apache.flink.runtime.heartbeat.HeartbeatServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; -import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; -import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; import static org.apache.flink.runtime.entrypoint.ClusterEntrypoint.EXECUTION_MODE; /** * {@link DispatcherFactory} which creates a {@link MiniDispatcher}. */ -public class JobDispatcherFactory implements DispatcherFactory { +public class JobDispatcherFactory implements DispatcherFactory { private final JobGraphRetriever jobGraphRetriever; @@ -48,17 +41,9 @@ public class JobDispatcherFactory implements DispatcherFactory { @Override public MiniDispatcher createDispatcher( - Configuration configuration, - RpcService rpcService, - HighAvailabilityServices highAvailabilityServices, - GatewayRetriever resourceManagerGatewayRetriever, - BlobServer blobServer, - HeartbeatServices heartbeatServices, - JobManagerMetricGroup jobManagerMetricGroup, - @Nullable String metricQueryServiceAddress, - ArchivedExecutionGraphStore archivedExecutionGraphStore, - FatalErrorHandler fatalErrorHandler, - HistoryServerArchivist historyServerArchivist) throws Exception { + @Nonnull RpcService rpcService, + @Nonnull PartialDispatcherServices partialDispatcherServices) throws Exception { + final Configuration configuration = partialDispatcherServices.getConfiguration(); final JobGraph jobGraph = jobGraphRetriever.retrieveJobGraph(configuration); final String executionModeValue = configuration.getString(EXECUTION_MODE); @@ -68,17 +53,7 @@ public class JobDispatcherFactory implements DispatcherFactory { return new MiniDispatcher( rpcService, getEndpointId(), - configuration, - highAvailabilityServices, - resourceManagerGatewayRetriever, - blobServer, - heartbeatServices, - jobManagerMetricGroup, - metricQueryServiceAddress, - archivedExecutionGraphStore, - DefaultJobManagerRunnerFactory.INSTANCE, - fatalErrorHandler, - historyServerArchivist, + DispatcherServices.from(partialDispatcherServices, DefaultJobManagerRunnerFactory.INSTANCE), jobGraph, executionMode); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java index eaea0b9235878dc658361e03b051235db58e1228..2fc805c34fbc7f30bf8b388cfa9d4bf9dfef8632 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/MiniDispatcher.java @@ -20,26 +20,16 @@ package org.apache.flink.runtime.dispatcher; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.time.Time; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.clusterframework.ApplicationStatus; import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; import org.apache.flink.runtime.entrypoint.JobClusterEntrypoint; import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; -import org.apache.flink.runtime.heartbeat.HeartbeatServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobmaster.JobResult; import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; -import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; -import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; import org.apache.flink.util.FlinkException; -import javax.annotation.Nullable; - import java.util.concurrent.CompletableFuture; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -56,46 +46,19 @@ public class MiniDispatcher extends Dispatcher { private final JobClusterEntrypoint.ExecutionMode executionMode; - private final CompletableFuture jobTerminationFuture; - public MiniDispatcher( RpcService rpcService, String endpointId, - Configuration configuration, - HighAvailabilityServices highAvailabilityServices, - GatewayRetriever resourceManagerGatewayRetriever, - BlobServer blobServer, - HeartbeatServices heartbeatServices, - JobManagerMetricGroup jobManagerMetricGroup, - @Nullable String metricQueryServiceAddress, - ArchivedExecutionGraphStore archivedExecutionGraphStore, - JobManagerRunnerFactory jobManagerRunnerFactory, - FatalErrorHandler fatalErrorHandler, - HistoryServerArchivist historyServerArchivist, + DispatcherServices dispatcherServices, JobGraph jobGraph, JobClusterEntrypoint.ExecutionMode executionMode) throws Exception { super( rpcService, endpointId, - configuration, - highAvailabilityServices, - new SingleJobJobGraphStore(jobGraph), - resourceManagerGatewayRetriever, - blobServer, - heartbeatServices, - jobManagerMetricGroup, - metricQueryServiceAddress, - archivedExecutionGraphStore, - jobManagerRunnerFactory, - fatalErrorHandler, - historyServerArchivist); + dispatcherServices, + new SingleJobJobGraphStore(jobGraph)); this.executionMode = checkNotNull(executionMode); - this.jobTerminationFuture = new CompletableFuture<>(); - } - - public CompletableFuture getJobTerminationFuture() { - return jobTerminationFuture; } @Override @@ -124,7 +87,7 @@ public class MiniDispatcher extends Dispatcher { ApplicationStatus status = result.getSerializedThrowable().isPresent() ? ApplicationStatus.FAILED : ApplicationStatus.SUCCEEDED; - jobTerminationFuture.complete(status); + shutDownFuture.complete(status); }); } @@ -137,7 +100,7 @@ public class MiniDispatcher extends Dispatcher { if (executionMode == ClusterEntrypoint.ExecutionMode.DETACHED) { // shut down since we don't have to wait for the execution result retrieval - jobTerminationFuture.complete(ApplicationStatus.fromJobStatus(archivedExecutionGraph.getState())); + shutDownFuture.complete(ApplicationStatus.fromJobStatus(archivedExecutionGraph.getState())); } } @@ -146,6 +109,6 @@ public class MiniDispatcher extends Dispatcher { super.jobNotFinished(jobId); // shut down since we have done our job - jobTerminationFuture.complete(ApplicationStatus.UNKNOWN); + shutDownFuture.complete(ApplicationStatus.UNKNOWN); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/PartialDispatcherServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/PartialDispatcherServices.java new file mode 100644 index 0000000000000000000000000000000000000000..3425ec71513b267a187daaf8fbc7af7666023d19 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/PartialDispatcherServices.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.dispatcher; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.blob.BlobServer; +import org.apache.flink.runtime.heartbeat.HeartbeatServices; +import org.apache.flink.runtime.highavailability.HighAvailabilityServices; +import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; +import org.apache.flink.runtime.rpc.FatalErrorHandler; +import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Partial {@link DispatcherServices} services container which needs to + * be completed before being given to the {@link Dispatcher}. + */ +public class PartialDispatcherServices { + + @Nonnull + private final Configuration configuration; + + @Nonnull + private final HighAvailabilityServices highAvailabilityServices; + + @Nonnull + private final GatewayRetriever resourceManagerGatewayRetriever; + + @Nonnull + private final BlobServer blobServer; + + @Nonnull + private final HeartbeatServices heartbeatServices; + + @Nonnull + private final JobManagerMetricGroup jobManagerMetricGroup; + + @Nonnull + private final ArchivedExecutionGraphStore archivedExecutionGraphStore; + + @Nonnull + private final FatalErrorHandler fatalErrorHandler; + + @Nonnull + private final HistoryServerArchivist historyServerArchivist; + + @Nullable + private final String metricQueryServiceAddress; + + public PartialDispatcherServices( + @Nonnull Configuration configuration, + @Nonnull HighAvailabilityServices highAvailabilityServices, + @Nonnull GatewayRetriever resourceManagerGatewayRetriever, + @Nonnull BlobServer blobServer, + @Nonnull HeartbeatServices heartbeatServices, + @Nonnull JobManagerMetricGroup jobManagerMetricGroup, + @Nonnull ArchivedExecutionGraphStore archivedExecutionGraphStore, + @Nonnull FatalErrorHandler fatalErrorHandler, + @Nonnull HistoryServerArchivist historyServerArchivist, + @Nullable String metricQueryServiceAddress) { + this.configuration = configuration; + this.highAvailabilityServices = highAvailabilityServices; + this.resourceManagerGatewayRetriever = resourceManagerGatewayRetriever; + this.blobServer = blobServer; + this.heartbeatServices = heartbeatServices; + this.jobManagerMetricGroup = jobManagerMetricGroup; + this.archivedExecutionGraphStore = archivedExecutionGraphStore; + this.fatalErrorHandler = fatalErrorHandler; + this.historyServerArchivist = historyServerArchivist; + this.metricQueryServiceAddress = metricQueryServiceAddress; + } + + @Nonnull + public Configuration getConfiguration() { + return configuration; + } + + @Nonnull + public HighAvailabilityServices getHighAvailabilityServices() { + return highAvailabilityServices; + } + + @Nonnull + public GatewayRetriever getResourceManagerGatewayRetriever() { + return resourceManagerGatewayRetriever; + } + + @Nonnull + public BlobServer getBlobServer() { + return blobServer; + } + + @Nonnull + public HeartbeatServices getHeartbeatServices() { + return heartbeatServices; + } + + @Nonnull + public JobManagerMetricGroup getJobManagerMetricGroup() { + return jobManagerMetricGroup; + } + + @Nonnull + public ArchivedExecutionGraphStore getArchivedExecutionGraphStore() { + return archivedExecutionGraphStore; + } + + @Nonnull + public FatalErrorHandler getFatalErrorHandler() { + return fatalErrorHandler; + } + + @Nonnull + public HistoryServerArchivist getHistoryServerArchivist() { + return historyServerArchivist; + } + + @Nullable + public String getMetricQueryServiceAddress() { + return metricQueryServiceAddress; + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/SessionDispatcherFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/SessionDispatcherFactory.java index 18fb0be9374f2a3865aaedca5a9d07910e02786e..a87992bb20a06671fae16ccd5e1a09afb96dcce3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/SessionDispatcherFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/SessionDispatcherFactory.java @@ -18,51 +18,22 @@ package org.apache.flink.runtime.dispatcher; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.blob.BlobServer; -import org.apache.flink.runtime.heartbeat.HeartbeatServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; -import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; -import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; - -import javax.annotation.Nullable; /** * {@link DispatcherFactory} which creates a {@link StandaloneDispatcher}. */ -public enum SessionDispatcherFactory implements DispatcherFactory { +public enum SessionDispatcherFactory implements DispatcherFactory { INSTANCE; @Override - public Dispatcher createDispatcher( - Configuration configuration, - RpcService rpcService, - HighAvailabilityServices highAvailabilityServices, - GatewayRetriever resourceManagerGatewayRetriever, - BlobServer blobServer, - HeartbeatServices heartbeatServices, - JobManagerMetricGroup jobManagerMetricGroup, - @Nullable String metricQueryServiceAddress, - ArchivedExecutionGraphStore archivedExecutionGraphStore, - FatalErrorHandler fatalErrorHandler, - HistoryServerArchivist historyServerArchivist) throws Exception { + public StandaloneDispatcher createDispatcher( + RpcService rpcService, + PartialDispatcherServices partialDispatcherServices) throws Exception { // create the default dispatcher return new StandaloneDispatcher( rpcService, getEndpointId(), - configuration, - highAvailabilityServices, - resourceManagerGatewayRetriever, - blobServer, - heartbeatServices, - jobManagerMetricGroup, - metricQueryServiceAddress, - archivedExecutionGraphStore, - DefaultJobManagerRunnerFactory.INSTANCE, - fatalErrorHandler, - historyServerArchivist); + DispatcherServices.from(partialDispatcherServices, DefaultJobManagerRunnerFactory.INSTANCE)); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java index bd14ca9f039f789490f4f0b2406e6eff34a3384e..90b7faa647a4b38a2d1d9961efcbfac99aaec631 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/StandaloneDispatcher.java @@ -18,19 +18,9 @@ package org.apache.flink.runtime.dispatcher; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.blob.BlobServer; -import org.apache.flink.runtime.heartbeat.HeartbeatServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobmaster.JobMaster; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; -import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; -import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; - -import javax.annotation.Nullable; /** * Dispatcher implementation which spawns a {@link JobMaster} for each @@ -41,31 +31,11 @@ public class StandaloneDispatcher extends Dispatcher { public StandaloneDispatcher( RpcService rpcService, String endpointId, - Configuration configuration, - HighAvailabilityServices highAvailabilityServices, - GatewayRetriever resourceManagerGatewayRetriever, - BlobServer blobServer, - HeartbeatServices heartbeatServices, - JobManagerMetricGroup jobManagerMetricGroup, - @Nullable String metricQueryServiceAddress, - ArchivedExecutionGraphStore archivedExecutionGraphStore, - JobManagerRunnerFactory jobManagerRunnerFactory, - FatalErrorHandler fatalErrorHandler, - HistoryServerArchivist historyServerArchivist) throws Exception { + DispatcherServices dispatcherServices) throws Exception { super( rpcService, endpointId, - configuration, - highAvailabilityServices, - highAvailabilityServices.getJobGraphStore(), - resourceManagerGatewayRetriever, - blobServer, - heartbeatServices, - jobManagerMetricGroup, - metricQueryServiceAddress, - archivedExecutionGraphStore, - jobManagerRunnerFactory, - fatalErrorHandler, - historyServerArchivist); + dispatcherServices, + dispatcherServices.getHighAvailabilityServices().getJobGraphStore()); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunner.java new file mode 100644 index 0000000000000000000000000000000000000000..2157564cbcc03bfb721203ffbccf16e7b0596718 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunner.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.dispatcher.runner; + +import org.apache.flink.runtime.clusterframework.ApplicationStatus; +import org.apache.flink.runtime.dispatcher.Dispatcher; +import org.apache.flink.util.AutoCloseableAsync; + +import java.util.concurrent.CompletableFuture; + +/** + * The {@link DispatcherRunner} encapsulates how a {@link Dispatcher} is being executed. + */ +public interface DispatcherRunner extends AutoCloseableAsync { + + /** + * Get the currently running {@link Dispatcher}. + * + * @return the currently running dispatcher + */ + Dispatcher getDispatcher(); + + /** + * Return shut down future of this runner. The shut down future is being + * completed with the final {@link ApplicationStatus} once the runner wants + * to shut down. + * + * @return future with the final application status + */ + CompletableFuture getShutDownFuture(); +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunnerFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunnerFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..9cf8232563a4c1c41d7693aaef86ceee8dd54c3e --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunnerFactory.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.dispatcher.runner; + +import org.apache.flink.runtime.dispatcher.PartialDispatcherServices; +import org.apache.flink.runtime.rpc.RpcService; + +import java.util.concurrent.Executor; + +/** + * Factory interface for the {@link DispatcherRunner}. + */ +public interface DispatcherRunnerFactory { + + DispatcherRunner createDispatcherRunner( + RpcService rpcService, + Executor ioExecutor, + PartialDispatcherServices partialDispatcherServices) throws Exception; +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunnerFactoryImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunnerFactoryImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..dc2654714ad96d2a19f5834dca37a670cfa97a88 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunnerFactoryImpl.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.dispatcher.runner; + +import org.apache.flink.runtime.dispatcher.DispatcherFactory; +import org.apache.flink.runtime.dispatcher.PartialDispatcherServices; +import org.apache.flink.runtime.dispatcher.StandaloneDispatcher; +import org.apache.flink.runtime.rpc.RpcService; + +import java.util.concurrent.Executor; + +/** + * Factory which creates a {@link DispatcherRunnerImpl} which runs a {@link StandaloneDispatcher}. + */ +public class DispatcherRunnerFactoryImpl implements DispatcherRunnerFactory { + + private final DispatcherFactory dispatcherFactory; + + public DispatcherRunnerFactoryImpl(DispatcherFactory dispatcherFactory) { + this.dispatcherFactory = dispatcherFactory; + } + + @Override + public DispatcherRunnerImpl createDispatcherRunner( + RpcService rpcService, + Executor ioExecutor, + PartialDispatcherServices partialDispatcherServices) throws Exception { + return new DispatcherRunnerImpl(dispatcherFactory, rpcService, partialDispatcherServices); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/SessionDispatcherResourceManagerComponent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunnerImpl.java similarity index 41% rename from flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/SessionDispatcherResourceManagerComponent.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunnerImpl.java index 8be7b74c86f19b65d5e05c030443048216621e60..a5f7b6a58335f2ed119e7736849e83832b0186bc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/SessionDispatcherResourceManagerComponent.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/dispatcher/runner/DispatcherRunnerImpl.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,25 +16,46 @@ * limitations under the License. */ -package org.apache.flink.runtime.entrypoint.component; +package org.apache.flink.runtime.dispatcher.runner; +import org.apache.flink.runtime.clusterframework.ApplicationStatus; import org.apache.flink.runtime.dispatcher.Dispatcher; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManager; -import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint; +import org.apache.flink.runtime.dispatcher.DispatcherFactory; +import org.apache.flink.runtime.dispatcher.PartialDispatcherServices; +import org.apache.flink.runtime.rpc.RpcService; + +import java.util.concurrent.CompletableFuture; /** - * {@link DispatcherResourceManagerComponent} used by session clusters. + * Runner responsible for executing a {@link Dispatcher} or a subclass thereof. */ -class SessionDispatcherResourceManagerComponent extends DispatcherResourceManagerComponent { - SessionDispatcherResourceManagerComponent( - Dispatcher dispatcher, - ResourceManager resourceManager, - LeaderRetrievalService dispatcherLeaderRetrievalService, - LeaderRetrievalService resourceManagerRetrievalService, - WebMonitorEndpoint webMonitorEndpoint, - JobManagerMetricGroup jobManagerMetricGroup) { - super(dispatcher, resourceManager, dispatcherLeaderRetrievalService, resourceManagerRetrievalService, webMonitorEndpoint, jobManagerMetricGroup); +class DispatcherRunnerImpl implements DispatcherRunner { + + private final Dispatcher dispatcher; + + DispatcherRunnerImpl( + DispatcherFactory dispatcherFactory, + RpcService rpcService, + PartialDispatcherServices partialDispatcherServices) throws Exception { + this.dispatcher = dispatcherFactory.createDispatcher( + rpcService, + partialDispatcherServices); + + dispatcher.start(); + } + + @Override + public Dispatcher getDispatcher() { + return dispatcher; + } + + @Override + public CompletableFuture closeAsync() { + return dispatcher.closeAsync(); + } + + @Override + public CompletableFuture getShutDownFuture() { + return dispatcher.getShutDownFuture(); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java index 0538becf3b24e54c7a8fbc024d6a8172ea94bc95..68bb95bb529e197bea3d6862cd8d1edc4ab2c30a 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java @@ -46,10 +46,12 @@ import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; import org.apache.flink.runtime.metrics.MetricRegistryConfiguration; import org.apache.flink.runtime.metrics.MetricRegistryImpl; import org.apache.flink.runtime.metrics.ReporterSetup; +import org.apache.flink.runtime.metrics.groups.ProcessMetricGroup; import org.apache.flink.runtime.metrics.util.MetricUtils; import org.apache.flink.runtime.resourcemanager.ResourceManager; import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; +import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils; import org.apache.flink.runtime.security.SecurityConfiguration; import org.apache.flink.runtime.security.SecurityContext; @@ -116,11 +118,14 @@ public abstract class ClusterEntrypoint implements AutoCloseableAsync, FatalErro private final AtomicBoolean isShutDown = new AtomicBoolean(false); @GuardedBy("lock") - private DispatcherResourceManagerComponent clusterComponent; + private DispatcherResourceManagerComponent clusterComponent; @GuardedBy("lock") private MetricRegistryImpl metricRegistry; + @GuardedBy("lock") + private ProcessMetricGroup processMetricGroup; + @GuardedBy("lock") private HighAvailabilityServices haServices; @@ -205,10 +210,11 @@ public abstract class ClusterEntrypoint implements AutoCloseableAsync, FatalErro configuration.setString(JobManagerOptions.ADDRESS, commonRpcService.getAddress()); configuration.setInteger(JobManagerOptions.PORT, commonRpcService.getPort()); - final DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory(configuration); + final DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createDispatcherResourceManagerComponentFactory(configuration); clusterComponent = dispatcherResourceManagerComponentFactory.create( configuration, + ioExecutor, commonRpcService, haServices, blobServer, @@ -263,6 +269,13 @@ public abstract class ClusterEntrypoint implements AutoCloseableAsync, FatalErro final RpcService metricQueryServiceRpcService = MetricUtils.startMetricsRpcService(configuration, bindAddress); metricRegistry.startQueryService(metricQueryServiceRpcService, null); + final String hostname = RpcUtils.getHostname(commonRpcService); + + processMetricGroup = MetricUtils.instantiateProcessMetricGroup( + metricRegistry, + hostname, + ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration)); + archivedExecutionGraphStore = createSerializableExecutionGraphStore(configuration, commonRpcService.getScheduledExecutor()); } } @@ -349,6 +362,10 @@ public abstract class ClusterEntrypoint implements AutoCloseableAsync, FatalErro } } + if (processMetricGroup != null) { + processMetricGroup.close(); + } + if (metricRegistry != null) { terminationFutures.add(metricRegistry.shutdown()); } @@ -459,7 +476,7 @@ public abstract class ClusterEntrypoint implements AutoCloseableAsync, FatalErro // Abstract methods // -------------------------------------------------- - protected abstract DispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration); + protected abstract DispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration); protected abstract ArchivedExecutionGraphStore createSerializableExecutionGraphStore( Configuration configuration, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java index 1fb693cf83baa0210dcea6fddce40bf5e80f85d6..42daeae0138a75a89469c60d7fc2e72b158b721b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/SessionClusterEntrypoint.java @@ -47,11 +47,13 @@ public abstract class SessionClusterEntrypoint extends ClusterEntrypoint { final File tmpDir = new File(ConfigurationUtils.parseTempDirectories(configuration)[0]); final Time expirationTime = Time.seconds(configuration.getLong(JobManagerOptions.JOB_STORE_EXPIRATION_TIME)); + final int maximumCapacity = configuration.getInteger(JobManagerOptions.JOB_STORE_MAX_CAPACITY); final long maximumCacheSizeBytes = configuration.getLong(JobManagerOptions.JOB_STORE_CACHE_SIZE); return new FileArchivedExecutionGraphStore( tmpDir, expirationTime, + maximumCapacity, maximumCacheSizeBytes, scheduledExecutor, Ticker.systemTicker()); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/StandaloneSessionClusterEntrypoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/StandaloneSessionClusterEntrypoint.java index 127fc8b983120c1580ead6984be52113277175f4..e549ca9c5c181736f9a338971a69b2e0e4d1b936 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/StandaloneSessionClusterEntrypoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/StandaloneSessionClusterEntrypoint.java @@ -19,8 +19,7 @@ package org.apache.flink.runtime.entrypoint; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponentFactory; -import org.apache.flink.runtime.entrypoint.component.SessionDispatcherResourceManagerComponentFactory; +import org.apache.flink.runtime.entrypoint.component.DefaultDispatcherResourceManagerComponentFactory; import org.apache.flink.runtime.entrypoint.parser.CommandLineParser; import org.apache.flink.runtime.resourcemanager.StandaloneResourceManagerFactory; import org.apache.flink.runtime.util.EnvironmentInformation; @@ -37,8 +36,8 @@ public class StandaloneSessionClusterEntrypoint extends SessionClusterEntrypoint } @Override - protected DispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration) { - return new SessionDispatcherResourceManagerComponentFactory(StandaloneResourceManagerFactory.INSTANCE); + protected DefaultDispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration) { + return DefaultDispatcherResourceManagerComponentFactory.createSessionComponentFactory(StandaloneResourceManagerFactory.INSTANCE); } public static void main(String[] args) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/AbstractDispatcherResourceManagerComponentFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DefaultDispatcherResourceManagerComponentFactory.java similarity index 73% rename from flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/AbstractDispatcherResourceManagerComponentFactory.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DefaultDispatcherResourceManagerComponentFactory.java index 4a51d7575bc8faaf43304067abe2d19262115ee9..9b72997cdc4cf340e5bf0e84bd0618bc6f60f16b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/AbstractDispatcherResourceManagerComponentFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DefaultDispatcherResourceManagerComponentFactory.java @@ -20,36 +20,42 @@ package org.apache.flink.runtime.entrypoint.component; import org.apache.flink.api.common.time.Time; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.ConfigurationUtils; import org.apache.flink.configuration.MetricOptions; import org.apache.flink.configuration.RestOptions; import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.dispatcher.ArchivedExecutionGraphStore; -import org.apache.flink.runtime.dispatcher.Dispatcher; -import org.apache.flink.runtime.dispatcher.DispatcherFactory; import org.apache.flink.runtime.dispatcher.DispatcherGateway; import org.apache.flink.runtime.dispatcher.DispatcherId; import org.apache.flink.runtime.dispatcher.HistoryServerArchivist; +import org.apache.flink.runtime.dispatcher.JobDispatcherFactory; +import org.apache.flink.runtime.dispatcher.PartialDispatcherServices; +import org.apache.flink.runtime.dispatcher.SessionDispatcherFactory; +import org.apache.flink.runtime.dispatcher.runner.DispatcherRunner; +import org.apache.flink.runtime.dispatcher.runner.DispatcherRunnerFactory; +import org.apache.flink.runtime.dispatcher.runner.DispatcherRunnerFactoryImpl; import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.runtime.metrics.MetricRegistry; import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.metrics.util.MetricUtils; import org.apache.flink.runtime.resourcemanager.ResourceManager; import org.apache.flink.runtime.resourcemanager.ResourceManagerFactory; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.resourcemanager.ResourceManagerId; +import org.apache.flink.runtime.rest.JobRestEndpointFactory; import org.apache.flink.runtime.rest.RestEndpointFactory; +import org.apache.flink.runtime.rest.SessionRestEndpointFactory; import org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcher; import org.apache.flink.runtime.rest.handler.legacy.metrics.MetricFetcherImpl; import org.apache.flink.runtime.rest.handler.legacy.metrics.VoidMetricFetcher; import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; -import org.apache.flink.runtime.webmonitor.RestfulGateway; +import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint; import org.apache.flink.runtime.webmonitor.retriever.LeaderGatewayRetriever; import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever; @@ -65,39 +71,38 @@ import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Collection; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; /** * Abstract class which implements the creation of the {@link DispatcherResourceManagerComponent} components. - * - * @param type of the {@link Dispatcher} - * @param type of the {@link RestfulGateway} given to the {@link WebMonitorEndpoint} */ -public abstract class AbstractDispatcherResourceManagerComponentFactory implements DispatcherResourceManagerComponentFactory { +public class DefaultDispatcherResourceManagerComponentFactory implements DispatcherResourceManagerComponentFactory { private final Logger log = LoggerFactory.getLogger(getClass()); @Nonnull - private final DispatcherFactory dispatcherFactory; + private final DispatcherRunnerFactory dispatcherRunnerFactory; @Nonnull private final ResourceManagerFactory resourceManagerFactory; @Nonnull - private final RestEndpointFactory restEndpointFactory; + private final RestEndpointFactory restEndpointFactory; - public AbstractDispatcherResourceManagerComponentFactory( - @Nonnull DispatcherFactory dispatcherFactory, + DefaultDispatcherResourceManagerComponentFactory( + @Nonnull DispatcherRunnerFactory dispatcherRunnerFactory, @Nonnull ResourceManagerFactory resourceManagerFactory, - @Nonnull RestEndpointFactory restEndpointFactory) { - this.dispatcherFactory = dispatcherFactory; + @Nonnull RestEndpointFactory restEndpointFactory) { + this.dispatcherRunnerFactory = dispatcherRunnerFactory; this.resourceManagerFactory = resourceManagerFactory; this.restEndpointFactory = restEndpointFactory; } @Override - public DispatcherResourceManagerComponent create( + public DispatcherResourceManagerComponent create( Configuration configuration, + Executor ioExecutor, RpcService rpcService, HighAvailabilityServices highAvailabilityServices, BlobServer blobServer, @@ -109,10 +114,11 @@ public abstract class AbstractDispatcherResourceManagerComponentFactory webMonitorEndpoint = null; + WebMonitorEndpoint webMonitorEndpoint = null; ResourceManager resourceManager = null; JobManagerMetricGroup jobManagerMetricGroup = null; - T dispatcher = null; + ResourceManagerMetricGroup resourceManagerMetricGroup = null; + DispatcherRunner dispatcherRunner = null; try { dispatcherLeaderRetrievalService = highAvailabilityServices.getDispatcherLeaderRetriever(); @@ -160,55 +166,56 @@ public abstract class AbstractDispatcherResourceManagerComponentFactory terminationFuture = FutureUtils.completeAll(terminationFutures); @@ -254,20 +261,28 @@ public abstract class AbstractDispatcherResourceManagerComponentFactory resourceManagerFactory) { + return new DefaultDispatcherResourceManagerComponentFactory( + new DispatcherRunnerFactoryImpl(SessionDispatcherFactory.INSTANCE), + resourceManagerFactory, + SessionRestEndpointFactory.INSTANCE); } - protected abstract DispatcherResourceManagerComponent createDispatcherResourceManagerComponent( - T dispatcher, - ResourceManager resourceManager, - LeaderRetrievalService dispatcherLeaderRetrievalService, - LeaderRetrievalService resourceManagerRetrievalService, - WebMonitorEndpoint webMonitorEndpoint, - JobManagerMetricGroup jobManagerMetricGroup); + public static DefaultDispatcherResourceManagerComponentFactory createJobComponentFactory( + ResourceManagerFactory resourceManagerFactory, + JobGraphRetriever jobGraphRetriever) { + return new DefaultDispatcherResourceManagerComponentFactory( + new DispatcherRunnerFactoryImpl(new JobDispatcherFactory(jobGraphRetriever)), + resourceManagerFactory, + JobRestEndpointFactory.INSTANCE); + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DispatcherResourceManagerComponent.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DispatcherResourceManagerComponent.java index 700b3c40100dd0d54e901d02e4906e6092cded4a..770e92aea3bce9478e7bbe43358d169228cd6bd8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DispatcherResourceManagerComponent.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DispatcherResourceManagerComponent.java @@ -21,8 +21,8 @@ package org.apache.flink.runtime.entrypoint.component; import org.apache.flink.runtime.clusterframework.ApplicationStatus; import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.dispatcher.Dispatcher; +import org.apache.flink.runtime.dispatcher.runner.DispatcherRunner; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.ResourceManager; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint; @@ -41,10 +41,10 @@ import java.util.concurrent.atomic.AtomicBoolean; * Component which starts a {@link Dispatcher}, {@link ResourceManager} and {@link WebMonitorEndpoint} * in the same process. */ -public class DispatcherResourceManagerComponent implements AutoCloseableAsync { +public class DispatcherResourceManagerComponent implements AutoCloseableAsync { @Nonnull - private final T dispatcher; + private final DispatcherRunner dispatcherRunner; @Nonnull private final ResourceManager resourceManager; @@ -58,9 +58,6 @@ public class DispatcherResourceManagerComponent implements @Nonnull private final WebMonitorEndpoint webMonitorEndpoint; - @Nonnull - private final JobManagerMetricGroup jobManagerMetricGroup; - private final CompletableFuture terminationFuture; private final CompletableFuture shutDownFuture; @@ -68,18 +65,16 @@ public class DispatcherResourceManagerComponent implements private final AtomicBoolean isRunning = new AtomicBoolean(true); DispatcherResourceManagerComponent( - @Nonnull T dispatcher, + @Nonnull DispatcherRunner dispatcherRunner, @Nonnull ResourceManager resourceManager, @Nonnull LeaderRetrievalService dispatcherLeaderRetrievalService, @Nonnull LeaderRetrievalService resourceManagerRetrievalService, - @Nonnull WebMonitorEndpoint webMonitorEndpoint, - @Nonnull JobManagerMetricGroup jobManagerMetricGroup) { + @Nonnull WebMonitorEndpoint webMonitorEndpoint) { + this.dispatcherRunner = dispatcherRunner; this.resourceManager = resourceManager; - this.dispatcher = dispatcher; this.dispatcherLeaderRetrievalService = dispatcherLeaderRetrievalService; this.resourceManagerRetrievalService = resourceManagerRetrievalService; this.webMonitorEndpoint = webMonitorEndpoint; - this.jobManagerMetricGroup = jobManagerMetricGroup; this.terminationFuture = new CompletableFuture<>(); this.shutDownFuture = new CompletableFuture<>(); @@ -87,25 +82,7 @@ public class DispatcherResourceManagerComponent implements } private void registerShutDownFuture() { - terminationFuture.whenComplete( - (aVoid, throwable) -> { - if (throwable != null) { - shutDownFuture.completeExceptionally(throwable); - } else { - shutDownFuture.complete(ApplicationStatus.SUCCEEDED); - } - }); - - dispatcher - .getTerminationFuture() - .whenComplete( - (aVoid, throwable) -> { - if (throwable != null) { - shutDownFuture.completeExceptionally(throwable); - } else { - shutDownFuture.complete(ApplicationStatus.SUCCEEDED); - } - }); + FutureUtils.forward(dispatcherRunner.getShutDownFuture(), shutDownFuture); } public final CompletableFuture getShutDownFuture() { @@ -113,8 +90,8 @@ public class DispatcherResourceManagerComponent implements } @Nonnull - public T getDispatcher() { - return dispatcher; + public DispatcherRunner getDispatcherRunner() { + return dispatcherRunner; } @Nonnull @@ -169,7 +146,7 @@ public class DispatcherResourceManagerComponent implements exception = ExceptionUtils.firstOrSuppressed(e, exception); } - terminationFutures.add(dispatcher.closeAsync()); + terminationFutures.add(dispatcherRunner.closeAsync()); terminationFutures.add(resourceManager.closeAsync()); @@ -179,11 +156,7 @@ public class DispatcherResourceManagerComponent implements final CompletableFuture componentTerminationFuture = FutureUtils.completeAll(terminationFutures); - final CompletableFuture metricGroupTerminationFuture = FutureUtils.runAfterwards( - componentTerminationFuture, - jobManagerMetricGroup::close); - - metricGroupTerminationFuture.whenComplete((aVoid, throwable) -> { + componentTerminationFuture.whenComplete((aVoid, throwable) -> { if (throwable != null) { terminationFuture.completeExceptionally(throwable); } else { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DispatcherResourceManagerComponentFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DispatcherResourceManagerComponentFactory.java index 51ddb8f6df9216e0427190f6e833b11b9dfffdf5..752291d7307c603a8ee7ff796a5af012f2404a9a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DispatcherResourceManagerComponentFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/DispatcherResourceManagerComponentFactory.java @@ -21,7 +21,6 @@ package org.apache.flink.runtime.entrypoint.component; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.dispatcher.ArchivedExecutionGraphStore; -import org.apache.flink.runtime.dispatcher.Dispatcher; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.metrics.MetricRegistry; @@ -29,13 +28,16 @@ import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.runtime.webmonitor.retriever.MetricQueryServiceRetriever; +import java.util.concurrent.Executor; + /** * Factory for the {@link DispatcherResourceManagerComponent}. */ -public interface DispatcherResourceManagerComponentFactory { +public interface DispatcherResourceManagerComponentFactory { - DispatcherResourceManagerComponent create( + DispatcherResourceManagerComponent create( Configuration configuration, + Executor ioExecutor, RpcService rpcService, HighAvailabilityServices highAvailabilityServices, BlobServer blobServer, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/JobDispatcherResourceManagerComponentFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/JobDispatcherResourceManagerComponentFactory.java deleted file mode 100644 index c7ce14c1db3e61d15bbe82a5da34dd9630f2a3a0..0000000000000000000000000000000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/JobDispatcherResourceManagerComponentFactory.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.entrypoint.component; - -import org.apache.flink.runtime.dispatcher.JobDispatcherFactory; -import org.apache.flink.runtime.dispatcher.MiniDispatcher; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManager; -import org.apache.flink.runtime.resourcemanager.ResourceManagerFactory; -import org.apache.flink.runtime.rest.JobRestEndpointFactory; -import org.apache.flink.runtime.webmonitor.RestfulGateway; -import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint; - -import javax.annotation.Nonnull; - -/** - * {@link DispatcherResourceManagerComponentFactory} for a {@link JobDispatcherResourceManagerComponent}. - */ -public class JobDispatcherResourceManagerComponentFactory extends AbstractDispatcherResourceManagerComponentFactory { - - public JobDispatcherResourceManagerComponentFactory(@Nonnull ResourceManagerFactory resourceManagerFactory, @Nonnull JobGraphRetriever jobGraphRetriever) { - super(new JobDispatcherFactory(jobGraphRetriever), resourceManagerFactory, JobRestEndpointFactory.INSTANCE); - } - - @Override - protected DispatcherResourceManagerComponent createDispatcherResourceManagerComponent( - MiniDispatcher dispatcher, - ResourceManager resourceManager, - LeaderRetrievalService dispatcherLeaderRetrievalService, - LeaderRetrievalService resourceManagerRetrievalService, - WebMonitorEndpoint webMonitorEndpoint, - JobManagerMetricGroup jobManagerMetricGroup) { - return new JobDispatcherResourceManagerComponent( - dispatcher, - resourceManager, - dispatcherLeaderRetrievalService, - resourceManagerRetrievalService, - webMonitorEndpoint, - jobManagerMetricGroup); - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/SessionDispatcherResourceManagerComponentFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/SessionDispatcherResourceManagerComponentFactory.java deleted file mode 100644 index eb71e40c49e27c425a79f5bdb2ff798a7e31014c..0000000000000000000000000000000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/SessionDispatcherResourceManagerComponentFactory.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.entrypoint.component; - -import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.runtime.dispatcher.Dispatcher; -import org.apache.flink.runtime.dispatcher.DispatcherFactory; -import org.apache.flink.runtime.dispatcher.DispatcherGateway; -import org.apache.flink.runtime.dispatcher.SessionDispatcherFactory; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManager; -import org.apache.flink.runtime.resourcemanager.ResourceManagerFactory; -import org.apache.flink.runtime.rest.SessionRestEndpointFactory; -import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint; - -import javax.annotation.Nonnull; - -/** - * {@link DispatcherResourceManagerComponentFactory} for a {@link SessionDispatcherResourceManagerComponent}. - */ -public class SessionDispatcherResourceManagerComponentFactory extends AbstractDispatcherResourceManagerComponentFactory { - - public SessionDispatcherResourceManagerComponentFactory(@Nonnull ResourceManagerFactory resourceManagerFactory) { - this(SessionDispatcherFactory.INSTANCE, resourceManagerFactory); - } - - @VisibleForTesting - public SessionDispatcherResourceManagerComponentFactory( - @Nonnull DispatcherFactory dispatcherFactory, - @Nonnull ResourceManagerFactory resourceManagerFactory) { - super(dispatcherFactory, resourceManagerFactory, SessionRestEndpointFactory.INSTANCE); - } - - @Override - protected DispatcherResourceManagerComponent createDispatcherResourceManagerComponent( - Dispatcher dispatcher, - ResourceManager resourceManager, - LeaderRetrievalService dispatcherLeaderRetrievalService, - LeaderRetrievalService resourceManagerRetrievalService, - WebMonitorEndpoint webMonitorEndpoint, - JobManagerMetricGroup jobManagerMetricGroup) { - return new SessionDispatcherResourceManagerComponent( - dispatcher, - resourceManager, - dispatcherLeaderRetrievalService, - resourceManagerRetrievalService, - webMonitorEndpoint, - jobManagerMetricGroup); - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java index 7ba2353d3280c54c6d5db445f5cd88e938854d1b..d9cdf5a9e41b7fc722e35ebb6501dc503cde3eeb 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/Execution.java @@ -57,12 +57,12 @@ import org.apache.flink.runtime.shuffle.PartitionDescriptor; import org.apache.flink.runtime.shuffle.ProducerDescriptor; import org.apache.flink.runtime.shuffle.ShuffleDescriptor; import org.apache.flink.runtime.shuffle.ShuffleMaster; -import org.apache.flink.runtime.state.KeyGroupRangeAssignment; import org.apache.flink.runtime.taskmanager.TaskManagerLocation; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.OptionalFailure; +import org.apache.flink.util.Preconditions; import org.apache.flink.util.function.ThrowingRunnable; import org.slf4j.Logger; @@ -295,8 +295,7 @@ public class Execution implements AccessExecution, Archiveable registerProducedPartitions(TaskManagerLocation location) { + public CompletableFuture registerProducedPartitions(TaskManagerLocation location) { + Preconditions.checkState(isLegacyScheduling()); + return registerProducedPartitions(location, vertex.getExecutionGraph().getScheduleMode().allowLazyDeployment()); + } + + public CompletableFuture registerProducedPartitions( + TaskManagerLocation location, + boolean sendScheduleOrUpdateConsumersMessage) { + assertRunningInJobMasterMainThread(); return FutureUtils.thenApplyAsyncIfNotDone( - registerProducedPartitions(vertex, location, attemptId), + registerProducedPartitions(vertex, location, attemptId, sendScheduleOrUpdateConsumersMessage), vertex.getExecutionGraph().getJobMasterMainThreadExecutor(), producedPartitionsCache -> { producedPartitions = producedPartitionsCache; @@ -615,10 +621,10 @@ public class Execution implements AccessExecution, Archiveable> registerProducedPartitions( ExecutionVertex vertex, TaskManagerLocation location, - ExecutionAttemptID attemptId) { - ProducerDescriptor producerDescriptor = ProducerDescriptor.create(location, attemptId); + ExecutionAttemptID attemptId, + boolean sendScheduleOrUpdateConsumersMessage) { - boolean lazyScheduling = vertex.getExecutionGraph().getScheduleMode().allowLazyDeployment(); + ProducerDescriptor producerDescriptor = ProducerDescriptor.create(location, attemptId); Collection partitions = vertex.getProducedPartitions().values(); Collection> partitionRegistrations = @@ -637,7 +643,7 @@ public class Execution implements AccessExecution, Archiveable> consumers = partition.getConsumers(); - int maxParallelism = KeyGroupRangeAssignment.UPPER_BOUND_MAX_PARALLELISM; - if (!consumers.isEmpty()) { - List consumer = consumers.get(0); - ExecutionJobVertex consumerVertex = consumer.get(0).getTarget().getJobVertex(); - maxParallelism = consumerVertex.getMaxParallelism(); - } + Preconditions.checkArgument(!consumers.isEmpty(), "Currently there has to be exactly one consumer in real jobs"); + List consumer = consumers.get(0); + ExecutionJobVertex consumerVertex = consumer.get(0).getTarget().getJobVertex(); + int maxParallelism = consumerVertex.getMaxParallelism(); return maxParallelism; } @@ -753,7 +756,10 @@ public class Execution implements AccessExecution, Archiveable> userAccumulators, IOMetrics metrics) { - // skip release of partitions since this is only called if the TM actually sent the FAILED state update - // in this case all partitions have already been cleaned up - processFail(t, true, userAccumulators, metrics, false); + markFailed(t, userAccumulators, metrics, false); + } + + void markFailed(Throwable t, Map> userAccumulators, IOMetrics metrics, boolean fromSchedulerNg) { + processFail(t, true, userAccumulators, metrics, false, fromSchedulerNg); } @VisibleForTesting @@ -1179,11 +1196,15 @@ public class Execution implements AccessExecution, Archiveable> userAccumulators, IOMetrics metrics, boolean releasePartitions) { + private void processFail(Throwable t, boolean isCallback, Map> userAccumulators, IOMetrics metrics, boolean releasePartitions, boolean fromSchedulerNg) { // damn, we failed. This means only that we keep our books and notify our parent JobExecutionVertex // the actual computation on the task manager is cleaned up by the TaskManager that noticed the failure @@ -1211,7 +1232,15 @@ public class Execution implements AccessExecution, Archiveable { + transitionState(JobStatus.FAILED, cause); + onTerminalState(JobStatus.FAILED); + })); + } + private void onTerminalState(JobStatus status) { try { CheckpointCoordinator coord = this.checkpointCoordinator; @@ -1557,7 +1611,7 @@ public class ExecutionGraph implements AccessExecutionGraph { case FAILED: // this deserialization is exception-free accumulators = deserializeAccumulators(state); - attempt.markFailed(state.getError(userClassLoader), accumulators, state.getIOMetrics()); + attempt.markFailed(state.getError(userClassLoader), accumulators, state.getIOMetrics(), !isLegacyScheduling()); return true; default: @@ -1727,6 +1781,10 @@ public class ExecutionGraph implements AccessExecutionGraph { final ExecutionState newExecutionState, final Throwable error) { + if (!isLegacyScheduling()) { + return; + } + // see what this means for us. currently, the first FAILED state means -> FAILED if (newExecutionState == ExecutionState.FAILED) { final Throwable ex = error != null ? error : new FlinkException("Unknown Error (missing cause)"); @@ -1757,6 +1815,12 @@ public class ExecutionGraph implements AccessExecutionGraph { } } + void notifySchedulerNgAboutInternalTaskFailure(final ExecutionAttemptID attemptId, final Throwable t) { + if (internalTaskFailuresListener != null) { + internalTaskFailuresListener.notifyFailed(attemptId, t); + } + } + ShuffleMaster getShuffleMaster() { return shuffleMaster; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java index 6d262ee58deedd55a99f536c778324d700cac6f8..500691b0496b376af707590f2790856ea18fcb87 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/ExecutionVertex.java @@ -58,6 +58,7 @@ import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.IntStream; @@ -514,6 +515,18 @@ public class ExecutionVertex implements AccessExecutionVertex, Archiveable getPreferredLocationBasedOnState() { + if (currentExecution.getTaskRestore() != null) { + return Optional.ofNullable(getLatestPriorLocation()); + } + + return Optional.empty(); + } + /** * Gets the location preferences of the vertex's current task execution, as determined by the locations * of the predecessors from which it receives input data. @@ -599,61 +612,75 @@ public class ExecutionVertex implements AccessExecutionVertex, Archiveable allocateSlot( + public abstract CompletableFuture allocateSlot( SlotRequestId slotRequestId, ScheduledUnit scheduledUnit, SlotProfile slotProfile); @@ -70,14 +70,14 @@ abstract class SlotProviderStrategy { * @param slotSharingGroupId identifying the slot request to cancel * @param cause of the cancellation */ - void cancelSlotRequest( + public void cancelSlotRequest( SlotRequestId slotRequestId, @Nullable SlotSharingGroupId slotSharingGroupId, Throwable cause) { slotProvider.cancelSlotRequest(slotRequestId, slotSharingGroupId, cause); } - static SlotProviderStrategy from( + public static SlotProviderStrategy from( ScheduleMode scheduleMode, SlotProvider slotProvider, Time allocationTimeout, diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/AdaptedRestartPipelinedRegionStrategyNG.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/AdaptedRestartPipelinedRegionStrategyNG.java index 164cd91e8a064536e26cbef893d01edd7cb60f0d..eb43a2c35499b553ce6b4f14f0d29ca33bad00ed 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/AdaptedRestartPipelinedRegionStrategyNG.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/AdaptedRestartPipelinedRegionStrategyNG.java @@ -268,7 +268,7 @@ public class AdaptedRestartPipelinedRegionStrategyNG extends FailoverStrategy { } private ExecutionVertexID getExecutionVertexID(final ExecutionVertex vertex) { - return new ExecutionVertexID(vertex.getJobvertexId(), vertex.getParallelSubtaskIndex()); + return vertex.getID(); } private List sortVerticesTopologically(final Set vertices) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverRegion.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverRegion.java deleted file mode 100644 index 9a93bb8dbde3bfa973282c4821c6dcd81f1ed93e..0000000000000000000000000000000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverRegion.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.executiongraph.failover; - -import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.runtime.checkpoint.CheckpointException; -import org.apache.flink.runtime.checkpoint.CheckpointFailureReason; -import org.apache.flink.runtime.clusterframework.types.AllocationID; -import org.apache.flink.runtime.concurrent.FutureUtils; -import org.apache.flink.runtime.executiongraph.Execution; -import org.apache.flink.runtime.executiongraph.ExecutionGraph; -import org.apache.flink.runtime.executiongraph.ExecutionJobVertex; -import org.apache.flink.runtime.executiongraph.ExecutionVertex; -import org.apache.flink.runtime.executiongraph.GlobalModVersionMismatch; -import org.apache.flink.runtime.jobgraph.JobStatus; -import org.apache.flink.runtime.jobgraph.JobVertexID; -import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup; -import org.apache.flink.runtime.jobmanager.scheduler.LocationPreferenceConstraint; -import org.apache.flink.util.AbstractID; -import org.apache.flink.util.FlinkException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; - -import static org.apache.flink.util.Preconditions.checkNotNull; - -/** - * FailoverRegion manages the failover of a minimal pipeline connected sub graph. - * It will change from CREATED to CANCELING and then to CANCELLED and at last to RUNNING, - */ -public class FailoverRegion { - - private static final AtomicReferenceFieldUpdater STATE_UPDATER = - AtomicReferenceFieldUpdater.newUpdater(FailoverRegion.class, JobStatus.class, "state"); - - /** The log object used for debugging. */ - private static final Logger LOG = LoggerFactory.getLogger(FailoverRegion.class); - - // ------------------------------------------------------------------------ - - /** a unique id for debugging */ - private final AbstractID id = new AbstractID(); - - private final ExecutionGraph executionGraph; - - private final List connectedExecutionVertexes; - - private final Map tasks; - - /** Current status of the job execution */ - private volatile JobStatus state = JobStatus.RUNNING; - - public FailoverRegion( - ExecutionGraph executionGraph, - List connectedExecutions, - Map tasks) { - - this.executionGraph = checkNotNull(executionGraph); - this.connectedExecutionVertexes = checkNotNull(connectedExecutions); - this.tasks = checkNotNull(tasks); - - LOG.debug("Created failover region {} with vertices: {}", id, connectedExecutions); - } - - public void onExecutionFail(Execution taskExecution, Throwable cause) { - // TODO: check if need to failover the preceding region - if (!executionGraph.getRestartStrategy().canRestart()) { - // delegate the failure to a global fail that will check the restart strategy and not restart - executionGraph.failGlobal(cause); - } - else { - cancel(taskExecution.getGlobalModVersion()); - } - } - - private void allVerticesInTerminalState(long globalModVersionOfFailover) { - while (true) { - JobStatus curStatus = this.state; - if (curStatus.equals(JobStatus.CANCELLING)) { - if (transitionState(curStatus, JobStatus.CANCELED)) { - reset(globalModVersionOfFailover); - break; - } - } - else { - LOG.info("FailoverRegion {} is {} when allVerticesInTerminalState.", id, state); - break; - } - } - } - - public JobStatus getState() { - return state; - } - - // Notice the region to failover, - private void failover(long globalModVersionOfFailover) { - if (!executionGraph.getRestartStrategy().canRestart()) { - executionGraph.failGlobal(new FlinkException("RestartStrategy validate fail")); - } - else { - JobStatus curStatus = this.state; - if (curStatus.equals(JobStatus.RUNNING)) { - cancel(globalModVersionOfFailover); - } - else if (curStatus.equals(JobStatus.CANCELED)) { - reset(globalModVersionOfFailover); - } - else { - LOG.info("FailoverRegion {} is {} when notified to failover.", id, state); - } - } - } - - // cancel all executions in this sub graph - private void cancel(final long globalModVersionOfFailover) { - executionGraph.getJobMasterMainThreadExecutor().assertRunningInMainThread(); - while (true) { - JobStatus curStatus = this.state; - if (curStatus.equals(JobStatus.RUNNING)) { - if (transitionState(curStatus, JobStatus.CANCELLING)) { - - createTerminationFutureOverAllConnectedVertexes() - .thenAccept((nullptr) -> allVerticesInTerminalState(globalModVersionOfFailover)); - break; - } - } else { - LOG.info("FailoverRegion {} is {} when cancel.", id, state); - break; - } - } - } - - @VisibleForTesting - protected CompletableFuture createTerminationFutureOverAllConnectedVertexes() { - // we build a future that is complete once all vertices have reached a terminal state - final ArrayList> futures = new ArrayList<>(connectedExecutionVertexes.size()); - - // cancel all tasks (that still need cancelling) - for (ExecutionVertex vertex : connectedExecutionVertexes) { - futures.add(vertex.cancel()); - } - - return FutureUtils.waitForAll(futures); - } - - // reset all executions in this sub graph - private void reset(long globalModVersionOfFailover) { - try { - // reset all connected ExecutionVertexes - final Collection colGroups = new HashSet<>(); - final long restartTimestamp = System.currentTimeMillis(); - - for (ExecutionVertex ev : connectedExecutionVertexes) { - CoLocationGroup cgroup = ev.getJobVertex().getCoLocationGroup(); - if (cgroup != null && !colGroups.contains(cgroup)){ - cgroup.resetConstraints(); - colGroups.add(cgroup); - } - - ev.resetForNewExecution(restartTimestamp, globalModVersionOfFailover); - } - if (transitionState(JobStatus.CANCELED, JobStatus.CREATED)) { - restart(globalModVersionOfFailover); - } - else { - LOG.info("FailoverRegion {} switched from CANCELLING to CREATED fail, will fail this region again.", id); - failover(globalModVersionOfFailover); - } - } - catch (GlobalModVersionMismatch e) { - // happens when a global recovery happens concurrently to the regional recovery - // go back to a clean state - state = JobStatus.RUNNING; - } - catch (Throwable e) { - LOG.info("FailoverRegion {} reset fail, will failover again.", id); - failover(globalModVersionOfFailover); - } - } - - // restart all executions in this sub graph - private void restart(long globalModVersionOfFailover) { - try { - if (transitionState(JobStatus.CREATED, JobStatus.RUNNING)) { - // if we have checkpointed state, reload it into the executions - if (executionGraph.getCheckpointCoordinator() != null) { - // we abort pending checkpoints for - // i) enable new checkpoint could be triggered without waiting for last checkpoint expired. - // ii) ensure the EXACTLY_ONCE semantics if needed. - executionGraph.getCheckpointCoordinator().abortPendingCheckpoints( - new CheckpointException(CheckpointFailureReason.JOB_FAILOVER_REGION)); - - executionGraph.getCheckpointCoordinator().restoreLatestCheckpointedState( - tasks, false, true); - } - - HashSet previousAllocationsInRegion = new HashSet<>(connectedExecutionVertexes.size()); - for (ExecutionVertex connectedExecutionVertex : connectedExecutionVertexes) { - AllocationID latestPriorAllocation = connectedExecutionVertex.getLatestPriorAllocation(); - if (latestPriorAllocation != null) { - previousAllocationsInRegion.add(latestPriorAllocation); - } - } - - //TODO, use restart strategy to schedule them. - //restart all connected ExecutionVertexes - for (ExecutionVertex ev : connectedExecutionVertexes) { - try { - ev.scheduleForExecution( - executionGraph.getSlotProviderStrategy(), - LocationPreferenceConstraint.ANY, - previousAllocationsInRegion); // some inputs not belonging to the failover region might have failed concurrently - } - catch (Throwable e) { - failover(globalModVersionOfFailover); - } - } - } - else { - LOG.info("FailoverRegion {} switched from CREATED to RUNNING fail, will fail this region again.", id); - failover(globalModVersionOfFailover); - } - } catch (Exception e) { - LOG.info("FailoverRegion {} restart failed, failover again.", id, e); - failover(globalModVersionOfFailover); - } - } - - private boolean transitionState(JobStatus current, JobStatus newState) { - if (STATE_UPDATER.compareAndSet(this, current, newState)) { - LOG.info("FailoverRegion {} switched from state {} to {}.", id, current, newState); - return true; - } - else { - return false; - } - } - -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java index e80c6edf2a207c4469338adbd10c0a408214113a..e0818275fee0924c491c115c0843ba2325c3989b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/FailoverStrategyLoader.java @@ -41,8 +41,8 @@ public class FailoverStrategyLoader { /** Config name for the {@link AdaptedRestartPipelinedRegionStrategyNG}. */ public static final String PIPELINED_REGION_RESTART_STRATEGY_NAME = "region"; - /** Config name for the {@link RestartPipelinedRegionStrategy}. */ - public static final String LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME = "region-legacy"; + /** Config name for the {@link NoOpFailoverStrategy}. */ + public static final String NO_OP_FAILOVER_STRATEGY = "noop"; // ------------------------------------------------------------------------ @@ -50,7 +50,11 @@ public class FailoverStrategyLoader { * Loads a FailoverStrategy Factory from the given configuration. */ public static FailoverStrategy.Factory loadFailoverStrategy(Configuration config, @Nullable Logger logger) { - final String strategyParam = config.getString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY); + // The new generation scheduler does not depend on the FailoverStrategy loaded here. + // Therefore, we load a noop failover strategy if the new generation scheduler is configured. + final String strategyParam = config.getString(JobManagerOptions.SCHEDULER).equals("ng") ? + NO_OP_FAILOVER_STRATEGY : + config.getString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY); if (StringUtils.isNullOrWhitespaceOnly(strategyParam)) { if (logger != null) { @@ -68,12 +72,12 @@ public class FailoverStrategyLoader { case PIPELINED_REGION_RESTART_STRATEGY_NAME: return new AdaptedRestartPipelinedRegionStrategyNG.Factory(); - case LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME: - return new RestartPipelinedRegionStrategy.Factory(); - case INDIVIDUAL_RESTART_STRATEGY_NAME: return new RestartIndividualStrategy.Factory(); + case NO_OP_FAILOVER_STRATEGY: + return new NoOpFailoverStrategy.Factory(); + default: // we could interpret the parameter as a factory class name and instantiate that // for now we simply do not support this diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/NoOpFailoverStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/NoOpFailoverStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..0cc37808eb555f32573c1b7d05c99fd4e78bc9bb --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/NoOpFailoverStrategy.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.executiongraph.failover; + +import org.apache.flink.runtime.executiongraph.Execution; +import org.apache.flink.runtime.executiongraph.ExecutionGraph; +import org.apache.flink.runtime.executiongraph.ExecutionJobVertex; + +import java.util.List; + +/** + * FailoverStrategy that does not do anything. + */ +public class NoOpFailoverStrategy extends FailoverStrategy { + + @Override + public void onTaskFailure(final Execution taskExecution, final Throwable cause) { + } + + @Override + public void notifyNewVertices(final List newJobVerticesTopological) { + } + + @Override + public String getStrategyName() { + return "NoOp failover strategy"; + } + + public static class Factory implements FailoverStrategy.Factory { + + @Override + public FailoverStrategy create(final ExecutionGraph executionGraph) { + return new NoOpFailoverStrategy(); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java deleted file mode 100644 index 811c0a01456379fd0310b2b0e52c5c6d27a79be0..0000000000000000000000000000000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/RestartPipelinedRegionStrategy.java +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.executiongraph.failover; - -import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.runtime.executiongraph.Execution; -import org.apache.flink.runtime.executiongraph.ExecutionEdge; -import org.apache.flink.runtime.executiongraph.ExecutionGraph; -import org.apache.flink.runtime.executiongraph.ExecutionJobVertex; -import org.apache.flink.runtime.executiongraph.ExecutionVertex; -import org.apache.flink.runtime.executiongraph.IntermediateResult; -import org.apache.flink.runtime.jobgraph.JobVertexID; -import org.apache.flink.util.FlinkException; -import org.apache.flink.util.FlinkRuntimeException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.IdentityHashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.flink.util.Preconditions.checkNotNull; - -/** - * A failover strategy that restarts regions of the ExecutionGraph with state. A region is defined - * by this strategy as the weakly connected component of tasks that communicate via pipelined - * data exchange. - */ -public class RestartPipelinedRegionStrategy extends FailoverStrategy { - - /** The log object used for debugging. */ - private static final Logger LOG = LoggerFactory.getLogger(RestartPipelinedRegionStrategy.class); - - /** The execution graph on which this FailoverStrategy works */ - private final ExecutionGraph executionGraph; - - /** Fast lookup from vertex to failover region */ - private final HashMap vertexToRegion; - - /** - * Creates a new failover strategy to restart pipelined regions that works on the given - * execution graph. - * - * @param executionGraph The execution graph on which this FailoverStrategy will work - */ - public RestartPipelinedRegionStrategy(ExecutionGraph executionGraph) { - this.executionGraph = checkNotNull(executionGraph); - this.vertexToRegion = new HashMap<>(); - } - - // ------------------------------------------------------------------------ - // failover implementation - // ------------------------------------------------------------------------ - - @Override - public void onTaskFailure(Execution taskExecution, Throwable cause) { - final ExecutionVertex ev = taskExecution.getVertex(); - final FailoverRegion failoverRegion = vertexToRegion.get(ev); - - if (failoverRegion == null) { - executionGraph.failGlobal(new FlinkException( - "Can not find a failover region for the execution " + ev.getTaskNameWithSubtaskIndex(), cause)); - } - else { - LOG.info("Recovering task failure for {} #{} ({}) via restart of failover region", - taskExecution.getVertex().getTaskNameWithSubtaskIndex(), - taskExecution.getAttemptNumber(), - taskExecution.getAttemptId()); - - failoverRegion.onExecutionFail(taskExecution, cause); - } - } - - @Override - public void notifyNewVertices(List newJobVerticesTopological) { - generateAllFailoverRegion(newJobVerticesTopological); - } - - @Override - public String getStrategyName() { - return "Pipelined Region Failover"; - } - - /** - * Generate all the FailoverRegion from the new added job vertexes - */ - private void generateAllFailoverRegion(List newJobVerticesTopological) { - final IdentityHashMap> vertexToRegion = new IdentityHashMap<>(); - - // we use the map (list -> null) to imitate an IdentityHashSet (which does not exist) - final IdentityHashMap, Object> distinctRegions = new IdentityHashMap<>(); - - // this loop will worst case iterate over every edge in the graph (complexity is O(#edges)) - - for (ExecutionJobVertex ejv : newJobVerticesTopological) { - - // currently, jobs with a co-location constraint fail as one - // we want to improve that in the future (or get rid of co-location constraints) - if (ejv.getCoLocationGroup() != null) { - makeAllOneRegion(newJobVerticesTopological); - return; - } - - // see if this JobVertex one has pipelined inputs at all - final List inputs = ejv.getInputs(); - final int numInputs = inputs.size(); - boolean hasPipelinedInputs = false; - - for (IntermediateResult input : inputs) { - if (input.getResultType().isPipelined()) { - hasPipelinedInputs = true; - break; - } - } - - if (hasPipelinedInputs) { - // build upon the predecessors - for (ExecutionVertex ev : ejv.getTaskVertices()) { - - // remember the region in which we are - ArrayList thisRegion = null; - - for (int inputNum = 0; inputNum < numInputs; inputNum++) { - if (inputs.get(inputNum).getResultType().isPipelined()) { - - for (ExecutionEdge edge : ev.getInputEdges(inputNum)) { - final ExecutionVertex predecessor = edge.getSource().getProducer(); - final ArrayList predecessorRegion = vertexToRegion.get(predecessor); - - if (thisRegion != null) { - // we already have a region. see if it is the same as the predecessor's region - if (predecessorRegion != thisRegion) { - - // we need to merge our region and the predecessor's region - predecessorRegion.addAll(thisRegion); - distinctRegions.remove(thisRegion); - thisRegion = predecessorRegion; - - // remap the vertices from that merged region - for (ExecutionVertex inPredRegion: predecessorRegion) { - vertexToRegion.put(inPredRegion, thisRegion); - } - } - } - else if (predecessor != null) { - // first case, make this our region - thisRegion = predecessorRegion; - thisRegion.add(ev); - vertexToRegion.put(ev, thisRegion); - } - else { - // throw an uncaught exception here - // this is a bug and not a recoverable situation - throw new FlinkRuntimeException( - "bug in the logic to construct the pipelined failover regions"); - } - } - } - } - } - } - else { - // no pipelined inputs, start a new region - for (ExecutionVertex ev : ejv.getTaskVertices()) { - ArrayList region = new ArrayList<>(1); - region.add(ev); - vertexToRegion.put(ev, region); - distinctRegions.put(region, null); - } - } - } - - // now that we have all regions, create the failover region objects - LOG.info("Creating {} individual failover regions for job {} ({})", - distinctRegions.size(), executionGraph.getJobName(), executionGraph.getJobID()); - - for (List region : distinctRegions.keySet()) { - final FailoverRegion failoverRegion = createFailoverRegion(executionGraph, region); - for (ExecutionVertex ev : region) { - this.vertexToRegion.put(ev, failoverRegion); - } - } - } - - private void makeAllOneRegion(List jobVertices) { - LOG.warn("Cannot decompose ExecutionGraph into individual failover regions due to use of " + - "Co-Location constraints (iterations). Job will fail over as one holistic unit."); - - final ArrayList allVertices = new ArrayList<>(); - - for (ExecutionJobVertex ejv : jobVertices) { - - // safe some incremental size growing - allVertices.ensureCapacity(allVertices.size() + ejv.getParallelism()); - - allVertices.addAll(Arrays.asList(ejv.getTaskVertices())); - } - - final FailoverRegion singleRegion = createFailoverRegion(executionGraph, allVertices); - for (ExecutionVertex ev : allVertices) { - vertexToRegion.put(ev, singleRegion); - } - } - - @VisibleForTesting - protected FailoverRegion createFailoverRegion(ExecutionGraph eg, List connectedExecutions) { - Map tasks = initTasks(connectedExecutions); - return new FailoverRegion(eg, connectedExecutions, tasks); - } - - @VisibleForTesting - protected Map initTasks(List connectedExecutions) { - Map tasks = new HashMap<>(connectedExecutions.size()); - for (ExecutionVertex executionVertex : connectedExecutions) { - JobVertexID jobvertexId = executionVertex.getJobvertexId(); - ExecutionJobVertex jobVertex = executionVertex.getJobVertex(); - tasks.putIfAbsent(jobvertexId, jobVertex); - } - return tasks; - } - - // ------------------------------------------------------------------------ - // testing - // ------------------------------------------------------------------------ - - /** - * Finds the failover region that contains the given execution vertex. - */ - @VisibleForTesting - public FailoverRegion getFailoverRegion(ExecutionVertex ev) { - return vertexToRegion.get(ev); - } - - // ------------------------------------------------------------------------ - // factory - // ------------------------------------------------------------------------ - - /** - * Factory that instantiates the RestartPipelinedRegionStrategy. - */ - public static class Factory implements FailoverStrategy.Factory { - - @Override - public FailoverStrategy create(ExecutionGraph executionGraph) { - return new RestartPipelinedRegionStrategy(executionGraph); - } - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/adapter/DefaultFailoverTopology.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/adapter/DefaultFailoverTopology.java index 7f832f0bfb85138ec72f347de43e0a441c609f52..e27d2134276df8b7e3d6b4ad58d89307b60d69ea 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/adapter/DefaultFailoverTopology.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/adapter/DefaultFailoverTopology.java @@ -24,7 +24,6 @@ import org.apache.flink.runtime.executiongraph.ExecutionVertex; import org.apache.flink.runtime.executiongraph.IntermediateResultPartition; import org.apache.flink.runtime.executiongraph.failover.flip1.FailoverTopology; import org.apache.flink.runtime.executiongraph.failover.flip1.FailoverVertex; -import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID; import java.util.ArrayList; import java.util.Collection; @@ -56,7 +55,7 @@ public class DefaultFailoverTopology implements FailoverTopology { final Map failoverVertexMap = new IdentityHashMap<>(); for (ExecutionVertex vertex : executionGraph.getAllExecutionVertices()) { final DefaultFailoverVertex failoverVertex = new DefaultFailoverVertex( - new ExecutionVertexID(vertex.getJobvertexId(), vertex.getParallelSubtaskIndex()), + vertex.getID(), vertex.getTaskNameWithSubtaskIndex()); this.failoverVertices.add(failoverVertex); failoverVertexMap.put(vertex, failoverVertex); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FailureRateRestartBackoffTimeStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FailureRateRestartBackoffTimeStrategy.java index 8404ddaf1b28f99a11a1a4cf1d80139782ae6240..2d589112e8a2c29185a928914829e37bb3e47936 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FailureRateRestartBackoffTimeStrategy.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FailureRateRestartBackoffTimeStrategy.java @@ -19,15 +19,14 @@ package org.apache.flink.runtime.executiongraph.failover.flip1; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestartStrategyOptions; import org.apache.flink.runtime.util.clock.Clock; import org.apache.flink.runtime.util.clock.SystemClock; +import org.apache.flink.util.TimeUtils; import java.util.ArrayDeque; import java.util.Deque; -import static org.apache.flink.configuration.RestartBackoffTimeStrategyOptions.RESTART_BACKOFF_TIME_STRATEGY_FAILURE_RATE_FAILURE_RATE_BACKOFF_TIME; -import static org.apache.flink.configuration.RestartBackoffTimeStrategyOptions.RESTART_BACKOFF_TIME_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL; -import static org.apache.flink.configuration.RestartBackoffTimeStrategyOptions.RESTART_BACKOFF_TIME_STRATEGY_FAILURE_RATE_MAX_FAILURES_PER_INTERVAL; import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -110,10 +109,29 @@ public class FailureRateRestartBackoffTimeStrategy implements RestartBackoffTime } public static FailureRateRestartBackoffTimeStrategyFactory createFactory(final Configuration configuration) { - return new FailureRateRestartBackoffTimeStrategyFactory( - configuration.getInteger(RESTART_BACKOFF_TIME_STRATEGY_FAILURE_RATE_MAX_FAILURES_PER_INTERVAL), - configuration.getLong(RESTART_BACKOFF_TIME_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL), - configuration.getLong(RESTART_BACKOFF_TIME_STRATEGY_FAILURE_RATE_FAILURE_RATE_BACKOFF_TIME)); + int maxFailuresPerInterval = configuration.getInteger(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_MAX_FAILURES_PER_INTERVAL); + String failuresIntervalString = configuration.getString(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL); + String delayString = configuration.getString(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_DELAY); + + long failuresInterval; + try { + failuresInterval = TimeUtils.parseDuration(failuresIntervalString).toMillis(); + } catch (IllegalArgumentException ex) { + throw new IllegalArgumentException("Invalid config value for " + + RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL.key() + ": " + failuresIntervalString + + ". Value must be a valid duration (such as '100 milli' or '10 s')", ex); + } + + long delay; + try { + delay = TimeUtils.parseDuration(delayString).toMillis(); + } catch (IllegalArgumentException ex) { + throw new IllegalArgumentException("Invalid config value for " + + RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_DELAY.key() + ": " + delayString + + ". Value must be a valid duration (such as '100 milli' or '10 s')", ex); + } + + return new FailureRateRestartBackoffTimeStrategyFactory(maxFailuresPerInterval, failuresInterval, delay); } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FixedDelayRestartBackoffTimeStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FixedDelayRestartBackoffTimeStrategy.java index 64dc1bf928b6b98342ce0bddb04276e0e7b2a7b9..ca786d3fb5d7a29dd0500f80972bf1b8a07e6f9c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FixedDelayRestartBackoffTimeStrategy.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/FixedDelayRestartBackoffTimeStrategy.java @@ -19,9 +19,9 @@ package org.apache.flink.runtime.executiongraph.failover.flip1; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestartStrategyOptions; +import org.apache.flink.util.TimeUtils; -import static org.apache.flink.configuration.RestartBackoffTimeStrategyOptions.RESTART_BACKOFF_TIME_STRATEGY_FIXED_DELAY_ATTEMPTS; -import static org.apache.flink.configuration.RestartBackoffTimeStrategyOptions.RESTART_BACKOFF_TIME_STRATEGY_FIXED_DELAY_BACKOFF_TIME; import static org.apache.flink.util.Preconditions.checkArgument; /** @@ -57,6 +57,10 @@ public class FixedDelayRestartBackoffTimeStrategy implements RestartBackoffTimeS return backoffTimeMS; } + public int getMaxNumberRestartAttempts() { + return maxNumberRestartAttempts; + } + @Override public void notifyFailure(Throwable cause) { currentRestartAttempt++; @@ -79,9 +83,19 @@ public class FixedDelayRestartBackoffTimeStrategy implements RestartBackoffTimeS } public static FixedDelayRestartBackoffTimeStrategyFactory createFactory(final Configuration configuration) { - return new FixedDelayRestartBackoffTimeStrategyFactory( - configuration.getInteger(RESTART_BACKOFF_TIME_STRATEGY_FIXED_DELAY_ATTEMPTS), - configuration.getLong(RESTART_BACKOFF_TIME_STRATEGY_FIXED_DELAY_BACKOFF_TIME)); + int maxAttempts = configuration.getInteger(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS); + String delayString = configuration.getString(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY); + + long delay; + try { + delay = TimeUtils.parseDuration(delayString).toMillis(); + } catch (IllegalArgumentException ex) { + throw new IllegalArgumentException("Invalid config value for " + + RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY.key() + ": " + delayString + + ". Value must be a valid duration (such as '100 milli' or '10 s')", ex); + } + + return new FixedDelayRestartBackoffTimeStrategyFactory(maxAttempts, delay); } /** diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlMultisetType.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/NoRestartBackoffTimeStrategy.java similarity index 49% rename from flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlMultisetType.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/NoRestartBackoffTimeStrategy.java index 3b6f19c2e6296eff284326a4a76b3d6ce93e6de8..fb625068b0cbade50b5620c769e836b888f6a7a1 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlMultisetType.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/NoRestartBackoffTimeStrategy.java @@ -16,34 +16,48 @@ * limitations under the License. */ -package org.apache.flink.sql.parser.type; - -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlWriter; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.SqlTypeName; +package org.apache.flink.runtime.executiongraph.failover.flip1; /** - * Parse column of MULTISET type. + * Restart strategy which does not restart tasks when tasks fail. */ -public class SqlMultisetType extends SqlIdentifier implements ExtendedSqlType { +public enum NoRestartBackoffTimeStrategy implements RestartBackoffTimeStrategy { + INSTANCE; - private final SqlDataTypeSpec elementType; + @Override + public boolean canRestart() { + return false; + } - public SqlMultisetType(SqlParserPos pos, SqlDataTypeSpec elementType) { - super(SqlTypeName.MULTISET.getName(), pos); - this.elementType = elementType; + @Override + public long getBackoffTime() { + return 0L; } - public SqlDataTypeSpec getElementType() { - return elementType; + @Override + public void notifyFailure(final Throwable cause) { + // nothing to do } @Override - public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - writer.keyword("MULTISET<"); - ExtendedSqlType.unparseType(this.elementType, writer, leftPrec, rightPrec); - writer.keyword(">"); + public String toString() { + return "NoRestartBackoffTimeStrategy"; + } + + /** + * The factory for creating {@link NoRestartBackoffTimeStrategy}. + */ + public enum NoRestartBackoffTimeStrategyFactory implements Factory { + INSTANCE; + + @Override + public RestartBackoffTimeStrategy create() { + return NoRestartBackoffTimeStrategy.INSTANCE; + } + + @Override + public String toString() { + return "NoRestartBackoffTimeStrategyFactory"; + } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartBackoffTimeStrategyFactoryLoader.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartBackoffTimeStrategyFactoryLoader.java new file mode 100644 index 0000000000000000000000000000000000000000..6323f46accb4590804910e2cef0da9f0c6da6540 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartBackoffTimeStrategyFactoryLoader.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.executiongraph.failover.flip1; + +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.common.restartstrategy.RestartStrategies.FailureRateRestartStrategyConfiguration; +import org.apache.flink.api.common.restartstrategy.RestartStrategies.FallbackRestartStrategyConfiguration; +import org.apache.flink.api.common.restartstrategy.RestartStrategies.FixedDelayRestartStrategyConfiguration; +import org.apache.flink.api.common.restartstrategy.RestartStrategies.NoRestartStrategyConfiguration; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestartStrategyOptions; +import org.apache.flink.runtime.executiongraph.restart.NoOrFixedIfCheckpointingEnabledRestartStrategyFactory; +import org.apache.flink.runtime.executiongraph.restart.RestartStrategy; + +import java.util.Optional; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A utility class to load {@link RestartBackoffTimeStrategy.Factory} from the configuration. + * It respects the configs for the legacy {@link RestartStrategy}. + */ +public final class RestartBackoffTimeStrategyFactoryLoader { + + private RestartBackoffTimeStrategyFactoryLoader() { + } + + /** + * Creates {@link RestartBackoffTimeStrategy.Factory} from the given configuration. + * + *

The strategy factory is decided in order as follows: + *

    + *
  1. Strategy set within job graph, i.e. {@link RestartStrategies.RestartStrategyConfiguration}, + * unless the config is {@link RestartStrategies.FallbackRestartStrategyConfiguration}.
    2. + *
  2. Strategy set in the cluster(server-side) config (flink-conf.yaml), + * unless the strategy is not specified
    3. + *
  3. {@link FixedDelayRestartBackoffTimeStrategy.FixedDelayRestartBackoffTimeStrategyFactory} if + * checkpointing is enabled. Otherwise {@link NoRestartBackoffTimeStrategy.NoRestartBackoffTimeStrategyFactory}
    4. + *
+ * + * @param jobRestartStrategyConfiguration restart configuration given within the job graph + * @param clusterConfiguration cluster(server-side) configuration + * @param isCheckpointingEnabled if checkpointing is enabled for the job + * @return new version restart strategy factory + */ + public static RestartBackoffTimeStrategy.Factory createRestartStrategyFactory( + final RestartStrategies.RestartStrategyConfiguration jobRestartStrategyConfiguration, + final Configuration clusterConfiguration, + final boolean isCheckpointingEnabled) { + + checkNotNull(jobRestartStrategyConfiguration); + checkNotNull(clusterConfiguration); + + return getJobRestartStrategyFactory(jobRestartStrategyConfiguration) + .orElse(getClusterRestartStrategyFactory(clusterConfiguration) + .orElse(getDefaultRestartStrategyFactory(isCheckpointingEnabled))); + } + + private static Optional getJobRestartStrategyFactory( + final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration) { + + if (restartStrategyConfiguration instanceof NoRestartStrategyConfiguration) { + return Optional.of(NoRestartBackoffTimeStrategy.NoRestartBackoffTimeStrategyFactory.INSTANCE); + } else if (restartStrategyConfiguration instanceof FixedDelayRestartStrategyConfiguration) { + final FixedDelayRestartStrategyConfiguration fixedDelayConfig = + (FixedDelayRestartStrategyConfiguration) restartStrategyConfiguration; + + return Optional.of(new FixedDelayRestartBackoffTimeStrategy.FixedDelayRestartBackoffTimeStrategyFactory( + fixedDelayConfig.getRestartAttempts(), + fixedDelayConfig.getDelayBetweenAttemptsInterval().toMilliseconds())); + } else if (restartStrategyConfiguration instanceof FailureRateRestartStrategyConfiguration) { + final FailureRateRestartStrategyConfiguration failureRateConfig = + (FailureRateRestartStrategyConfiguration) restartStrategyConfiguration; + + return Optional.of(new FailureRateRestartBackoffTimeStrategy.FailureRateRestartBackoffTimeStrategyFactory( + failureRateConfig.getMaxFailureRate(), + failureRateConfig.getFailureInterval().toMilliseconds(), + failureRateConfig.getDelayBetweenAttemptsInterval().toMilliseconds())); + } else if (restartStrategyConfiguration instanceof FallbackRestartStrategyConfiguration) { + return Optional.empty(); + } else { + throw new IllegalArgumentException("Unknown restart strategy configuration " + + restartStrategyConfiguration + "."); + } + } + + private static Optional getClusterRestartStrategyFactory( + final Configuration clusterConfiguration) { + + final String restartStrategyName = clusterConfiguration.getString(RestartStrategyOptions.RESTART_STRATEGY); + if (restartStrategyName == null) { + return Optional.empty(); + } + + switch (restartStrategyName.toLowerCase()) { + case "none": + case "off": + case "disable": + return Optional.of(NoRestartBackoffTimeStrategy.NoRestartBackoffTimeStrategyFactory.INSTANCE); + case "fixeddelay": + case "fixed-delay": + return Optional.of(FixedDelayRestartBackoffTimeStrategy.createFactory(clusterConfiguration)); + case "failurerate": + case "failure-rate": + return Optional.of(FailureRateRestartBackoffTimeStrategy.createFactory(clusterConfiguration)); + default: + throw new IllegalArgumentException("Unknown restart strategy " + restartStrategyName + "."); + } + } + + private static RestartBackoffTimeStrategy.Factory getDefaultRestartStrategyFactory( + final boolean isCheckpointingEnabled) { + + if (isCheckpointingEnabled) { + // fixed delay restart strategy with default params + return new FixedDelayRestartBackoffTimeStrategy.FixedDelayRestartBackoffTimeStrategyFactory( + NoOrFixedIfCheckpointingEnabledRestartStrategyFactory.DEFAULT_RESTART_ATTEMPTS, + NoOrFixedIfCheckpointingEnabledRestartStrategyFactory.DEFAULT_RESTART_DELAY); + } else { + return NoRestartBackoffTimeStrategy.NoRestartBackoffTimeStrategyFactory.INSTANCE; + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartPipelinedRegionStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartPipelinedRegionStrategy.java index a87764bc01798fc63d16030dde1d5e8961578ae6..941b0497bb49a35ee0e696e13011c1097b74a151 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartPipelinedRegionStrategy.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartPipelinedRegionStrategy.java @@ -257,4 +257,12 @@ public class RestartPipelinedRegionStrategy implements FailoverStrategy { failedPartitions.remove(resultPartitionID); } } + + public static class Factory implements FailoverStrategy.Factory { + + @Override + public FailoverStrategy create(final FailoverTopology topology) { + return new RestartPipelinedRegionStrategy(topology); + } + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/FailureRateRestartStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/FailureRateRestartStrategy.java index 291ba5eda3400ad68709c4fbfc257eff286e6a3d..69c6350baf1203ca70fb42ca47c2a61e621055be 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/FailureRateRestartStrategy.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/FailureRateRestartStrategy.java @@ -25,12 +25,12 @@ import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.concurrent.ScheduledExecutor; import org.apache.flink.runtime.executiongraph.ExecutionGraph; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.TimeUtils; +import java.time.Duration; import java.util.ArrayDeque; import java.util.concurrent.CompletableFuture; -import scala.concurrent.duration.Duration; - /** * Restart strategy which tries to restart the given {@link ExecutionGraph} when failure rate exceeded * with a fixed time delay in between. @@ -94,8 +94,8 @@ public class FailureRateRestartStrategy implements RestartStrategy { String failuresIntervalString = configuration.getString(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL); String delayString = configuration.getString(RestartStrategyOptions.RESTART_STRATEGY_FAILURE_RATE_DELAY); - Duration failuresInterval = Duration.apply(failuresIntervalString); - Duration delay = Duration.apply(delayString); + Duration failuresInterval = TimeUtils.parseDuration(failuresIntervalString); + Duration delay = TimeUtils.parseDuration(delayString); return new FailureRateRestartStrategyFactory(maxFailuresPerInterval, Time.milliseconds(failuresInterval.toMillis()), Time.milliseconds(delay.toMillis())); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/FixedDelayRestartStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/FixedDelayRestartStrategy.java index 01e305639b1c1f27473f656a7fb8ae28ce299a98..95b84cb49f08524f28528c1d9d085d51737f4cfb 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/FixedDelayRestartStrategy.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/FixedDelayRestartStrategy.java @@ -25,11 +25,10 @@ import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.concurrent.ScheduledExecutor; import org.apache.flink.runtime.executiongraph.ExecutionGraph; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.TimeUtils; import java.util.concurrent.CompletableFuture; -import scala.concurrent.duration.Duration; - /** * Restart strategy which tries to restart the given {@link ExecutionGraph} a fixed number of times * with a fixed time delay in between. @@ -82,11 +81,11 @@ public class FixedDelayRestartStrategy implements RestartStrategy { long delay; try { - delay = Duration.apply(delayString).toMillis(); - } catch (NumberFormatException nfe) { + delay = TimeUtils.parseDuration(delayString).toMillis(); + } catch (IllegalArgumentException ex) { throw new Exception("Invalid config value for " + RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_DELAY.key() + ": " + delayString + - ". Value must be a valid duration (such as '100 milli' or '10 s')"); + ". Value must be a valid duration (such as '100 milli' or '10 s')", ex); } return new FixedDelayRestartStrategyFactory(maxAttempts, delay); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/NoOrFixedIfCheckpointingEnabledRestartStrategyFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/NoOrFixedIfCheckpointingEnabledRestartStrategyFactory.java index dbdf63aee4571f8f2224f1105fd2a06c08c95d83..3e40acd6f781e1ac93432242c8f17f290e4a0156 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/NoOrFixedIfCheckpointingEnabledRestartStrategyFactory.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/executiongraph/restart/NoOrFixedIfCheckpointingEnabledRestartStrategyFactory.java @@ -27,7 +27,9 @@ import java.time.Duration; public class NoOrFixedIfCheckpointingEnabledRestartStrategyFactory extends RestartStrategyFactory { private static final long serialVersionUID = -1809462525812787862L; - private static final long DEFAULT_RESTART_DELAY = Duration.ofSeconds(1L).toMillis(); + public static final int DEFAULT_RESTART_ATTEMPTS = Integer.MAX_VALUE; + + public static final long DEFAULT_RESTART_DELAY = Duration.ofSeconds(1L).toMillis(); @Override public RestartStrategy createRestartStrategy() { @@ -36,7 +38,7 @@ public class NoOrFixedIfCheckpointingEnabledRestartStrategyFactory extends Resta RestartStrategy createRestartStrategy(boolean isCheckpointingEnabled) { if (isCheckpointingEnabled) { - return new FixedDelayRestartStrategy(Integer.MAX_VALUE, DEFAULT_RESTART_DELAY); + return new FixedDelayRestartStrategy(DEFAULT_RESTART_ATTEMPTS, DEFAULT_RESTART_DELAY); } else { return new NoRestartStrategy(); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/NoOpHeartbeatManager.java b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/NoOpHeartbeatManager.java new file mode 100644 index 0000000000000000000000000000000000000000..965a50b3f3539236a7e706dd51137b6b66ffcc57 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/heartbeat/NoOpHeartbeatManager.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.heartbeat; + +import org.apache.flink.runtime.clusterframework.types.ResourceID; + +/** + * {@link HeartbeatManager} implementation which does nothing. + * + * @param ignored + * @param ignored + */ +public class NoOpHeartbeatManager implements HeartbeatManager { + private static final NoOpHeartbeatManager INSTANCE = new NoOpHeartbeatManager<>(); + + private NoOpHeartbeatManager() {} + + @Override + public void monitorTarget(ResourceID resourceID, HeartbeatTarget heartbeatTarget) {} + + @Override + public void unmonitorTarget(ResourceID resourceID) {} + + @Override + public void stop() {} + + @Override + public long getLastHeartbeatFrom(ResourceID resourceId) { + return 0; + } + + @Override + public void receiveHeartbeat(ResourceID heartbeatOrigin, I heartbeatPayload) {} + + @Override + public void requestHeartbeat(ResourceID requestOrigin, I heartbeatPayload) {} + + @SuppressWarnings("unchecked") + public static NoOpHeartbeatManager getInstance() { + return (NoOpHeartbeatManager) INSTANCE; + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/FsNegativeRunningJobsRegistry.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/FsNegativeRunningJobsRegistry.java deleted file mode 100644 index d29dcec47a84e8fb6cde895a1717e8dbd4b49ac1..0000000000000000000000000000000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/FsNegativeRunningJobsRegistry.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.highavailability; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.core.fs.FSDataOutputStream; -import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.FileSystem.WriteMode; -import org.apache.flink.core.fs.Path; - -import java.io.FileNotFoundException; -import java.io.IOException; - -import static org.apache.flink.util.Preconditions.checkNotNull; - -/** - * This {@link RunningJobsRegistry} tracks the status jobs via marker files, - * marking running jobs via running marker files, marking finished jobs via finished marker files. - * - *

The general contract is the following: - *

    - *
  • Initially, a marker file does not exist (no one created it, yet), which means - * the specific job is pending.
    • - *
  • The first JobManager that granted leadership calls this service to create the running marker file, - * which marks the job as running.
    • - *
  • If a JobManager gains leadership but sees the running marker file, - * it will realize that the job has been scheduled already and needs reconciling.
    • - *
  • The JobManager that finishes calls this service to create the marker file, - * which marks the job as finished.
    • - *
  • If a JobManager gains leadership at some point when shutdown is in progress, - * it will see the marker file and realize that the job is finished.
    • - *
  • The application framework is expected to clean the file once the application - * is completely shut down. At that point, no JobManager will attempt to - * start the job, even if it gains leadership.
    • - *
- * - *

This registry is especially tailored towards deployment modes like for example - * YARN, where HDFS is available as a persistent file system, and the YARN - * application's working directories on HDFS are automatically cleaned - * up after the application completed. - */ -public class FsNegativeRunningJobsRegistry implements RunningJobsRegistry { - - private static final String DONE_PREFIX = ".job_complete_"; - - private static final String RUNNING_PREFIX = ".job_runing_"; - - private final FileSystem fileSystem; - - private final Path basePath; - - /** - * Creates a new registry that writes to the FileSystem and working directory - * denoted by the given path. - * - *

The initialization will attempt to write to the given working directory, in - * order to catch setup/configuration errors early. - * - * @param workingDirectory The working directory for files to track the job status. - * - * @throws IOException Thrown, if the specified directory cannot be accessed. - */ - public FsNegativeRunningJobsRegistry(Path workingDirectory) throws IOException { - this(workingDirectory.getFileSystem(), workingDirectory); - } - - /** - * Creates a new registry that writes its files to the given FileSystem at - * the given working directory path. - * - *

The initialization will attempt to write to the given working directory, in - * order to catch setup/configuration errors early. - * - * @param fileSystem The FileSystem to use for the marker files. - * @param workingDirectory The working directory for files to track the job status. - * - * @throws IOException Thrown, if the specified directory cannot be accessed. - */ - public FsNegativeRunningJobsRegistry(FileSystem fileSystem, Path workingDirectory) throws IOException { - this.fileSystem = checkNotNull(fileSystem, "fileSystem"); - this.basePath = checkNotNull(workingDirectory, "workingDirectory"); - - // to be safe, attempt to write to the working directory, to - // catch problems early - final Path testFile = new Path(workingDirectory, ".registry_test"); - try { - createFile(testFile, false); - } - catch (IOException e) { - throw new IOException("Unable to write to working directory: " + workingDirectory, e); - } - finally { - fileSystem.delete(testFile, false); - } - } - - // ------------------------------------------------------------------------ - - @Override - public void setJobRunning(JobID jobID) throws IOException { - checkNotNull(jobID, "jobID"); - final Path filePath = createMarkerFilePath(RUNNING_PREFIX, jobID); - - // create the file - // to avoid an exception if the job already exists, set overwrite=true - createFile(filePath, true); - } - - @Override - public void setJobFinished(JobID jobID) throws IOException { - checkNotNull(jobID, "jobID"); - final Path filePath = createMarkerFilePath(DONE_PREFIX, jobID); - - // create the file - // to avoid an exception if the job already exists, set overwrite=true - createFile(filePath, true); - } - - @Override - public JobSchedulingStatus getJobSchedulingStatus(JobID jobID) throws IOException { - checkNotNull(jobID, "jobID"); - - // first check for the existence of the complete file - if (fileSystem.exists(createMarkerFilePath(DONE_PREFIX, jobID))) { - // complete file was found --> job is terminated - return JobSchedulingStatus.DONE; - } - // check for the existence of the running file - else if (fileSystem.exists(createMarkerFilePath(RUNNING_PREFIX, jobID))) { - // running file was found --> job is terminated - return JobSchedulingStatus.RUNNING; - } - else { - // file does not exist, job is not scheduled - return JobSchedulingStatus.PENDING; - } - } - - @Override - public void clearJob(JobID jobID) throws IOException { - checkNotNull(jobID, "jobID"); - final Path runningFilePath = createMarkerFilePath(RUNNING_PREFIX, jobID); - final Path doneFilePath = createMarkerFilePath(DONE_PREFIX, jobID); - - // delete the running marker file, if it exists - try { - fileSystem.delete(runningFilePath, false); - } - catch (FileNotFoundException ignored) {} - - // delete the finished marker file, if it exists - try { - fileSystem.delete(doneFilePath, false); - } - catch (FileNotFoundException ignored) {} - } - - private Path createMarkerFilePath(String prefix, JobID jobId) { - return new Path(basePath, prefix + jobId.toString()); - } - - private void createFile(Path path, boolean overwrite) throws IOException { - final WriteMode writeMode = overwrite ? WriteMode.OVERWRITE : WriteMode.NO_OVERWRITE; - - try (FSDataOutputStream out = fileSystem.create(path, writeMode)) { - out.write(42); - } - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java index e3e98e1090450f9fc8bba5cba3fc532eb4a7632d..ad84cc47a41ad555889159178037665035b67392 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedLeaderService.java @@ -236,7 +236,10 @@ public class EmbeddedLeaderService { /** * Callback from leader contenders when they confirm a leader grant. */ - private void confirmLeader(final EmbeddedLeaderElectionService service, final UUID leaderSessionId) { + private void confirmLeader( + final EmbeddedLeaderElectionService service, + final UUID leaderSessionId, + final String leaderAddress) { synchronized (lock) { // if the service was shut down in the meantime, ignore this confirmation if (!service.running || shutdown) { @@ -246,16 +249,15 @@ public class EmbeddedLeaderService { try { // check if the confirmation is for the same grant, or whether it is a stale grant if (service == currentLeaderProposed && currentLeaderSessionId.equals(leaderSessionId)) { - final String address = service.contender.getAddress(); - LOG.info("Received confirmation of leadership for leader {} , session={}", address, leaderSessionId); + LOG.info("Received confirmation of leadership for leader {} , session={}", leaderAddress, leaderSessionId); // mark leadership currentLeaderConfirmed = service; - currentLeaderAddress = address; + currentLeaderAddress = leaderAddress; currentLeaderProposed = null; // notify all listeners - notifyAllListeners(address, leaderSessionId); + notifyAllListeners(leaderAddress, leaderSessionId); } else { LOG.debug("Received confirmation of leadership for a stale leadership grant. Ignoring."); @@ -297,8 +299,7 @@ public class EmbeddedLeaderService { currentLeaderProposed = leaderService; currentLeaderProposed.isLeader = true; - LOG.info("Proposing leadership to contender {} @ {}", - leaderService.contender, leaderService.contender.getAddress()); + LOG.info("Proposing leadership to contender {}", leaderService.contender.getDescription()); return execute(new GrantLeadershipCall(leaderService.contender, leaderSessionId, LOG)); } @@ -434,9 +435,10 @@ public class EmbeddedLeaderService { } @Override - public void confirmLeaderSessionID(UUID leaderSessionID) { + public void confirmLeadership(UUID leaderSessionID, String leaderAddress) { checkNotNull(leaderSessionID); - confirmLeader(this, leaderSessionID); + checkNotNull(leaderAddress); + confirmLeader(this, leaderSessionID, leaderAddress); } @Override diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionService.java deleted file mode 100644 index 89b297722e0dbc09e2f769347f9408cc9eb42752..0000000000000000000000000000000000000000 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionService.java +++ /dev/null @@ -1,390 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.highavailability.nonha.leaderelection; - -import org.apache.flink.runtime.leaderelection.LeaderContender; -import org.apache.flink.runtime.leaderelection.LeaderElectionService; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; - -import java.util.HashSet; -import java.util.UUID; -import java.util.concurrent.Executor; - -import static org.apache.flink.util.Preconditions.checkArgument; -import static org.apache.flink.util.Preconditions.checkNotNull; -import static org.apache.flink.util.Preconditions.checkState; - -/** - * An implementation of the {@link LeaderElectionService} interface that handles a single - * leader contender. When started, this service immediately grants the contender the leadership. - * - *

The implementation accepts a single static leader session ID and is hence compatible with - * pre-configured single leader (no leader failover) setups. - * - *

This implementation supports a series of leader listeners that receive notifications about - * the leader contender. - */ -public class SingleLeaderElectionService implements LeaderElectionService { - - private static final Logger LOG = LoggerFactory.getLogger(SingleLeaderElectionService.class); - - // ------------------------------------------------------------------------ - - /** lock for all operations on this instance. */ - private final Object lock = new Object(); - - /** The executor service that dispatches notifications. */ - private final Executor notificationExecutor; - - /** The leader ID assigned to the immediate leader. */ - private final UUID leaderId; - - @GuardedBy("lock") - private final HashSet listeners; - - /** The currently proposed leader. */ - @GuardedBy("lock") - private volatile LeaderContender proposedLeader; - - /** The confirmed leader. */ - @GuardedBy("lock") - private volatile LeaderContender leader; - - /** The address of the confirmed leader. */ - @GuardedBy("lock") - private volatile String leaderAddress; - - /** Flag marking this service as shutdown, meaning it cannot be started again. */ - @GuardedBy("lock") - private volatile boolean shutdown; - - // ------------------------------------------------------------------------ - - /** - * Creates a new leader election service. The service assigns the given leader ID - * to the leader contender. - * - * @param leaderId The constant leader ID assigned to the leader. - */ - public SingleLeaderElectionService(Executor notificationsDispatcher, UUID leaderId) { - this.notificationExecutor = checkNotNull(notificationsDispatcher); - this.leaderId = checkNotNull(leaderId); - this.listeners = new HashSet<>(); - - shutdown = false; - } - - // ------------------------------------------------------------------------ - // leader election service - // ------------------------------------------------------------------------ - - @Override - public void start(LeaderContender contender) throws Exception { - checkNotNull(contender, "contender"); - - synchronized (lock) { - checkState(!shutdown, "service is shut down"); - checkState(proposedLeader == null, "service already started"); - - // directly grant leadership to the given contender - proposedLeader = contender; - notificationExecutor.execute(new GrantLeadershipCall(contender, leaderId)); - } - } - - @Override - public void stop() { - synchronized (lock) { - // notify all listeners that there is no leader - for (EmbeddedLeaderRetrievalService listener : listeners) { - notificationExecutor.execute( - new NotifyOfLeaderCall(null, null, listener.listener, LOG)); - } - - // if there was a leader, revoke its leadership - if (leader != null) { - try { - leader.revokeLeadership(); - } catch (Throwable t) { - leader.handleError(t instanceof Exception ? (Exception) t : new Exception(t)); - } - } - - proposedLeader = null; - leader = null; - leaderAddress = null; - } - } - - @Override - public void confirmLeaderSessionID(UUID leaderSessionID) { - checkNotNull(leaderSessionID, "leaderSessionID"); - checkArgument(leaderSessionID.equals(leaderId), "confirmed wrong leader session id"); - - synchronized (lock) { - checkState(!shutdown, "service is shut down"); - checkState(proposedLeader != null, "no leader proposed yet"); - checkState(leader == null, "leader already confirmed"); - - // accept the confirmation - final String address = proposedLeader.getAddress(); - leaderAddress = address; - leader = proposedLeader; - - // notify all listeners - for (EmbeddedLeaderRetrievalService listener : listeners) { - notificationExecutor.execute( - new NotifyOfLeaderCall(address, leaderId, listener.listener, LOG)); - } - } - } - - @Override - public boolean hasLeadership(@Nonnull UUID leaderSessionId) { - synchronized (lock) { - return proposedLeader != null && leaderSessionId.equals(leaderId); - } - } - - void errorOnGrantLeadership(LeaderContender contender, Throwable error) { - LOG.warn("Error granting leadership to contender", error); - contender.handleError(error instanceof Exception ? (Exception) error : new Exception(error)); - - synchronized (lock) { - if (proposedLeader == contender) { - proposedLeader = null; - leader = null; - } - } - } - - // ------------------------------------------------------------------------ - // shutdown - // ------------------------------------------------------------------------ - - public boolean isShutdown() { - synchronized (lock) { - return shutdown; - } - } - - public void shutdown() { - shutdownInternally(new Exception("The leader service is shutting down")); - } - - private void shutdownInternally(Exception exceptionForHandlers) { - synchronized (lock) { - if (shutdown) { - return; - } - - shutdown = true; - - // fail the leader (if there is one) - if (leader != null) { - try { - leader.handleError(exceptionForHandlers); - } catch (Throwable ignored) {} - } - - // clear all leader status - leader = null; - proposedLeader = null; - leaderAddress = null; - - // fail all registered listeners - for (EmbeddedLeaderRetrievalService service : listeners) { - service.shutdown(exceptionForHandlers); - } - listeners.clear(); - } - } - - private void fatalError(Throwable error) { - LOG.error("Embedded leader election service encountered a fatal error. Shutting down service.", error); - - shutdownInternally(new Exception("Leader election service is shutting down after a fatal error", error)); - } - - // ------------------------------------------------------------------------ - // leader listeners - // ------------------------------------------------------------------------ - - public LeaderRetrievalService createLeaderRetrievalService() { - synchronized (lock) { - checkState(!shutdown, "leader election service is shut down"); - return new EmbeddedLeaderRetrievalService(); - } - } - - void addListener(EmbeddedLeaderRetrievalService service, LeaderRetrievalListener listener) { - synchronized (lock) { - checkState(!shutdown, "leader election service is shut down"); - checkState(!service.running, "leader retrieval service is already started"); - - try { - if (!listeners.add(service)) { - throw new IllegalStateException("leader retrieval service was added to this service multiple times"); - } - - service.listener = listener; - service.running = true; - - // if we already have a leader, immediately notify this new listener - if (leader != null) { - notificationExecutor.execute( - new NotifyOfLeaderCall(leaderAddress, leaderId, listener, LOG)); - } - } - catch (Throwable t) { - fatalError(t); - } - } - } - - void removeListener(EmbeddedLeaderRetrievalService service) { - synchronized (lock) { - // if the service was not even started, simply do nothing - if (!service.running || shutdown) { - return; - } - - try { - if (!listeners.remove(service)) { - throw new IllegalStateException("leader retrieval service does not belong to this service"); - } - - // stop the service - service.listener = null; - service.running = false; - } - catch (Throwable t) { - fatalError(t); - } - } - } - - // ------------------------------------------------------------------------ - - private class EmbeddedLeaderRetrievalService implements LeaderRetrievalService { - - volatile LeaderRetrievalListener listener; - - volatile boolean running; - - @Override - public void start(LeaderRetrievalListener listener) throws Exception { - checkNotNull(listener); - addListener(this, listener); - } - - @Override - public void stop() throws Exception { - removeListener(this); - } - - void shutdown(Exception cause) { - if (running) { - final LeaderRetrievalListener lst = listener; - running = false; - listener = null; - - try { - lst.handleError(cause); - } catch (Throwable ignored) {} - } - } - } - - // ------------------------------------------------------------------------ - // asynchronous notifications - // ------------------------------------------------------------------------ - - /** - * This runnable informs a leader contender that it gained leadership. - */ - private class GrantLeadershipCall implements Runnable { - - private final LeaderContender contender; - private final UUID leaderSessionId; - - GrantLeadershipCall(LeaderContender contender, UUID leaderSessionId) { - - this.contender = checkNotNull(contender); - this.leaderSessionId = checkNotNull(leaderSessionId); - } - - @Override - public void run() { - try { - contender.grantLeadership(leaderSessionId); - } - catch (Throwable t) { - errorOnGrantLeadership(contender, t); - } - } - } - - // ------------------------------------------------------------------------ - - /** - * This runnable informs a leader listener of a new leader. - */ - private static class NotifyOfLeaderCall implements Runnable { - - @Nullable - private final String address; // null if leader revoked without new leader - @Nullable - private final UUID leaderSessionId; // null if leader revoked without new leader - - private final LeaderRetrievalListener listener; - private final Logger logger; - - NotifyOfLeaderCall( - @Nullable String address, - @Nullable UUID leaderSessionId, - LeaderRetrievalListener listener, - Logger logger) { - - this.address = address; - this.leaderSessionId = leaderSessionId; - this.listener = checkNotNull(listener); - this.logger = checkNotNull(logger); - } - - @Override - public void run() { - try { - listener.notifyLeaderAddress(address, leaderSessionId); - } - catch (Throwable t) { - logger.warn("Error notifying leader listener about new leader", t); - listener.handleError(t instanceof Exception ? (Exception) t : new Exception(t)); - } - } - } -} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/history/FsJobArchivist.java b/flink-runtime/src/main/java/org/apache/flink/runtime/history/FsJobArchivist.java index d0fbc5eb8ce26f60777ce37af7a3d36ec150e184..ab1e34d74078a89fda138bc8d4fcc9c656e58700 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/history/FsJobArchivist.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/history/FsJobArchivist.java @@ -109,15 +109,20 @@ public class FsJobArchivist { ByteArrayOutputStream output = new ByteArrayOutputStream()) { IOUtils.copyBytes(input, output); - JsonNode archive = mapper.readTree(output.toByteArray()); + try { + JsonNode archive = mapper.readTree(output.toByteArray()); - Collection archives = new ArrayList<>(); - for (JsonNode archivePart : archive.get(ARCHIVE)) { - String path = archivePart.get(PATH).asText(); - String json = archivePart.get(JSON).asText(); - archives.add(new ArchivedJson(path, json)); + Collection archives = new ArrayList<>(); + for (JsonNode archivePart : archive.get(ARCHIVE)) { + String path = archivePart.get(PATH).asText(); + String json = archivePart.get(JSON).asText(); + archives.add(new ArchivedJson(path, json)); + } + return archives; + } catch (NullPointerException npe) { + // occurs if the archive is empty or any of the expected fields are not present + throw new IOException("Job archive (" + file.getPath() + ") did not conform to expected format."); } - return archives; } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/AvailabilityListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/AvailabilityProvider.java similarity index 85% rename from flink-runtime/src/main/java/org/apache/flink/runtime/io/AvailabilityListener.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/io/AvailabilityProvider.java index f26a9194ad67204d6e32bba7ff26fab5e13c5454..42e09366de7e44a0b1bad5ad8d1ed63988169f89 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/AvailabilityListener.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/AvailabilityProvider.java @@ -23,21 +23,16 @@ import java.util.concurrent.CompletableFuture; /** * Interface defining couple of essential methods for listening on data availability using - * {@link CompletableFuture}. For usage check out for example {@link AsyncDataInput}. + * {@link CompletableFuture}. For usage check out for example {@link PullingAsyncDataInput}. */ @Internal -public interface AvailabilityListener { +public interface AvailabilityProvider { /** * Constant that allows to avoid volatile checks {@link CompletableFuture#isDone()}. Check * {@link #isAvailable()} for more explanation. */ CompletableFuture AVAILABLE = CompletableFuture.completedFuture(null); - /** - * @return true if is finished and for example end of input was reached, false otherwise. - */ - boolean isFinished(); - /** * Check if this instance is available for further processing. * @@ -45,8 +40,8 @@ public interface AvailabilityListener { * this method should do the following check: * 

 	 * {@code
-	 *	AvailabilityListener input = ...;
-	 *	if (input.isAvailable() == AvailabilityListener.AVAILABLE || input.isAvailable().isDone()) {
+	 *	AvailabilityProvider input = ...;
+	 *	if (input.isAvailable() == AvailabilityProvider.AVAILABLE || input.isAvailable().isDone()) {
 	 *		// do something;
 	 *	}
 	 * }
@@ -55,7 +50,7 @@ public interface AvailabilityListener {
 	 * @return a future that is completed if there are more records available. If there are more
 	 * records available immediately, {@link #AVAILABLE} should be returned. Previously returned
 	 * not completed futures should become completed once there is more input available or if
-	 * the input {@link #isFinished()}.
+	 * the input is finished.
 	 */
 	CompletableFuture isAvailable();
 }
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/NullableAsyncDataInput.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/NullableAsyncDataInput.java
index f73970d05ecc0a0ee54e13da6a98475dd1ff1182..4b0095547968136c881a8b6ba287544253e2f4eb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/NullableAsyncDataInput.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/NullableAsyncDataInput.java
@@ -22,12 +22,12 @@ import org.apache.flink.annotation.Internal;
 import javax.annotation.Nullable;
 
 /**
- * The variant of {@link AsyncDataInput} that for performance reasons returns {@code null} from
- * {@link #pollNextNullable()} instead returning {@code Optional.empty()} from
- * {@link AsyncDataInput#pollNext()}.
+ * The variant of {@link PullingAsyncDataInput} that for performance reasons returns {@code null}
+ * from {@link #pollNextNullable()} instead returning {@code Optional.empty()} from
+ * {@link PullingAsyncDataInput#pollNext()}.
  */
 @Internal
-public interface NullableAsyncDataInput extends AvailabilityListener {
+public interface NullableAsyncDataInput extends AvailabilityProvider {
 	/**
 	 * Poll the next element. This method should be non blocking.
 	 *
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/AsyncDataInput.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/PullingAsyncDataInput.java
similarity index 91%
rename from flink-runtime/src/main/java/org/apache/flink/runtime/io/AsyncDataInput.java
rename to flink-runtime/src/main/java/org/apache/flink/runtime/io/PullingAsyncDataInput.java
index 7f76e84370e0975744c6f314484f9fd8a3194f29..17109ccebb0d5044d034a31f275004912e966865 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/AsyncDataInput.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/PullingAsyncDataInput.java
@@ -50,7 +50,7 @@ import java.util.concurrent.CompletableFuture;
  *
*/ @Internal -public interface AsyncDataInput extends AvailabilityListener { +public interface PullingAsyncDataInput extends AvailabilityProvider { /** * Poll the next element. This method should be non blocking. * @@ -58,4 +58,9 @@ public interface AsyncDataInput extends AvailabilityListener { * if {@link #isFinished()} returns true. Otherwise {@code Optional.of(element)}. */ Optional pollNext() throws Exception; + + /** + * @return true if is finished and for example end of input was reached, false otherwise. + */ + boolean isFinished(); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SimpleCollectingOutputView.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SimpleCollectingOutputView.java index 1a45ff2a8312766b876b9aeec2b728849b3af418..2fa6c883c6b7cbd1dd39808ea1c4001c60d15edc 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SimpleCollectingOutputView.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/disk/SimpleCollectingOutputView.java @@ -76,7 +76,7 @@ public class SimpleCollectingOutputView extends AbstractPagedOutputView { this.segmentNum++; return next; } else { - throw new EOFException(); + throw new EOFException("Can't collect further: memorySource depleted"); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/BroadcastRecordWriter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/BroadcastRecordWriter.java index effff59a105afce1104ab2f5b360ce3a40c637ac..b4999835d7fc56c24e23ce8f3e7830663f1446e0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/BroadcastRecordWriter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/BroadcastRecordWriter.java @@ -18,30 +18,162 @@ package org.apache.flink.runtime.io.network.api.writer; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.core.io.IOReadableWritable; +import org.apache.flink.runtime.io.network.buffer.BufferBuilder; +import org.apache.flink.runtime.io.network.buffer.BufferConsumer; + +import javax.annotation.Nullable; import java.io.IOException; +import static org.apache.flink.util.Preconditions.checkState; + /** * A special record-oriented runtime result writer only for broadcast mode. * - *

The BroadcastRecordWriter extends the {@link RecordWriter} and handles {@link #emit(IOReadableWritable)} - * operation via {@link #broadcastEmit(IOReadableWritable)} directly in a more efficient way. + *

The BroadcastRecordWriter extends the {@link RecordWriter} and maintain a single {@link BufferBuilder} + * for all the channels. Then the serialization results need be copied only once to this buffer which would be + * shared for all the channels in a more efficient way. * * @param the type of the record that can be emitted with this record writer */ -public class BroadcastRecordWriter extends RecordWriter { +public final class BroadcastRecordWriter extends RecordWriter { + + /** The current buffer builder shared for all the channels. */ + @Nullable + private BufferBuilder bufferBuilder; + + /** + * The flag for judging whether {@link #requestNewBufferBuilder(int)} and {@link #flushTargetPartition(int)} + * is triggered by {@link #randomEmit(IOReadableWritable)} or not. + */ + private boolean randomTriggered; BroadcastRecordWriter( ResultPartitionWriter writer, - ChannelSelector channelSelector, long timeout, String taskName) { - super(writer, channelSelector, timeout, taskName); + super(writer, timeout, taskName); } @Override public void emit(T record) throws IOException, InterruptedException { broadcastEmit(record); } + + @Override + public void randomEmit(T record) throws IOException, InterruptedException { + randomEmit(record, rng.nextInt(numberOfChannels)); + } + + /** + * For non-broadcast emit, we try to finish the current {@link BufferBuilder} first, and then request + * a new {@link BufferBuilder} for the random channel. If this new {@link BufferBuilder} is not full, + * it can be shared for all the other channels via initializing readable position in created + * {@link BufferConsumer}. + */ + @VisibleForTesting + void randomEmit(T record, int targetChannelIndex) throws IOException, InterruptedException { + tryFinishCurrentBufferBuilder(targetChannelIndex); + + randomTriggered = true; + emit(record, targetChannelIndex); + randomTriggered = false; + + if (bufferBuilder != null) { + for (int index = 0; index < numberOfChannels; index++) { + if (index != targetChannelIndex) { + targetPartition.addBufferConsumer(bufferBuilder.createBufferConsumer(), index); + } + } + } + } + + @Override + public void broadcastEmit(T record) throws IOException, InterruptedException { + // We could actually select any target channel here because all the channels + // are sharing the same BufferBuilder in broadcast mode. + emit(record, 0); + } + + /** + * The flush could be triggered by {@link #randomEmit(IOReadableWritable)}, {@link #emit(IOReadableWritable)} + * or {@link #broadcastEmit(IOReadableWritable)}. Only random emit should flush a single target channel, + * otherwise we should flush all the channels. + */ + @Override + public void flushTargetPartition(int targetChannel) { + if (randomTriggered) { + super.flushTargetPartition(targetChannel); + } else { + flushAll(); + } + } + + @Override + public BufferBuilder getBufferBuilder(int targetChannel) throws IOException, InterruptedException { + return bufferBuilder != null ? bufferBuilder : requestNewBufferBuilder(targetChannel); + } + + /** + * The request could be from broadcast or non-broadcast modes like {@link #randomEmit(IOReadableWritable)}. + * + *

For non-broadcast, the created {@link BufferConsumer} is only for the target channel. + * + *

For broadcast, all the channels share the same requested {@link BufferBuilder} and the created + * {@link BufferConsumer} is copied for every channel. + */ + @Override + public BufferBuilder requestNewBufferBuilder(int targetChannel) throws IOException, InterruptedException { + checkState(bufferBuilder == null || bufferBuilder.isFinished()); + + BufferBuilder builder = targetPartition.getBufferBuilder(); + if (randomTriggered) { + targetPartition.addBufferConsumer(builder.createBufferConsumer(), targetChannel); + } else { + try (BufferConsumer bufferConsumer = builder.createBufferConsumer()) { + for (int channel = 0; channel < numberOfChannels; channel++) { + targetPartition.addBufferConsumer(bufferConsumer.copy(), channel); + } + } + } + + bufferBuilder = builder; + return builder; + } + + @Override + public void tryFinishCurrentBufferBuilder(int targetChannel) { + if (bufferBuilder == null) { + return; + } + + BufferBuilder builder = bufferBuilder; + bufferBuilder = null; + + finishBufferBuilder(builder); + } + + @Override + public void emptyCurrentBufferBuilder(int targetChannel) { + bufferBuilder = null; + } + + @Override + public void closeBufferBuilder(int targetChannel) { + closeBufferBuilder(); + } + + @Override + public void clearBuffers() { + closeBufferBuilder(); + } + + private void closeBufferBuilder() { + if (bufferBuilder != null) { + bufferBuilder.finish(); + bufferBuilder = null; + } + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/ChannelSelectorRecordWriter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/ChannelSelectorRecordWriter.java new file mode 100644 index 0000000000000000000000000000000000000000..5f5e5964b9889ea22463c2f217eea4e4202dbbd8 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/ChannelSelectorRecordWriter.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.io.network.api.writer; + +import org.apache.flink.core.io.IOReadableWritable; +import org.apache.flink.runtime.io.network.buffer.BufferBuilder; + +import java.io.IOException; + +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.Preconditions.checkState; + +/** + * A regular record-oriented runtime result writer. + * + *

The ChannelSelectorRecordWriter extends the {@link RecordWriter} and maintains an array of + * {@link BufferBuilder}s for all the channels. The {@link #emit(IOReadableWritable)} + * operation is based on {@link ChannelSelector} to select the target channel. + * + * @param the type of the record that can be emitted with this record writer + */ +public final class ChannelSelectorRecordWriter extends RecordWriter { + + private final ChannelSelector channelSelector; + + /** Every subpartition maintains a separate buffer builder which might be null. */ + private final BufferBuilder[] bufferBuilders; + + ChannelSelectorRecordWriter( + ResultPartitionWriter writer, + ChannelSelector channelSelector, + long timeout, + String taskName) { + super(writer, timeout, taskName); + + this.channelSelector = checkNotNull(channelSelector); + this.channelSelector.setup(numberOfChannels); + + this.bufferBuilders = new BufferBuilder[numberOfChannels]; + } + + @Override + public void emit(T record) throws IOException, InterruptedException { + emit(record, channelSelector.selectChannel(record)); + } + + @Override + public void randomEmit(T record) throws IOException, InterruptedException { + emit(record, rng.nextInt(numberOfChannels)); + } + + /** + * The record is serialized into intermediate serialization buffer which is then copied + * into the target buffer for every channel. + */ + @Override + public void broadcastEmit(T record) throws IOException, InterruptedException { + checkErroneous(); + + serializer.serializeRecord(record); + + boolean pruneAfterCopying = false; + for (int targetChannel = 0; targetChannel < numberOfChannels; targetChannel++) { + if (copyFromSerializerToTargetChannel(targetChannel)) { + pruneAfterCopying = true; + } + } + + if (pruneAfterCopying) { + serializer.prune(); + } + } + + @Override + public BufferBuilder getBufferBuilder(int targetChannel) throws IOException, InterruptedException { + if (bufferBuilders[targetChannel] != null) { + return bufferBuilders[targetChannel]; + } else { + return requestNewBufferBuilder(targetChannel); + } + } + + @Override + public BufferBuilder requestNewBufferBuilder(int targetChannel) throws IOException, InterruptedException { + checkState(bufferBuilders[targetChannel] == null || bufferBuilders[targetChannel].isFinished()); + + BufferBuilder bufferBuilder = targetPartition.getBufferBuilder(); + targetPartition.addBufferConsumer(bufferBuilder.createBufferConsumer(), targetChannel); + bufferBuilders[targetChannel] = bufferBuilder; + return bufferBuilder; + } + + @Override + public void tryFinishCurrentBufferBuilder(int targetChannel) { + if (bufferBuilders[targetChannel] == null) { + return; + } + BufferBuilder bufferBuilder = bufferBuilders[targetChannel]; + bufferBuilders[targetChannel] = null; + + finishBufferBuilder(bufferBuilder); + } + + @Override + public void emptyCurrentBufferBuilder(int targetChannel) { + bufferBuilders[targetChannel] = null; + } + + @Override + public void closeBufferBuilder(int targetChannel) { + if (bufferBuilders[targetChannel] != null) { + bufferBuilders[targetChannel].finish(); + bufferBuilders[targetChannel] = null; + } + } + + @Override + public void clearBuffers() { + for (int index = 0; index < numberOfChannels; index++) { + closeBufferBuilder(index); + } + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java index cc40df064ee7854d48701a1d2eedbbeca109ae86..93fa826305461a2fc191f4e8209955773ec9cfb9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriter.java @@ -18,6 +18,7 @@ package org.apache.flink.runtime.io.network.api.writer; +import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.core.io.IOReadableWritable; import org.apache.flink.metrics.Counter; import org.apache.flink.metrics.SimpleCounter; @@ -33,8 +34,9 @@ import org.apache.flink.util.XORShiftRandom; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + import java.io.IOException; -import java.util.Optional; import java.util.Random; import static org.apache.flink.runtime.io.network.api.serialization.RecordSerializer.SerializationResult; @@ -42,7 +44,7 @@ import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkState; /** - * A record-oriented runtime result writer. + * An abstract record-oriented runtime result writer. * *

The RecordWriter wraps the runtime's {@link ResultPartitionWriter} and takes care of * serializing records into buffers. @@ -54,23 +56,21 @@ import static org.apache.flink.util.Preconditions.checkState; * * @param the type of the record that can be emitted with this record writer */ -public class RecordWriter { - - private static final Logger LOG = LoggerFactory.getLogger(RecordWriter.class); - - private final ResultPartitionWriter targetPartition; +public abstract class RecordWriter { - private final ChannelSelector channelSelector; + /** Default name for the output flush thread, if no name with a task reference is given. */ + @VisibleForTesting + public static final String DEFAULT_OUTPUT_FLUSH_THREAD_NAME = "OutputFlusher"; - private final int numberOfChannels; + private static final Logger LOG = LoggerFactory.getLogger(RecordWriter.class); - private final int[] broadcastChannels; + protected final ResultPartitionWriter targetPartition; - private final RecordSerializer serializer; + protected final int numberOfChannels; - private final Optional[] bufferBuilders; + protected final RecordSerializer serializer; - private final Random rng = new XORShiftRandom(); + protected final Random rng = new XORShiftRandom(); private Counter numBytesOut = new SimpleCounter(); @@ -78,79 +78,39 @@ public class RecordWriter { private final boolean flushAlways; - /** Default name for teh output flush thread, if no name with a task reference is given. */ - private static final String DEFAULT_OUTPUT_FLUSH_THREAD_NAME = "OutputFlusher"; - /** The thread that periodically flushes the output, to give an upper latency bound. */ - private final Optional outputFlusher; + @Nullable + private final OutputFlusher outputFlusher; /** To avoid synchronization overhead on the critical path, best-effort error tracking is enough here.*/ private Throwable flusherException; - RecordWriter(ResultPartitionWriter writer, ChannelSelector channelSelector, long timeout, String taskName) { + RecordWriter(ResultPartitionWriter writer, long timeout, String taskName) { this.targetPartition = writer; - this.channelSelector = channelSelector; this.numberOfChannels = writer.getNumberOfSubpartitions(); - this.channelSelector.setup(numberOfChannels); this.serializer = new SpanningRecordSerializer(); - this.bufferBuilders = new Optional[numberOfChannels]; - this.broadcastChannels = new int[numberOfChannels]; - for (int i = 0; i < numberOfChannels; i++) { - broadcastChannels[i] = i; - bufferBuilders[i] = Optional.empty(); - } checkArgument(timeout >= -1); this.flushAlways = (timeout == 0); if (timeout == -1 || timeout == 0) { - outputFlusher = Optional.empty(); + outputFlusher = null; } else { String threadName = taskName == null ? DEFAULT_OUTPUT_FLUSH_THREAD_NAME : DEFAULT_OUTPUT_FLUSH_THREAD_NAME + " for " + taskName; - outputFlusher = Optional.of(new OutputFlusher(threadName, timeout)); - outputFlusher.get().start(); + outputFlusher = new OutputFlusher(threadName, timeout); + outputFlusher.start(); } } - public void emit(T record) throws IOException, InterruptedException { + protected void emit(T record, int targetChannel) throws IOException, InterruptedException { checkErroneous(); - emit(record, channelSelector.selectChannel(record)); - } - /** - * This is used to broadcast Streaming Watermarks in-band with records. This ignores - * the {@link ChannelSelector}. - */ - public void broadcastEmit(T record) throws IOException, InterruptedException { - checkErroneous(); serializer.serializeRecord(record); - boolean pruneAfterCopying = false; - for (int channel : broadcastChannels) { - if (copyFromSerializerToTargetChannel(channel)) { - pruneAfterCopying = true; - } - } - // Make sure we don't hold onto the large intermediate serialization buffer for too long - if (pruneAfterCopying) { - serializer.prune(); - } - } - - /** - * This is used to send LatencyMarks to a random target channel. - */ - public void randomEmit(T record) throws IOException, InterruptedException { - emit(record, rng.nextInt(numberOfChannels)); - } - - private void emit(T record, int targetChannel) throws IOException, InterruptedException { - serializer.serializeRecord(record); - if (copyFromSerializerToTargetChannel(targetChannel)) { serializer.prune(); } @@ -160,7 +120,7 @@ public class RecordWriter { * @param targetChannel * @return true if the intermediate serialization buffer should be pruned */ - private boolean copyFromSerializerToTargetChannel(int targetChannel) throws IOException, InterruptedException { + protected boolean copyFromSerializerToTargetChannel(int targetChannel) throws IOException, InterruptedException { // We should reset the initial position of the intermediate serialization buffer before // copying, so the serialization results can be copied to multiple target buffers. serializer.reset(); @@ -169,15 +129,14 @@ public class RecordWriter { BufferBuilder bufferBuilder = getBufferBuilder(targetChannel); SerializationResult result = serializer.copyToBufferBuilder(bufferBuilder); while (result.isFullBuffer()) { - numBytesOut.inc(bufferBuilder.finish()); - numBuffersOut.inc(); + finishBufferBuilder(bufferBuilder); // If this was a full record, we are done. Not breaking out of the loop at this point // will lead to another buffer request before breaking out (that would not be a // problem per se, but it can lead to stalls in the pipeline). if (result.isFullRecord()) { pruneTriggered = true; - bufferBuilders[targetChannel] = Optional.empty(); + emptyCurrentBufferBuilder(targetChannel); break; } @@ -187,7 +146,7 @@ public class RecordWriter { checkState(!serializer.hasSerializedData(), "All data should be written at once"); if (flushAlways) { - targetPartition.flush(targetChannel); + flushTargetPartition(targetChannel); } return pruneTriggered; } @@ -211,10 +170,8 @@ public class RecordWriter { targetPartition.flushAll(); } - public void clearBuffers() { - for (int targetChannel = 0; targetChannel < numberOfChannels; targetChannel++) { - closeBufferBuilder(targetChannel); - } + protected void flushTargetPartition(int targetChannel) { + targetPartition.flush(targetChannel); } /** @@ -225,46 +182,56 @@ public class RecordWriter { numBuffersOut = metrics.getNumBuffersOutCounter(); } - /** - * Marks the current {@link BufferBuilder} as finished and clears the state for next one. - */ - private void tryFinishCurrentBufferBuilder(int targetChannel) { - if (!bufferBuilders[targetChannel].isPresent()) { - return; - } - BufferBuilder bufferBuilder = bufferBuilders[targetChannel].get(); - bufferBuilders[targetChannel] = Optional.empty(); + protected void finishBufferBuilder(BufferBuilder bufferBuilder) { numBytesOut.inc(bufferBuilder.finish()); numBuffersOut.inc(); } + /** + * This is used to send regular records. + */ + public abstract void emit(T record) throws IOException, InterruptedException; + + /** + * This is used to send LatencyMarks to a random target channel. + */ + public abstract void randomEmit(T record) throws IOException, InterruptedException; + + /** + * This is used to broadcast streaming Watermarks in-band with records. + */ + public abstract void broadcastEmit(T record) throws IOException, InterruptedException; + /** * The {@link BufferBuilder} may already exist if not filled up last time, otherwise we need * request a new one for this target channel. */ - private BufferBuilder getBufferBuilder(int targetChannel) throws IOException, InterruptedException { - if (bufferBuilders[targetChannel].isPresent()) { - return bufferBuilders[targetChannel].get(); - } else { - return requestNewBufferBuilder(targetChannel); - } - } + abstract BufferBuilder getBufferBuilder(int targetChannel) throws IOException, InterruptedException; + + /** + * Requests a new {@link BufferBuilder} for the target channel and returns it. + */ + abstract BufferBuilder requestNewBufferBuilder(int targetChannel) throws IOException, InterruptedException; - private BufferBuilder requestNewBufferBuilder(int targetChannel) throws IOException, InterruptedException { - checkState(!bufferBuilders[targetChannel].isPresent() || bufferBuilders[targetChannel].get().isFinished()); + /** + * Marks the current {@link BufferBuilder} as finished if present and clears the state for next one. + */ + abstract void tryFinishCurrentBufferBuilder(int targetChannel); - BufferBuilder bufferBuilder = targetPartition.getBufferBuilder(); - bufferBuilders[targetChannel] = Optional.of(bufferBuilder); - targetPartition.addBufferConsumer(bufferBuilder.createBufferConsumer(), targetChannel); - return bufferBuilder; - } + /** + * Marks the current {@link BufferBuilder} as empty for the target channel. + */ + abstract void emptyCurrentBufferBuilder(int targetChannel); - private void closeBufferBuilder(int targetChannel) { - if (bufferBuilders[targetChannel].isPresent()) { - bufferBuilders[targetChannel].get().finish(); - bufferBuilders[targetChannel] = Optional.empty(); - } - } + /** + * Marks the current {@link BufferBuilder} as finished and releases the resources for the target channel. + */ + abstract void closeBufferBuilder(int targetChannel); + + /** + * Closes the {@link BufferBuilder}s for all the channels. + */ + public abstract void clearBuffers(); /** * Closes the writer. This stops the flushing thread (if there is one). @@ -272,10 +239,10 @@ public class RecordWriter { public void close() { clearBuffers(); // make sure we terminate the thread in any case - if (outputFlusher.isPresent()) { - outputFlusher.get().terminate(); + if (outputFlusher != null) { + outputFlusher.terminate(); try { - outputFlusher.get().join(); + outputFlusher.join(); } catch (InterruptedException e) { // ignore on close // restore interrupt flag to fast exit further blocking calls @@ -296,7 +263,7 @@ public class RecordWriter { } } - private void checkErroneous() throws IOException { + protected void checkErroneous() throws IOException { if (flusherException != null) { throw new IOException("An exception happened while flushing the outputs", flusherException); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterBuilder.java index 79b372b6bc21b0d04eeb466dfe6e5c62cdd0a311..b6c99b682e87b97f5ea0c85590949ef3f508cba8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterBuilder.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterBuilder.java @@ -18,37 +18,39 @@ package org.apache.flink.runtime.io.network.api.writer; +import org.apache.flink.core.io.IOReadableWritable; + /** * Utility class to encapsulate the logic of building a {@link RecordWriter} instance. */ -public class RecordWriterBuilder { +public class RecordWriterBuilder { - private ChannelSelector selector = new RoundRobinChannelSelector(); + private ChannelSelector selector = new RoundRobinChannelSelector<>(); private long timeout = -1; private String taskName = "test"; - public RecordWriterBuilder setChannelSelector(ChannelSelector selector) { + public RecordWriterBuilder setChannelSelector(ChannelSelector selector) { this.selector = selector; return this; } - public RecordWriterBuilder setTimeout(long timeout) { + public RecordWriterBuilder setTimeout(long timeout) { this.timeout = timeout; return this; } - public RecordWriterBuilder setTaskName(String taskName) { + public RecordWriterBuilder setTaskName(String taskName) { this.taskName = taskName; return this; } - public RecordWriter build(ResultPartitionWriter writer) { + public RecordWriter build(ResultPartitionWriter writer) { if (selector.isBroadcast()) { - return new BroadcastRecordWriter(writer, selector, timeout, taskName); + return new BroadcastRecordWriter<>(writer, timeout, taskName); } else { - return new RecordWriter(writer, selector, timeout, taskName); + return new ChannelSelectorRecordWriter<>(writer, selector, timeout, taskName); } } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferBuilder.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferBuilder.java index 6fb067ef8c36b2cab4edcdf7032fbc7acbd9b679..bcd42d23b1ba9c89e088dbd313ce37cedcec4b01 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferBuilder.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferBuilder.java @@ -40,24 +40,23 @@ public class BufferBuilder { private final SettablePositionMarker positionMarker = new SettablePositionMarker(); - private boolean bufferConsumerCreated = false; - public BufferBuilder(MemorySegment memorySegment, BufferRecycler recycler) { this.memorySegment = checkNotNull(memorySegment); this.recycler = checkNotNull(recycler); } /** - * @return created matching instance of {@link BufferConsumer} to this {@link BufferBuilder}. There can exist only - * one {@link BufferConsumer} per each {@link BufferBuilder} and vice versa. + * This method always creates a {@link BufferConsumer} starting from the current writer offset. Data written to + * {@link BufferBuilder} before creation of {@link BufferConsumer} won't be visible for that {@link BufferConsumer}. + * + * @return created matching instance of {@link BufferConsumer} to this {@link BufferBuilder}. */ public BufferConsumer createBufferConsumer() { - checkState(!bufferConsumerCreated, "There can not exists two BufferConsumer for one BufferBuilder"); - bufferConsumerCreated = true; return new BufferConsumer( memorySegment, recycler, - positionMarker); + positionMarker, + positionMarker.cachedPosition); } /** diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferConsumer.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferConsumer.java index b58a627dfe145a963ba2af288c8c17ce1a1b1c5f..ecebc6734495f420bd8d44b18ef232eee5687951 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferConsumer.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/buffer/BufferConsumer.java @@ -45,16 +45,17 @@ public class BufferConsumer implements Closeable { private int currentReaderPosition; /** - * Constructs {@link BufferConsumer} instance with content that can be changed by {@link BufferBuilder}. + * Constructs {@link BufferConsumer} instance with the initial reader position. */ public BufferConsumer( MemorySegment memorySegment, BufferRecycler recycler, - PositionMarker currentWriterPosition) { + PositionMarker currentWriterPosition, + int currentReaderPosition) { this( new NetworkBuffer(checkNotNull(memorySegment), checkNotNull(recycler), true), currentWriterPosition, - 0); + currentReaderPosition); } /** @@ -136,6 +137,17 @@ public class BufferConsumer implements Closeable { return writerPosition.getCached(); } + int getCurrentReaderPosition() { + return currentReaderPosition; + } + + /** + * Returns true if there is new data available for reading. + */ + public boolean isDataAvailable() { + return currentReaderPosition < writerPosition.getLatest(); + } + /** * Cached reading wrapper around {@link PositionMarker}. * @@ -163,6 +175,10 @@ public class BufferConsumer implements Closeable { return PositionMarker.getAbsolute(cachedPosition); } + private int getLatest() { + return PositionMarker.getAbsolute(positionMarker.get()); + } + private void update() { this.cachedPosition = positionMarker.get(); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java index 0c38ae2c2cb4df594fa05c5e509d3519f1471b30..ecf695696aabb116e9cef101b017bba1c2f1de92 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartition.java @@ -307,8 +307,8 @@ class PipelinedSubpartition extends ResultSubpartition { } // if there is more then 1 buffer, we already notified the reader // (at the latest when adding the second buffer) - notifyDataAvailable = !flushRequested && buffers.size() == 1; - flushRequested = true; + notifyDataAvailable = !flushRequested && buffers.size() == 1 && buffers.peek().isDataAvailable(); + flushRequested = flushRequested || buffers.size() > 1 || notifyDataAvailable; } if (notifyDataAvailable) { notifyDataAvailable(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGate.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGate.java index 7d6ea818e8ec2e3e6f5624c8e5d9c13c905c1949..6a9ac1744f809da335aeda40e710a9b018c2ca24 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGate.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/partition/consumer/InputGate.java @@ -19,7 +19,7 @@ package org.apache.flink.runtime.io.network.partition.consumer; import org.apache.flink.runtime.event.TaskEvent; -import org.apache.flink.runtime.io.AsyncDataInput; +import org.apache.flink.runtime.io.PullingAsyncDataInput; import java.io.IOException; import java.util.Optional; @@ -69,7 +69,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull; * will have an input gate attached to it. This will provide its input, which will consist of one * subpartition from each partition of the intermediate result. */ -public abstract class InputGate implements AsyncDataInput, AutoCloseable { +public abstract class InputGate implements PullingAsyncDataInput, AutoCloseable { protected CompletableFuture isAvailable = new CompletableFuture<>(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadTask.java b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadTask.java index 2ee76fcf5a53348617d37fcd23c0c94f1804cf17..70681fd145d9a098127b583cf5978dbf3cd44a1f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadTask.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/iterative/task/IterationHeadTask.java @@ -143,7 +143,7 @@ public class IterationHeadTask extends AbstractIte throw new Exception("Error: Inconsistent head task setup - wrong mapping of output gates."); } // now, we can instantiate the sync gate - this.toSync = new RecordWriterBuilder().build(getEnvironment().getWriter(syncGateIndex)); + this.toSync = new RecordWriterBuilder<>().build(getEnvironment().getWriter(syncGateIndex)); this.toSyncPartitionId = getEnvironment().getWriter(syncGateIndex).getPartitionId(); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java index 377f870ca0ad5b89e9ebeb1c5b55095cd17cf514..3c6f935753711d943176e57230d8252dd59e1590 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobgraph/JobGraph.java @@ -68,7 +68,7 @@ public class JobGraph implements Serializable { private final Configuration jobConfiguration = new Configuration(); /** ID of this job. May be set if specific job id is desired (e.g. session management) */ - private final JobID jobID; + private JobID jobID; /** Name of this job. */ private final String jobName; @@ -189,6 +189,13 @@ public class JobGraph implements Serializable { return this.jobID; } + /** + * Sets the ID of the job. + */ + public void setJobID(JobID jobID) { + this.jobID = jobID; + } + /** * Returns the name assigned to the job graph. * diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphStore.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphStore.java index 634ec3790ea0b637027b28a97fa2a8a9bfd37a93..61cf8c6f7415be07f5343562767940a5d18f02f6 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphStore.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphStore.java @@ -28,7 +28,7 @@ import java.util.Collection; /** * {@link JobGraph} instances for recovery. */ -public interface JobGraphStore { +public interface JobGraphStore extends JobGraphWriter { /** * Starts the {@link JobGraphStore} service. @@ -47,29 +47,6 @@ public interface JobGraphStore { @Nullable JobGraph recoverJobGraph(JobID jobId) throws Exception; - /** - * Adds the {@link JobGraph} instance. - * - *

If a job graph with the same {@link JobID} exists, it is replaced. - */ - void putJobGraph(JobGraph jobGraph) throws Exception; - - /** - * Removes the {@link JobGraph} with the given {@link JobID} if it exists. - */ - void removeJobGraph(JobID jobId) throws Exception; - - /** - * Releases the locks on the specified {@link JobGraph}. - * - * Releasing the locks allows that another instance can delete the job from - * the {@link JobGraphStore}. - * - * @param jobId specifying the job to release the locks for - * @throws Exception if the locks cannot be released - */ - void releaseJobGraph(JobID jobId) throws Exception; - /** * Get all job ids of submitted job graphs to the submitted job graph store. * diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphStoreFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphStoreFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..27110025b811b29c725d9e0ea53c530e425275ae --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphStoreFactory.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.jobmanager; + +/** + * Factory for {@link JobGraphStore}. + */ +public interface JobGraphStoreFactory { + + /** + * Creates a {@link JobGraphStore}. + * + * @return a {@link JobGraphStore} instance + */ + JobGraphStore create(); +} diff --git a/flink-clients/src/main/java/org/apache/flink/client/program/NewClusterClient.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphWriter.java similarity index 51% rename from flink-clients/src/main/java/org/apache/flink/client/program/NewClusterClient.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphWriter.java index 513f7da5f0c9613dbf2a99fc2271ede09e721ced..7a987952a94a8f20c4538b0f1c06710869268d5e 100644 --- a/flink-clients/src/main/java/org/apache/flink/client/program/NewClusterClient.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/JobGraphWriter.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,35 +16,35 @@ * limitations under the License. */ -package org.apache.flink.client.program; +package org.apache.flink.runtime.jobmanager; import org.apache.flink.api.common.JobID; -import org.apache.flink.api.common.JobSubmissionResult; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.jobmaster.JobResult; - -import javax.annotation.Nonnull; - -import java.util.concurrent.CompletableFuture; /** - * Interface for the new cluster client. + * Allows to store and remove job graphs. */ -public interface NewClusterClient { - +public interface JobGraphWriter { /** - * Submit the given {@link JobGraph} to the cluster. + * Adds the {@link JobGraph} instance. * - * @param jobGraph to submit - * @return Future which is completed with the {@link JobSubmissionResult} + *

If a job graph with the same {@link JobID} exists, it is replaced. */ - CompletableFuture submitJob(@Nonnull JobGraph jobGraph); + void putJobGraph(JobGraph jobGraph) throws Exception; /** - * Request the {@link JobResult} for the given {@link JobID}. + * Removes the {@link JobGraph} with the given {@link JobID} if it exists. + */ + void removeJobGraph(JobID jobId) throws Exception; + + /** + * Releases the locks on the specified {@link JobGraph}. + * + * Releasing the locks allows that another instance can delete the job from + * the {@link JobGraphStore}. * - * @param jobId for which to request the {@link JobResult} - * @return Future which is completed with the {@link JobResult} + * @param jobId specifying the job to release the locks for + * @throws Exception if the locks cannot be released */ - CompletableFuture requestJobResult(@Nonnull JobID jobId); + void releaseJobGraph(JobID jobId) throws Exception; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/NoResourceAvailableException.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/NoResourceAvailableException.java index bea7b8a06bc79493b55b619f74e08aedb2d07e07..905b224ced9585d754349143839ba5f5d3b52066 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/NoResourceAvailableException.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmanager/scheduler/NoResourceAvailableException.java @@ -19,40 +19,20 @@ package org.apache.flink.runtime.jobmanager.scheduler; import org.apache.flink.runtime.JobException; -import org.apache.flink.runtime.throwable.ThrowableAnnotation; -import org.apache.flink.runtime.throwable.ThrowableType; -@ThrowableAnnotation(ThrowableType.NonRecoverableError) +/** + * Indicates resource allocation failures. + */ public class NoResourceAvailableException extends JobException { private static final long serialVersionUID = -2249953165298717803L; - + private static final String BASE_MESSAGE = "Not enough free slots available to run the job. " - + "You can decrease the operator parallelism or increase the number of slots per TaskManager in the configuration."; + + "You can decrease the operator parallelism or increase the number of slots per TaskManager in the configuration."; public NoResourceAvailableException() { super(BASE_MESSAGE); } - - public NoResourceAvailableException(ScheduledUnit unit) { - super("No resource available to schedule unit " + unit - + ". You can decrease the operator parallelism or increase the number of slots per TaskManager in the configuration."); - } - - public NoResourceAvailableException(int numInstances, int numSlotsTotal, int availableSlots) { - super(String.format("%s Resources available to scheduler: Number of instances=%d, total number of slots=%d, available slots=%d", - BASE_MESSAGE, numInstances, numSlotsTotal, availableSlots)); - } - - NoResourceAvailableException(ScheduledUnit task, int numInstances, int numSlotsTotal, int availableSlots) { - super(String.format("%s Task to schedule: < %s > with groupID < %s > in sharing group < %s >. Resources available to scheduler: Number of instances=%d, total number of slots=%d, available slots=%d", - BASE_MESSAGE, task.getTaskToExecute(), - task.getCoLocationConstraint() == null ? task.getTaskToExecute().getVertex().getJobvertexId() : task.getCoLocationConstraint().getGroupId(), - task.getSlotSharingGroupId(), - numInstances, - numSlotsTotal, - availableSlots)); - } public NoResourceAvailableException(String message) { super(message); @@ -63,14 +43,13 @@ public class NoResourceAvailableException extends JobException { } // -------------------------------------------------------------------------------------------- - + @Override public boolean equals(Object obj) { - return obj instanceof NoResourceAvailableException && - getMessage().equals(((NoResourceAvailableException) obj).getMessage()); - + return obj instanceof NoResourceAvailableException && + getMessage().equals(((NoResourceAvailableException) obj).getMessage()); } - + @Override public int hashCode() { return getMessage().hashCode(); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java index 846018b96a7b3d8964cb656ae0b925d5b776edd8..4a1d1c7a13f2048cc362b849fbffc34d7994bb49 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunner.java @@ -18,424 +18,44 @@ package org.apache.flink.runtime.jobmaster; -import org.apache.flink.annotation.VisibleForTesting; -import org.apache.flink.runtime.client.JobExecutionException; -import org.apache.flink.runtime.concurrent.FutureUtils; -import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager; +import org.apache.flink.api.common.JobID; import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.highavailability.RunningJobsRegistry; -import org.apache.flink.runtime.highavailability.RunningJobsRegistry.JobSchedulingStatus; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.jobmanager.OnCompletionActions; -import org.apache.flink.runtime.jobmaster.factories.JobMasterServiceFactory; -import org.apache.flink.runtime.leaderelection.LeaderContender; -import org.apache.flink.runtime.leaderelection.LeaderElectionService; -import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.util.AutoCloseableAsync; -import org.apache.flink.util.ExceptionUtils; -import org.apache.flink.util.FlinkException; -import org.apache.flink.util.function.FunctionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nonnull; - -import java.io.IOException; -import java.util.UUID; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.Executor; - -import static org.apache.flink.util.Preconditions.checkArgument; -import static org.apache.flink.util.Preconditions.checkNotNull; /** - * The runner for the job manager. It deals with job level leader election and make underlying job manager - * properly reacted. + * Interface for a runner which executes a {@link JobMaster}. */ -public class JobManagerRunner implements LeaderContender, OnCompletionActions, AutoCloseableAsync { - - private static final Logger log = LoggerFactory.getLogger(JobManagerRunner.class); - - // ------------------------------------------------------------------------ - - /** Lock to ensure that this runner can deal with leader election event and job completion notifies simultaneously. */ - private final Object lock = new Object(); - - /** The job graph needs to run. */ - private final JobGraph jobGraph; - - /** Used to check whether a job needs to be run. */ - private final RunningJobsRegistry runningJobsRegistry; - - /** Leader election for this job. */ - private final LeaderElectionService leaderElectionService; - - private final LibraryCacheManager libraryCacheManager; - - private final Executor executor; - - private final JobMasterService jobMasterService; - - private final FatalErrorHandler fatalErrorHandler; - - private final CompletableFuture resultFuture; - - private final CompletableFuture terminationFuture; - - private CompletableFuture leadershipOperation; - - /** flag marking the runner as shut down. */ - private volatile boolean shutdown; - - private volatile CompletableFuture leaderGatewayFuture; - - // ------------------------------------------------------------------------ +public interface JobManagerRunner extends AutoCloseableAsync { /** - * Exceptions that occur while creating the JobManager or JobManagerRunner are directly - * thrown and not reported to the given {@code FatalErrorHandler}. + * Start the execution of the {@link JobMaster}. * - * @throws Exception Thrown if the runner cannot be set up, because either one of the - * required services could not be started, or the Job could not be initialized. + * @throws Exception if the JobMaster cannot be started */ - public JobManagerRunner( - final JobGraph jobGraph, - final JobMasterServiceFactory jobMasterFactory, - final HighAvailabilityServices haServices, - final LibraryCacheManager libraryCacheManager, - final Executor executor, - final FatalErrorHandler fatalErrorHandler) throws Exception { - - this.resultFuture = new CompletableFuture<>(); - this.terminationFuture = new CompletableFuture<>(); - this.leadershipOperation = CompletableFuture.completedFuture(null); - - // make sure we cleanly shut down out JobManager services if initialization fails - try { - this.jobGraph = checkNotNull(jobGraph); - this.libraryCacheManager = checkNotNull(libraryCacheManager); - this.executor = checkNotNull(executor); - this.fatalErrorHandler = checkNotNull(fatalErrorHandler); - - checkArgument(jobGraph.getNumberOfVertices() > 0, "The given job is empty"); - - // libraries and class loader first - try { - libraryCacheManager.registerJob( - jobGraph.getJobID(), jobGraph.getUserJarBlobKeys(), jobGraph.getClasspaths()); - } catch (IOException e) { - throw new Exception("Cannot set up the user code libraries: " + e.getMessage(), e); - } - - final ClassLoader userCodeLoader = libraryCacheManager.getClassLoader(jobGraph.getJobID()); - if (userCodeLoader == null) { - throw new Exception("The user code class loader could not be initialized."); - } - - // high availability services next - this.runningJobsRegistry = haServices.getRunningJobsRegistry(); - this.leaderElectionService = haServices.getJobManagerLeaderElectionService(jobGraph.getJobID()); - - this.leaderGatewayFuture = new CompletableFuture<>(); - - // now start the JobManager - this.jobMasterService = jobMasterFactory.createJobMasterService(jobGraph, this, userCodeLoader); - } - catch (Throwable t) { - terminationFuture.completeExceptionally(t); - resultFuture.completeExceptionally(t); - - throw new JobExecutionException(jobGraph.getJobID(), "Could not set up JobManager", t); - } - } - - //---------------------------------------------------------------------------------------------- - // Getter - //---------------------------------------------------------------------------------------------- - - public CompletableFuture getLeaderGatewayFuture() { - return leaderGatewayFuture; - } - - public JobGraph getJobGraph() { - return jobGraph; - } - - public CompletableFuture getResultFuture() { - return resultFuture; - } - - //---------------------------------------------------------------------------------------------- - // Lifecycle management - //---------------------------------------------------------------------------------------------- - - public void start() throws Exception { - try { - leaderElectionService.start(this); - } catch (Exception e) { - log.error("Could not start the JobManager because the leader election service did not start.", e); - throw new Exception("Could not start the leader election service.", e); - } - } - - @Override - public CompletableFuture closeAsync() { - synchronized (lock) { - if (!shutdown) { - shutdown = true; - - setNewLeaderGatewayFuture(); - leaderGatewayFuture.completeExceptionally(new FlinkException("JobMaster has been shut down.")); - - final CompletableFuture jobManagerTerminationFuture = jobMasterService.closeAsync(); - - jobManagerTerminationFuture.whenComplete( - (Void ignored, Throwable throwable) -> { - try { - leaderElectionService.stop(); - } catch (Throwable t) { - throwable = ExceptionUtils.firstOrSuppressed(t, ExceptionUtils.stripCompletionException(throwable)); - } - - libraryCacheManager.unregisterJob(jobGraph.getJobID()); - - if (throwable != null) { - terminationFuture.completeExceptionally( - new FlinkException("Could not properly shut down the JobManagerRunner", throwable)); - } else { - terminationFuture.complete(null); - } - }); - - terminationFuture.whenComplete( - (Void ignored, Throwable throwable) -> { - resultFuture.completeExceptionally(new JobNotFinishedException(jobGraph.getJobID())); - }); - } - - return terminationFuture; - } - } - - //---------------------------------------------------------------------------------------------- - // Result and error handling methods - //---------------------------------------------------------------------------------------------- + void start() throws Exception; /** - * Job completion notification triggered by JobManager. + * Get the {@link JobMasterGateway} of the {@link JobMaster}. The future is + * only completed if the JobMaster becomes leader. + * + * @return Future with the JobMasterGateway once the underlying JobMaster becomes leader */ - @Override - public void jobReachedGloballyTerminalState(ArchivedExecutionGraph executionGraph) { - unregisterJobFromHighAvailability(); - // complete the result future with the terminal execution graph - resultFuture.complete(executionGraph); - } + CompletableFuture getJobMasterGateway(); /** - * Job completion notification triggered by self. + * Get the result future of this runner. The future is completed once the executed + * job reaches a globally terminal state. + * + * @return Future which is completed with the job result */ - @Override - public void jobFinishedByOther() { - resultFuture.completeExceptionally(new JobNotFinishedException(jobGraph.getJobID())); - } - - @Override - public void jobMasterFailed(Throwable cause) { - handleJobManagerRunnerError(cause); - } - - private void handleJobManagerRunnerError(Throwable cause) { - if (ExceptionUtils.isJvmFatalError(cause)) { - fatalErrorHandler.onFatalError(cause); - } else { - resultFuture.completeExceptionally(cause); - } - } + CompletableFuture getResultFuture(); /** - * Marks this runner's job as not running. Other JobManager will not recover the job - * after this call. + * Get the job id of the executed job. * - *

This method never throws an exception. + * @return job id of the executed job */ - private void unregisterJobFromHighAvailability() { - try { - runningJobsRegistry.setJobFinished(jobGraph.getJobID()); - } - catch (Throwable t) { - log.error("Could not un-register from high-availability services job {} ({})." + - "Other JobManager's may attempt to recover it and re-execute it.", - jobGraph.getName(), jobGraph.getJobID(), t); - } - } - - //---------------------------------------------------------------------------------------------- - // Leadership methods - //---------------------------------------------------------------------------------------------- - - @Override - public void grantLeadership(final UUID leaderSessionID) { - synchronized (lock) { - if (shutdown) { - log.info("JobManagerRunner already shutdown."); - return; - } - - leadershipOperation = leadershipOperation.thenCompose( - (ignored) -> { - synchronized (lock) { - return verifyJobSchedulingStatusAndStartJobManager(leaderSessionID); - } - }); - - handleException(leadershipOperation, "Could not start the job manager."); - } - } - - private CompletableFuture verifyJobSchedulingStatusAndStartJobManager(UUID leaderSessionId) { - final CompletableFuture jobSchedulingStatusFuture = getJobSchedulingStatus(); - - return jobSchedulingStatusFuture.thenCompose( - jobSchedulingStatus -> { - if (jobSchedulingStatus == JobSchedulingStatus.DONE) { - return jobAlreadyDone(); - } else { - return startJobMaster(leaderSessionId); - } - }); - } - - private CompletionStage startJobMaster(UUID leaderSessionId) { - log.info("JobManager runner for job {} ({}) was granted leadership with session id {} at {}.", - jobGraph.getName(), jobGraph.getJobID(), leaderSessionId, getAddress()); - - try { - runningJobsRegistry.setJobRunning(jobGraph.getJobID()); - } catch (IOException e) { - return FutureUtils.completedExceptionally( - new FlinkException( - String.format("Failed to set the job %s to running in the running jobs registry.", jobGraph.getJobID()), - e)); - } - - final CompletableFuture startFuture; - try { - startFuture = jobMasterService.start(new JobMasterId(leaderSessionId)); - } catch (Exception e) { - return FutureUtils.completedExceptionally(new FlinkException("Failed to start the JobMaster.", e)); - } - - final CompletableFuture currentLeaderGatewayFuture = leaderGatewayFuture; - return startFuture.thenAcceptAsync( - (Acknowledge ack) -> confirmLeaderSessionIdIfStillLeader(leaderSessionId, currentLeaderGatewayFuture), - executor); - } - - @Nonnull - private CompletionStage jobAlreadyDone() { - log.info("Granted leader ship but job {} has been finished. ", jobGraph.getJobID()); - jobFinishedByOther(); - return CompletableFuture.completedFuture(null); - } - - private CompletableFuture getJobSchedulingStatus() { - try { - JobSchedulingStatus jobSchedulingStatus = runningJobsRegistry.getJobSchedulingStatus(jobGraph.getJobID()); - return CompletableFuture.completedFuture(jobSchedulingStatus); - } catch (IOException e) { - return FutureUtils.completedExceptionally( - new FlinkException( - String.format("Could not retrieve the job scheduling status for job %s.", jobGraph.getJobID()), - e)); - } - } - - private void confirmLeaderSessionIdIfStillLeader(UUID leaderSessionId, CompletableFuture currentLeaderGatewayFuture) { - if (leaderElectionService.hasLeadership(leaderSessionId)) { - currentLeaderGatewayFuture.complete(jobMasterService.getGateway()); - leaderElectionService.confirmLeaderSessionID(leaderSessionId); - } else { - log.debug("Ignoring confirmation of leader session id because {} is no longer the leader.", getAddress()); - } - } - - @Override - public void revokeLeadership() { - synchronized (lock) { - if (shutdown) { - log.info("JobManagerRunner already shutdown."); - return; - } - - leadershipOperation = leadershipOperation.thenCompose( - (ignored) -> { - synchronized (lock) { - return revokeJobMasterLeadership(); - } - }); - - handleException(leadershipOperation, "Could not suspend the job manager."); - } - } - - private CompletableFuture revokeJobMasterLeadership() { - log.info("JobManager for job {} ({}) was revoked leadership at {}.", - jobGraph.getName(), jobGraph.getJobID(), getAddress()); - - setNewLeaderGatewayFuture(); - - return jobMasterService - .suspend(new FlinkException("JobManager is no longer the leader.")) - .thenApply(FunctionUtils.nullFn()); - } - - private void handleException(CompletableFuture leadershipOperation, String message) { - leadershipOperation.whenComplete( - (ignored, throwable) -> { - if (throwable != null) { - handleJobManagerRunnerError(new FlinkException(message, throwable)); - } - }); - } - - private void setNewLeaderGatewayFuture() { - final CompletableFuture oldLeaderGatewayFuture = leaderGatewayFuture; - - leaderGatewayFuture = new CompletableFuture<>(); - - if (!oldLeaderGatewayFuture.isDone()) { - leaderGatewayFuture.whenComplete( - (JobMasterGateway jobMasterGateway, Throwable throwable) -> { - if (throwable != null) { - oldLeaderGatewayFuture.completeExceptionally(throwable); - } else { - oldLeaderGatewayFuture.complete(jobMasterGateway); - } - }); - } - } - - @Override - public String getAddress() { - return jobMasterService.getAddress(); - } - - @Override - public void handleError(Exception exception) { - log.error("Leader Election Service encountered a fatal error.", exception); - handleJobManagerRunnerError(exception); - } - - //---------------------------------------------------------------------------------------------- - // Testing - //---------------------------------------------------------------------------------------------- - - @VisibleForTesting - boolean isShutdown() { - return shutdown; - } + JobID getJobID(); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerImpl.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerImpl.java new file mode 100644 index 0000000000000000000000000000000000000000..9b3966400f656af233a97322273d6cab5588a37f --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerImpl.java @@ -0,0 +1,452 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.jobmaster; + +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.JobID; +import org.apache.flink.runtime.client.JobExecutionException; +import org.apache.flink.runtime.concurrent.FutureUtils; +import org.apache.flink.runtime.execution.librarycache.LibraryCacheManager; +import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; +import org.apache.flink.runtime.highavailability.HighAvailabilityServices; +import org.apache.flink.runtime.highavailability.RunningJobsRegistry; +import org.apache.flink.runtime.highavailability.RunningJobsRegistry.JobSchedulingStatus; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobmanager.OnCompletionActions; +import org.apache.flink.runtime.jobmaster.factories.JobMasterServiceFactory; +import org.apache.flink.runtime.leaderelection.LeaderContender; +import org.apache.flink.runtime.leaderelection.LeaderElectionService; +import org.apache.flink.runtime.messages.Acknowledge; +import org.apache.flink.runtime.rpc.FatalErrorHandler; +import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.FlinkException; +import org.apache.flink.util.function.FunctionUtils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nonnull; + +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; + +import static org.apache.flink.util.Preconditions.checkArgument; +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * The runner for the job manager. It deals with job level leader election and make underlying job manager + * properly reacted. + */ +public class JobManagerRunnerImpl implements LeaderContender, OnCompletionActions, JobManagerRunner { + + private static final Logger log = LoggerFactory.getLogger(JobManagerRunnerImpl.class); + + // ------------------------------------------------------------------------ + + /** Lock to ensure that this runner can deal with leader election event and job completion notifies simultaneously. */ + private final Object lock = new Object(); + + /** The job graph needs to run. */ + private final JobGraph jobGraph; + + /** Used to check whether a job needs to be run. */ + private final RunningJobsRegistry runningJobsRegistry; + + /** Leader election for this job. */ + private final LeaderElectionService leaderElectionService; + + private final LibraryCacheManager libraryCacheManager; + + private final Executor executor; + + private final JobMasterService jobMasterService; + + private final FatalErrorHandler fatalErrorHandler; + + private final CompletableFuture resultFuture; + + private final CompletableFuture terminationFuture; + + private CompletableFuture leadershipOperation; + + /** flag marking the runner as shut down. */ + private volatile boolean shutdown; + + private volatile CompletableFuture leaderGatewayFuture; + + // ------------------------------------------------------------------------ + + /** + * Exceptions that occur while creating the JobManager or JobManagerRunnerImpl are directly + * thrown and not reported to the given {@code FatalErrorHandler}. + * + * @throws Exception Thrown if the runner cannot be set up, because either one of the + * required services could not be started, or the Job could not be initialized. + */ + public JobManagerRunnerImpl( + final JobGraph jobGraph, + final JobMasterServiceFactory jobMasterFactory, + final HighAvailabilityServices haServices, + final LibraryCacheManager libraryCacheManager, + final Executor executor, + final FatalErrorHandler fatalErrorHandler) throws Exception { + + this.resultFuture = new CompletableFuture<>(); + this.terminationFuture = new CompletableFuture<>(); + this.leadershipOperation = CompletableFuture.completedFuture(null); + + // make sure we cleanly shut down out JobManager services if initialization fails + try { + this.jobGraph = checkNotNull(jobGraph); + this.libraryCacheManager = checkNotNull(libraryCacheManager); + this.executor = checkNotNull(executor); + this.fatalErrorHandler = checkNotNull(fatalErrorHandler); + + checkArgument(jobGraph.getNumberOfVertices() > 0, "The given job is empty"); + + // libraries and class loader first + try { + libraryCacheManager.registerJob( + jobGraph.getJobID(), jobGraph.getUserJarBlobKeys(), jobGraph.getClasspaths()); + } catch (IOException e) { + throw new Exception("Cannot set up the user code libraries: " + e.getMessage(), e); + } + + final ClassLoader userCodeLoader = libraryCacheManager.getClassLoader(jobGraph.getJobID()); + if (userCodeLoader == null) { + throw new Exception("The user code class loader could not be initialized."); + } + + // high availability services next + this.runningJobsRegistry = haServices.getRunningJobsRegistry(); + this.leaderElectionService = haServices.getJobManagerLeaderElectionService(jobGraph.getJobID()); + + this.leaderGatewayFuture = new CompletableFuture<>(); + + // now start the JobManager + this.jobMasterService = jobMasterFactory.createJobMasterService(jobGraph, this, userCodeLoader); + } + catch (Throwable t) { + terminationFuture.completeExceptionally(t); + resultFuture.completeExceptionally(t); + + throw new JobExecutionException(jobGraph.getJobID(), "Could not set up JobManager", t); + } + } + + //---------------------------------------------------------------------------------------------- + // Getter + //---------------------------------------------------------------------------------------------- + + @Override + public CompletableFuture getJobMasterGateway() { + return leaderGatewayFuture; + } + + @Override + public CompletableFuture getResultFuture() { + return resultFuture; + } + + @Override + public JobID getJobID() { + return jobGraph.getJobID(); + } + + //---------------------------------------------------------------------------------------------- + // Lifecycle management + //---------------------------------------------------------------------------------------------- + + @Override + public void start() throws Exception { + try { + leaderElectionService.start(this); + } catch (Exception e) { + log.error("Could not start the JobManager because the leader election service did not start.", e); + throw new Exception("Could not start the leader election service.", e); + } + } + + @Override + public CompletableFuture closeAsync() { + synchronized (lock) { + if (!shutdown) { + shutdown = true; + + setNewLeaderGatewayFuture(); + leaderGatewayFuture.completeExceptionally(new FlinkException("JobMaster has been shut down.")); + + final CompletableFuture jobManagerTerminationFuture = jobMasterService.closeAsync(); + + jobManagerTerminationFuture.whenComplete( + (Void ignored, Throwable throwable) -> { + try { + leaderElectionService.stop(); + } catch (Throwable t) { + throwable = ExceptionUtils.firstOrSuppressed(t, ExceptionUtils.stripCompletionException(throwable)); + } + + libraryCacheManager.unregisterJob(jobGraph.getJobID()); + + if (throwable != null) { + terminationFuture.completeExceptionally( + new FlinkException("Could not properly shut down the JobManagerRunner", throwable)); + } else { + terminationFuture.complete(null); + } + }); + + terminationFuture.whenComplete( + (Void ignored, Throwable throwable) -> { + resultFuture.completeExceptionally(new JobNotFinishedException(jobGraph.getJobID())); + }); + } + + return terminationFuture; + } + } + + //---------------------------------------------------------------------------------------------- + // Result and error handling methods + //---------------------------------------------------------------------------------------------- + + /** + * Job completion notification triggered by JobManager. + */ + @Override + public void jobReachedGloballyTerminalState(ArchivedExecutionGraph executionGraph) { + unregisterJobFromHighAvailability(); + // complete the result future with the terminal execution graph + resultFuture.complete(executionGraph); + } + + /** + * Job completion notification triggered by self. + */ + @Override + public void jobFinishedByOther() { + resultFuture.completeExceptionally(new JobNotFinishedException(jobGraph.getJobID())); + } + + @Override + public void jobMasterFailed(Throwable cause) { + handleJobManagerRunnerError(cause); + } + + private void handleJobManagerRunnerError(Throwable cause) { + if (ExceptionUtils.isJvmFatalError(cause)) { + fatalErrorHandler.onFatalError(cause); + } else { + resultFuture.completeExceptionally(cause); + } + } + + /** + * Marks this runner's job as not running. Other JobManager will not recover the job + * after this call. + * + *

This method never throws an exception. + */ + private void unregisterJobFromHighAvailability() { + try { + runningJobsRegistry.setJobFinished(jobGraph.getJobID()); + } + catch (Throwable t) { + log.error("Could not un-register from high-availability services job {} ({})." + + "Other JobManager's may attempt to recover it and re-execute it.", + jobGraph.getName(), jobGraph.getJobID(), t); + } + } + + //---------------------------------------------------------------------------------------------- + // Leadership methods + //---------------------------------------------------------------------------------------------- + + @Override + public void grantLeadership(final UUID leaderSessionID) { + synchronized (lock) { + if (shutdown) { + log.info("JobManagerRunner already shutdown."); + return; + } + + leadershipOperation = leadershipOperation.thenCompose( + (ignored) -> { + synchronized (lock) { + return verifyJobSchedulingStatusAndStartJobManager(leaderSessionID); + } + }); + + handleException(leadershipOperation, "Could not start the job manager."); + } + } + + private CompletableFuture verifyJobSchedulingStatusAndStartJobManager(UUID leaderSessionId) { + final CompletableFuture jobSchedulingStatusFuture = getJobSchedulingStatus(); + + return jobSchedulingStatusFuture.thenCompose( + jobSchedulingStatus -> { + if (jobSchedulingStatus == JobSchedulingStatus.DONE) { + return jobAlreadyDone(); + } else { + return startJobMaster(leaderSessionId); + } + }); + } + + private CompletionStage startJobMaster(UUID leaderSessionId) { + log.info("JobManager runner for job {} ({}) was granted leadership with session id {} at {}.", + jobGraph.getName(), jobGraph.getJobID(), leaderSessionId, jobMasterService.getAddress()); + + try { + runningJobsRegistry.setJobRunning(jobGraph.getJobID()); + } catch (IOException e) { + return FutureUtils.completedExceptionally( + new FlinkException( + String.format("Failed to set the job %s to running in the running jobs registry.", jobGraph.getJobID()), + e)); + } + + final CompletableFuture startFuture; + try { + startFuture = jobMasterService.start(new JobMasterId(leaderSessionId)); + } catch (Exception e) { + return FutureUtils.completedExceptionally(new FlinkException("Failed to start the JobMaster.", e)); + } + + final CompletableFuture currentLeaderGatewayFuture = leaderGatewayFuture; + return startFuture.thenAcceptAsync( + (Acknowledge ack) -> confirmLeaderSessionIdIfStillLeader( + leaderSessionId, + jobMasterService.getAddress(), + currentLeaderGatewayFuture), + executor); + } + + @Nonnull + private CompletionStage jobAlreadyDone() { + log.info("Granted leader ship but job {} has been finished. ", jobGraph.getJobID()); + jobFinishedByOther(); + return CompletableFuture.completedFuture(null); + } + + private CompletableFuture getJobSchedulingStatus() { + try { + JobSchedulingStatus jobSchedulingStatus = runningJobsRegistry.getJobSchedulingStatus(jobGraph.getJobID()); + return CompletableFuture.completedFuture(jobSchedulingStatus); + } catch (IOException e) { + return FutureUtils.completedExceptionally( + new FlinkException( + String.format("Could not retrieve the job scheduling status for job %s.", jobGraph.getJobID()), + e)); + } + } + + private void confirmLeaderSessionIdIfStillLeader( + UUID leaderSessionId, + String leaderAddress, + CompletableFuture currentLeaderGatewayFuture) { + + if (leaderElectionService.hasLeadership(leaderSessionId)) { + currentLeaderGatewayFuture.complete(jobMasterService.getGateway()); + leaderElectionService.confirmLeadership(leaderSessionId, leaderAddress); + } else { + log.debug("Ignoring confirmation of leader session id because {} is no longer the leader.", getDescription()); + } + } + + @Override + public void revokeLeadership() { + synchronized (lock) { + if (shutdown) { + log.info("JobManagerRunner already shutdown."); + return; + } + + leadershipOperation = leadershipOperation.thenCompose( + (ignored) -> { + synchronized (lock) { + return revokeJobMasterLeadership(); + } + }); + + handleException(leadershipOperation, "Could not suspend the job manager."); + } + } + + private CompletableFuture revokeJobMasterLeadership() { + log.info("JobManager for job {} ({}) at {} was revoked leadership.", + jobGraph.getName(), jobGraph.getJobID(), jobMasterService.getAddress()); + + setNewLeaderGatewayFuture(); + + return jobMasterService + .suspend(new FlinkException("JobManager is no longer the leader.")) + .thenApply(FunctionUtils.nullFn()); + } + + private void handleException(CompletableFuture leadershipOperation, String message) { + leadershipOperation.whenComplete( + (ignored, throwable) -> { + if (throwable != null) { + handleJobManagerRunnerError(new FlinkException(message, throwable)); + } + }); + } + + private void setNewLeaderGatewayFuture() { + final CompletableFuture oldLeaderGatewayFuture = leaderGatewayFuture; + + leaderGatewayFuture = new CompletableFuture<>(); + + if (!oldLeaderGatewayFuture.isDone()) { + leaderGatewayFuture.whenComplete( + (JobMasterGateway jobMasterGateway, Throwable throwable) -> { + if (throwable != null) { + oldLeaderGatewayFuture.completeExceptionally(throwable); + } else { + oldLeaderGatewayFuture.complete(jobMasterGateway); + } + }); + } + } + + @Override + public String getDescription() { + return jobMasterService.getAddress(); + } + + @Override + public void handleError(Exception exception) { + log.error("Leader Election Service encountered a fatal error.", exception); + handleJobManagerRunnerError(exception); + } + + //---------------------------------------------------------------------------------------------- + // Testing + //---------------------------------------------------------------------------------------------- + + @VisibleForTesting + boolean isShutdown() { + return shutdown; + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java index 6c6696fed329edf3be020b7745fbc7a713599def..72e870cfea38f43b8935dc32297ef03065dfacf5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobManagerSharedServices.java @@ -39,12 +39,11 @@ import org.apache.flink.util.ExceptionUtils; import javax.annotation.Nonnull; +import java.time.Duration; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import scala.concurrent.duration.FiniteDuration; - import static org.apache.flink.util.Preconditions.checkNotNull; /** @@ -152,7 +151,7 @@ public class JobManagerSharedServices { FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder), alwaysParentFirstLoaderPatterns); - final FiniteDuration timeout; + final Duration timeout; try { timeout = AkkaUtils.getTimeout(config); } catch (NumberFormatException e) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java index ce949d8ae3d8dc0309fddcd78189cc374e350e24..665c4aa479d08a98efeed8e1c6b3fee49a99cb38 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/jobmaster/JobMaster.java @@ -38,6 +38,7 @@ import org.apache.flink.runtime.heartbeat.HeartbeatListener; import org.apache.flink.runtime.heartbeat.HeartbeatManager; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.heartbeat.HeartbeatTarget; +import org.apache.flink.runtime.heartbeat.NoOpHeartbeatManager; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.io.network.partition.PartitionTracker; import org.apache.flink.runtime.io.network.partition.PartitionTrackerFactory; @@ -219,7 +220,7 @@ public class JobMaster extends FencedRpcEndpoint implements JobMast ShuffleMaster shuffleMaster, PartitionTrackerFactory partitionTrackerFactory) throws Exception { - super(rpcService, AkkaRpcServiceUtils.createRandomName(JOB_MANAGER_NAME)); + super(rpcService, AkkaRpcServiceUtils.createRandomName(JOB_MANAGER_NAME), null); this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration); this.resourceId = checkNotNull(resourceId); @@ -269,6 +270,8 @@ public class JobMaster extends FencedRpcEndpoint implements JobMast this.establishedResourceManagerConnection = null; this.accumulators = new HashMap<>(); + this.taskManagerHeartbeatManager = NoOpHeartbeatManager.getInstance(); + this.resourceManagerHeartbeatManager = NoOpHeartbeatManager.getInstance(); } private SchedulerNG createScheduler(final JobManagerJobMetricGroup jobManagerJobMetricGroup) throws Exception { @@ -785,15 +788,8 @@ public class JobMaster extends FencedRpcEndpoint implements JobMast } private void stopHeartbeatServices() { - if (taskManagerHeartbeatManager != null) { - taskManagerHeartbeatManager.stop(); - taskManagerHeartbeatManager = null; - } - - if (resourceManagerHeartbeatManager != null) { - resourceManagerHeartbeatManager.stop(); - resourceManagerHeartbeatManager = null; - } + taskManagerHeartbeatManager.stop(); + resourceManagerHeartbeatManager.stop(); } private void startHeartbeatServices() { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderContender.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderContender.java index dcf0a4e51291bc2185fcc4f7336a49ed38824bfb..8b28618e931896cd080b2d8371bd260afcc476e7 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderContender.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderContender.java @@ -41,14 +41,6 @@ public interface LeaderContender { */ void revokeLeadership(); - /** - * Returns the address of the {@link LeaderContender} under which other instances can connect - * to it. - * - * @return Address of this contender. - */ - String getAddress(); - /** * Callback method which is called by {@link LeaderElectionService} in case of an error in the * service thread. @@ -56,4 +48,13 @@ public interface LeaderContender { * @param exception Caught exception */ void handleError(Exception exception); + + /** + * Returns the description of the {@link LeaderContender} for logging purposes. + * + * @return Description of this contender. + */ + default String getDescription() { + return "LeaderContender: " + getClass().getSimpleName(); + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderElectionService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderElectionService.java index 10f2f68efa8bdd7629270d3d2409662fcee186e1..52b6986dbbc2abc64dd12faccce78e5257222ffe 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderElectionService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/LeaderElectionService.java @@ -25,14 +25,14 @@ import java.util.UUID; /** * Interface for a service which allows to elect a leader among a group of contenders. * - * Prior to using this service, it has to be started calling the start method. The start method + *

Prior to using this service, it has to be started calling the start method. The start method * takes the contender as a parameter. If there are multiple contenders, then each contender has * to instantiate its own leader election service. * - * Once a contender has been granted leadership he has to confirm the received leader session ID - * by calling the method confirmLeaderSessionID. This will notify the leader election service, that - * the contender has received the new leader session ID and that it can now be published for - * leader retrieval services. + *

Once a contender has been granted leadership he has to confirm the received leader session ID + * by calling the method {@link #confirmLeadership(UUID, String)}. This will notify the leader election + * service, that the contender has accepted the leadership specified and that the leader session id as + * well as the leader address can now be published for leader retrieval services. */ public interface LeaderElectionService { @@ -51,16 +51,18 @@ public interface LeaderElectionService { void stop() throws Exception; /** - * Confirms that the new leader session ID has been successfully received by the new leader. - * This method is usually called by the newly appointed {@link LeaderContender}. + * Confirms that the {@link LeaderContender} has accepted the leadership identified by the + * given leader session id. It also publishes the leader address under which the leader is + * reachable. * - * The rational behind this method is to establish an order between setting the new leader - * session ID in the {@link LeaderContender} and publishing the new leader session ID to the - * leader retrieval services. + *

The rational behind this method is to establish an order between setting the new leader + * session ID in the {@link LeaderContender} and publishing the new leader session ID as well + * as the leader address to the leader retrieval services. * * @param leaderSessionID The new leader session ID + * @param leaderAddress The address of the new leader */ - void confirmLeaderSessionID(UUID leaderSessionID); + void confirmLeadership(UUID leaderSessionID, String leaderAddress); /** * Returns true if the {@link LeaderContender} with which the service has been started owns diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/StandaloneLeaderElectionService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/StandaloneLeaderElectionService.java index ec997a332b37dac0c950fc9e248fc2186f8e1a79..d0bb0728985341cc6a59f15d51bcde65b0b1b5c8 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/StandaloneLeaderElectionService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/StandaloneLeaderElectionService.java @@ -57,7 +57,7 @@ public class StandaloneLeaderElectionService implements LeaderElectionService { } @Override - public void confirmLeaderSessionID(UUID leaderSessionID) {} + public void confirmLeadership(UUID leaderSessionID, String leaderAddress) {} @Override public boolean hasLeadership(@Nonnull UUID leaderSessionId) { diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java index 87684c871d7f76f4bcf4768bbf24d26291b97c2e..382b33e2fbeb6b7f37e688a772f405cbeb47b08a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionService.java @@ -72,6 +72,8 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le private volatile UUID confirmedLeaderSessionID; + private volatile String confirmedLeaderAddress; + /** The leader contender which applies for leadership. */ private volatile LeaderContender leaderContender; @@ -100,6 +102,7 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le issuedLeaderSessionID = null; confirmedLeaderSessionID = null; + confirmedLeaderAddress = null; leaderContender = null; running = false; @@ -177,12 +180,12 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le } @Override - public void confirmLeaderSessionID(UUID leaderSessionID) { + public void confirmLeadership(UUID leaderSessionID, String leaderAddress) { if (LOG.isDebugEnabled()) { LOG.debug( "Confirm leader session ID {} for leader {}.", leaderSessionID, - leaderContender.getAddress()); + leaderAddress); } Preconditions.checkNotNull(leaderSessionID); @@ -192,8 +195,8 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le synchronized (lock) { if (running) { if (leaderSessionID.equals(this.issuedLeaderSessionID)) { - confirmedLeaderSessionID = leaderSessionID; - writeLeaderInformation(confirmedLeaderSessionID); + confirmLeaderInformation(leaderSessionID, leaderAddress); + writeLeaderInformation(); } } else { LOG.debug("Ignoring the leader session Id {} confirmation, since the " + @@ -206,6 +209,11 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le } } + private void confirmLeaderInformation(UUID leaderSessionID, String leaderAddress) { + confirmedLeaderSessionID = leaderSessionID; + confirmedLeaderAddress = leaderAddress; + } + @Override public boolean hasLeadership(@Nonnull UUID leaderSessionId) { return leaderLatch.hasLeadership() && leaderSessionId.equals(issuedLeaderSessionID); @@ -216,12 +224,12 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le synchronized (lock) { if (running) { issuedLeaderSessionID = UUID.randomUUID(); - confirmedLeaderSessionID = null; + clearConfirmedLeaderInformation(); if (LOG.isDebugEnabled()) { LOG.debug( "Grant leadership to contender {} with session ID {}.", - leaderContender.getAddress(), + leaderContender.getDescription(), issuedLeaderSessionID); } @@ -233,16 +241,23 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le } } + private void clearConfirmedLeaderInformation() { + confirmedLeaderSessionID = null; + confirmedLeaderAddress = null; + } + @Override public void notLeader() { synchronized (lock) { if (running) { - issuedLeaderSessionID = null; - confirmedLeaderSessionID = null; + LOG.debug( + "Revoke leadership of {} ({}@{}).", + leaderContender.getDescription(), + confirmedLeaderSessionID, + confirmedLeaderAddress); - if (LOG.isDebugEnabled()) { - LOG.debug("Revoke leadership of {}.", leaderContender.getAddress()); - } + issuedLeaderSessionID = null; + clearConfirmedLeaderInformation(); leaderContender.revokeLeadership(); } else { @@ -262,7 +277,7 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le if (LOG.isDebugEnabled()) { LOG.debug( "Leader node changed while {} is the leader with session ID {}.", - leaderContender.getAddress(), + leaderContender.getDescription(), confirmedLeaderSessionID); } @@ -273,9 +288,9 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le if (LOG.isDebugEnabled()) { LOG.debug( "Writing leader information into empty node by {}.", - leaderContender.getAddress()); + leaderContender.getDescription()); } - writeLeaderInformation(confirmedLeaderSessionID); + writeLeaderInformation(); } else { byte[] data = childData.getData(); @@ -284,9 +299,9 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le if (LOG.isDebugEnabled()) { LOG.debug( "Writing leader information into node with empty data field by {}.", - leaderContender.getAddress()); + leaderContender.getDescription()); } - writeLeaderInformation(confirmedLeaderSessionID); + writeLeaderInformation(); } else { ByteArrayInputStream bais = new ByteArrayInputStream(data); ObjectInputStream ois = new ObjectInputStream(bais); @@ -294,15 +309,15 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le String leaderAddress = ois.readUTF(); UUID leaderSessionID = (UUID) ois.readObject(); - if (!leaderAddress.equals(this.leaderContender.getAddress()) || + if (!leaderAddress.equals(confirmedLeaderAddress) || (leaderSessionID == null || !leaderSessionID.equals(confirmedLeaderSessionID))) { // the data field does not correspond to the expected leader information if (LOG.isDebugEnabled()) { LOG.debug( "Correcting leader information by {}.", - leaderContender.getAddress()); + leaderContender.getDescription()); } - writeLeaderInformation(confirmedLeaderSessionID); + writeLeaderInformation(); } } } @@ -320,24 +335,22 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le /** * Writes the current leader's address as well the given leader session ID to ZooKeeper. - * - * @param leaderSessionID Leader session ID which is written to ZooKeeper */ - protected void writeLeaderInformation(UUID leaderSessionID) { + protected void writeLeaderInformation() { // this method does not have to be synchronized because the curator framework client // is thread-safe try { if (LOG.isDebugEnabled()) { LOG.debug( "Write leader information: Leader={}, session ID={}.", - leaderContender.getAddress(), - leaderSessionID); + confirmedLeaderAddress, + confirmedLeaderSessionID); } ByteArrayOutputStream baos = new ByteArrayOutputStream(); ObjectOutputStream oos = new ObjectOutputStream(baos); - oos.writeUTF(leaderContender.getAddress()); - oos.writeObject(leaderSessionID); + oos.writeUTF(confirmedLeaderAddress); + oos.writeObject(confirmedLeaderSessionID); oos.close(); @@ -381,8 +394,8 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le if (LOG.isDebugEnabled()) { LOG.debug( "Successfully wrote leader information: Leader={}, session ID={}.", - leaderContender.getAddress(), - leaderSessionID); + confirmedLeaderAddress, + confirmedLeaderSessionID); } } catch (Exception e) { leaderContender.handleError( @@ -397,7 +410,7 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le LOG.debug("Connected to ZooKeeper quorum. Leader election can start."); break; case SUSPENDED: - LOG.warn("Connection to ZooKeeper suspended. The contender " + leaderContender.getAddress() + LOG.warn("Connection to ZooKeeper suspended. The contender " + leaderContender.getDescription() + " no longer participates in the leader election."); break; case RECONNECTED: @@ -405,7 +418,7 @@ public class ZooKeeperLeaderElectionService implements LeaderElectionService, Le break; case LOST: // Maybe we have to throw an exception here to terminate the JobManager - LOG.warn("Connection to ZooKeeper lost. The contender " + leaderContender.getAddress() + LOG.warn("Connection to ZooKeeper lost. The contender " + leaderContender.getDescription() + " no longer participates in the leader election."); break; } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlTimestampType.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractImitatingJobManagerMetricGroup.java similarity index 31% rename from flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlTimestampType.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractImitatingJobManagerMetricGroup.java index 09e2d08bbaf7f716a90e67bcb8876eab26be24b5..440a4ed9bdbe0cd01c725c18c326055194b16061 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlTimestampType.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/AbstractImitatingJobManagerMetricGroup.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,58 +16,47 @@ * limitations under the License. */ -package org.apache.flink.sql.parser.type; +package org.apache.flink.runtime.metrics.groups; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlWriter; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.flink.metrics.CharacterFilter; +import org.apache.flink.runtime.metrics.MetricRegistry; +import org.apache.flink.runtime.metrics.dump.QueryScopeInfo; +import org.apache.flink.runtime.metrics.scope.ScopeFormat; + +import java.util.Map; /** - * Parse type "TIMESTAMP WITHOUT TIME ZONE", "TIMESTAMP(3) WITHOUT TIME ZONE", - * "TIMESTAMP WITH LOCAL TIME ZONE", or "TIMESTAMP(3) WITH LOCAL TIME ZONE". + * Base class which imitates the reporting of the existing {@link JobManagerMetricGroup} in order + * to guarantee backwards compatibility of metrics which are factored out of the {@link JobManagerMetricGroup}. */ -public class SqlTimestampType extends SqlIdentifier implements ExtendedSqlType { - private final int precision; - private final boolean withLocalTimeZone; +class AbstractImitatingJobManagerMetricGroup extends AbstractMetricGroup> { + protected final String hostname; - public SqlTimestampType(SqlParserPos pos, int precision, boolean withLocalTimeZone) { - super(getTypeName(withLocalTimeZone), pos); - this.precision = precision; - this.withLocalTimeZone = withLocalTimeZone; + AbstractImitatingJobManagerMetricGroup(MetricRegistry registry, String hostname) { + super(registry, getScope(registry, hostname), null); + this.hostname = hostname; } - private static String getTypeName(boolean withLocalTimeZone) { - if (withLocalTimeZone) { - return SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE.name(); - } else { - return SqlTypeName.TIMESTAMP.name(); - } + private static String[] getScope(MetricRegistry registry, String hostname) { + // returning jobmanager scope in order to guarantee backwards compatibility + // this can be changed once we introduce a proper scope for the process metric group + return registry.getScopeFormats().getJobManagerFormat().formatScope(hostname); } - public SqlTypeName getSqlTypeName() { - if (withLocalTimeZone) { - return SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE; - } else { - return SqlTypeName.TIMESTAMP; - } + @Override + protected final String getGroupName(CharacterFilter filter) { + // returning jobmanager in order to guarantee backwards compatibility + // this can be changed once we introduce a proper group name for the process metric group + return "jobmanager"; } - public int getPrecision() { - return this.precision; + @Override + protected final void putVariables(Map variables) { + variables.put(ScopeFormat.SCOPE_HOST, hostname); } @Override - public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - writer.keyword(SqlTypeName.TIMESTAMP.name()); - if (this.precision >= 0) { - final SqlWriter.Frame frame = - writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "(", ")"); - writer.print(precision); - writer.endList(frame); - } - if (this.withLocalTimeZone) { - writer.keyword("WITH LOCAL TIME ZONE"); - } + protected final QueryScopeInfo createQueryServiceMetricInfo(CharacterFilter filter) { + return new QueryScopeInfo.JobManagerQueryScopeInfo(); } } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlStringType.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/ProcessMetricGroup.java similarity index 57% rename from flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlStringType.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/ProcessMetricGroup.java index a134b13abe6635d03bb0014300a831b2b400dd2d..223a6b5a9a19cdd5158dd229fea8283028aa8eef 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlStringType.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/ProcessMetricGroup.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,23 +16,20 @@ * limitations under the License. */ -package org.apache.flink.sql.parser.type; +package org.apache.flink.runtime.metrics.groups; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlWriter; -import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.flink.runtime.metrics.MetricRegistry; /** - * Parse type "STRING" which is a synonym of VARCHAR(INT_MAX). + * {@link AbstractImitatingJobManagerMetricGroup} implementation for process related metrics. */ -public class SqlStringType extends SqlIdentifier implements ExtendedSqlType { +public class ProcessMetricGroup extends AbstractImitatingJobManagerMetricGroup { - public SqlStringType(SqlParserPos pos) { - super("STRING", pos); + ProcessMetricGroup(MetricRegistry registry, String hostname) { + super(registry, hostname); } - @Override - public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - writer.keyword("STRING"); + public static ProcessMetricGroup create(MetricRegistry metricRegistry, String hostname) { + return new ProcessMetricGroup(metricRegistry, hostname); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncCollectionResult.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/ResourceManagerMetricGroup.java similarity index 53% rename from flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncCollectionResult.java rename to flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/ResourceManagerMetricGroup.java index 6226ae64d7ded34c7304dd65f4378bcfa611a149..28cc948cc3716681d3b064b0fd7370aef128d4a6 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncCollectionResult.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/ResourceManagerMetricGroup.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -16,29 +16,20 @@ * limitations under the License. */ -package org.apache.flink.streaming.api.operators.async.queue; +package org.apache.flink.runtime.metrics.groups; -import org.apache.flink.annotation.Internal; - -import java.util.Collection; +import org.apache.flink.runtime.metrics.MetricRegistry; +import org.apache.flink.runtime.resourcemanager.ResourceManager; /** - * {@link AsyncResult} sub class for asynchronous result collections. - * - * @param Type of the collection elements. + * Metric group which is used by the {@link ResourceManager} to register metrics. */ -@Internal -public interface AsyncCollectionResult extends AsyncResult { - - boolean hasTimestamp(); - - long getTimestamp(); +public class ResourceManagerMetricGroup extends AbstractImitatingJobManagerMetricGroup { + ResourceManagerMetricGroup(MetricRegistry registry, String hostname) { + super(registry, hostname); + } - /** - * Return the asynchronous result collection. - * - * @return the asynchronous result collection - * @throws Exception if the asynchronous result collection could not be completed - */ - Collection get() throws Exception; + public static ResourceManagerMetricGroup create(MetricRegistry metricRegistry, String hostname) { + return new ResourceManagerMetricGroup(metricRegistry, hostname); + } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskMetricGroup.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskMetricGroup.java index 39d98d82fc895bedc456280883ac680cb7e8db5a..b5f389be66b17aceb64a382cd724fbb38a92a66f 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskMetricGroup.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/TaskMetricGroup.java @@ -139,24 +139,19 @@ public class TaskMetricGroup extends ComponentMetricGroup METRICS_OPERATOR_NAME_MAX_LENGTH) { LOG.warn("The operator name {} exceeded the {} characters length limit and was truncated.", name, METRICS_OPERATOR_NAME_MAX_LENGTH); - name = name.substring(0, METRICS_OPERATOR_NAME_MAX_LENGTH); + metricName = name.substring(0, METRICS_OPERATOR_NAME_MAX_LENGTH); + } else { + metricName = name; } - OperatorMetricGroup operator = new OperatorMetricGroup(this.registry, this, operatorID, name); + // unique OperatorIDs only exist in streaming, so we have to rely on the name for batch operators - final String key = operatorID + name; + final String key = operatorID + metricName; synchronized (this) { - OperatorMetricGroup previous = operators.put(key, operator); - if (previous == null) { - // no operator group so far - return operator; - } else { - // already had an operator group. restore that one. - operators.put(key, previous); - return previous; - } + return operators.computeIfAbsent(key, operator -> new OperatorMetricGroup(this.registry, this, operatorID, metricName)); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/UnregisteredMetricGroups.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/UnregisteredMetricGroups.java index 8c635eb994da25c45d59c6b07c26b55790cbc608..b60da9236dfad249c84c3f7ebf0846a9a32a503a 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/UnregisteredMetricGroups.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/groups/UnregisteredMetricGroups.java @@ -33,6 +33,14 @@ public class UnregisteredMetricGroups { private UnregisteredMetricGroups() { } + public static ProcessMetricGroup createUnregisteredProcessMetricGroup() { + return new UnregisteredProcessMetricGroup(); + } + + public static ResourceManagerMetricGroup createUnregisteredResourceManagerMetricGroup() { + return new UnregisteredResourceManagerMetricGroup(); + } + public static JobManagerMetricGroup createUnregisteredJobManagerMetricGroup() { return new UnregisteredJobManagerMetricGroup(); } @@ -57,6 +65,28 @@ public class UnregisteredMetricGroups { return new UnregisteredOperatorMetricGroup(); } + /** + * A safe drop-in replacement for {@link ProcessMetricGroup ProcessMetricGroups}. + */ + public static class UnregisteredProcessMetricGroup extends ProcessMetricGroup { + private static final String UNREGISTERED_HOST = "UnregisteredHost"; + + public UnregisteredProcessMetricGroup() { + super(NoOpMetricRegistry.INSTANCE, UNREGISTERED_HOST); + } + } + + /** + * A safe drop-in replacement for {@link ResourceManagerMetricGroup ResourceManagerMetricGroups}. + */ + public static class UnregisteredResourceManagerMetricGroup extends ResourceManagerMetricGroup { + private static final String UNREGISTERED_HOST = "UnregisteredHost"; + + UnregisteredResourceManagerMetricGroup() { + super(NoOpMetricRegistry.INSTANCE, UNREGISTERED_HOST); + } + } + /** * A safe drop-in replacement for {@link JobManagerMetricGroup}s. */ diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/util/MetricUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/util/MetricUtils.java index 4ce8f5dd7d2998439b08b0e7cb9c5999bbb5c065..cda453d4699c5580198c9a873ca1573809efb420 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/util/MetricUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/metrics/util/MetricUtils.java @@ -29,7 +29,9 @@ import org.apache.flink.runtime.clusterframework.BootstrapTools; import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.metrics.MetricNames; import org.apache.flink.runtime.metrics.MetricRegistry; +import org.apache.flink.runtime.metrics.groups.AbstractMetricGroup; import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ProcessMetricGroup; import org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup; import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils; @@ -71,22 +73,26 @@ public class MetricUtils { private MetricUtils() { } - public static JobManagerMetricGroup instantiateJobManagerMetricGroup( + public static ProcessMetricGroup instantiateProcessMetricGroup( final MetricRegistry metricRegistry, final String hostname, final Optional

The tuple consists of: + *

    + *
  • {@link ExecutionVertexVersion} + *
  • {@link ExecutionVertexDeploymentOption} + *
  • {@link SlotExecutionVertexAssignment} + *
+ */ +class DeploymentHandle { + + private final ExecutionVertexVersion requiredVertexVersion; + + private final ExecutionVertexDeploymentOption executionVertexDeploymentOption; + + private final SlotExecutionVertexAssignment slotExecutionVertexAssignment; + + public DeploymentHandle( + final ExecutionVertexVersion requiredVertexVersion, + final ExecutionVertexDeploymentOption executionVertexDeploymentOption, + final SlotExecutionVertexAssignment slotExecutionVertexAssignment) { + + this.requiredVertexVersion = Preconditions.checkNotNull(requiredVertexVersion); + this.executionVertexDeploymentOption = Preconditions.checkNotNull(executionVertexDeploymentOption); + this.slotExecutionVertexAssignment = Preconditions.checkNotNull(slotExecutionVertexAssignment); + } + + public ExecutionVertexID getExecutionVertexId() { + return requiredVertexVersion.getExecutionVertexId(); + } + + public ExecutionVertexVersion getRequiredVertexVersion() { + return requiredVertexVersion; + } + + public DeploymentOption getDeploymentOption() { + return executionVertexDeploymentOption.getDeploymentOption(); + } + + public SlotExecutionVertexAssignment getSlotExecutionVertexAssignment() { + return slotExecutionVertexAssignment; + } + + public Optional getLogicalSlot() { + final CompletableFuture logicalSlotFuture = slotExecutionVertexAssignment.getLogicalSlotFuture(); + Preconditions.checkState(logicalSlotFuture.isDone(), "method can only be called after slot future is done"); + + if (logicalSlotFuture.isCompletedExceptionally() || logicalSlotFuture.isCancelled()) { + return Optional.empty(); + } + return Optional.ofNullable(logicalSlotFuture.getNow(null)); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionGraphToInputsLocationsRetrieverAdapter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionGraphToInputsLocationsRetrieverAdapter.java index 392f00e0309c748764ee7ad5ef18c6de10d6c013..96f75459d60f4e115946ef873491e6b62a717b79 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionGraphToInputsLocationsRetrieverAdapter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionGraphToInputsLocationsRetrieverAdapter.java @@ -57,7 +57,7 @@ public class ExecutionGraphToInputsLocationsRetrieverAdapter implements InputsLo List producers = new ArrayList<>(inputEdges.length); for (ExecutionEdge inputEdge : inputEdges) { ExecutionVertex producer = inputEdge.getSource().getProducer(); - producers.add(new ExecutionVertexID(producer.getJobvertexId(), producer.getParallelSubtaskIndex())); + producers.add(producer.getID()); } resultPartitionProducers.add(producers); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionSlotAllocatorFactory.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionSlotAllocatorFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..cd763572e81f5c0dd23fbad133bbb042b09e0cd6 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionSlotAllocatorFactory.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +/** + * Interface for {@link ExecutionSlotAllocator} factories. + */ +public interface ExecutionSlotAllocatorFactory { + + ExecutionSlotAllocator createInstance(InputsLocationsRetriever inputsLocationsRetriever); + +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionVertexOperations.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionVertexOperations.java new file mode 100644 index 0000000000000000000000000000000000000000..23cca7c7097075f532b88b4c678896b1ee84aa06 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionVertexOperations.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +import org.apache.flink.runtime.JobException; +import org.apache.flink.runtime.executiongraph.ExecutionVertex; + +import java.util.concurrent.CompletableFuture; + +/** + * Operations on the {@link ExecutionVertex}. + */ +interface ExecutionVertexOperations { + + void deploy(ExecutionVertex executionVertex) throws JobException; + + CompletableFuture cancel(ExecutionVertex executionVertex); +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionVertexSchedulingRequirementsMapper.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionVertexSchedulingRequirementsMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..b089e5fe1f2a21c22d792bcf5aec596d36ec01e0 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/ExecutionVertexSchedulingRequirementsMapper.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +import org.apache.flink.runtime.clusterframework.types.AllocationID; +import org.apache.flink.runtime.executiongraph.ExecutionVertex; +import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup; +import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID; +import org.apache.flink.runtime.taskmanager.TaskManagerLocation; + +import java.util.Collection; +import java.util.Collections; + +final class ExecutionVertexSchedulingRequirementsMapper { + + public static ExecutionVertexSchedulingRequirements from(final ExecutionVertex executionVertex) { + + final ExecutionVertexID executionVertexId = executionVertex.getID(); + + final AllocationID latestPriorAllocation = executionVertex.getLatestPriorAllocation(); + final SlotSharingGroup slotSharingGroup = executionVertex.getJobVertex().getSlotSharingGroup(); + + return new ExecutionVertexSchedulingRequirements.Builder() + .withExecutionVertexId(executionVertexId) + .withPreviousAllocationId(latestPriorAllocation) + .withResourceProfile(executionVertex.getResourceProfile()) + .withSlotSharingGroupId(slotSharingGroup == null ? null : slotSharingGroup.getSlotSharingGroupId()) + .withCoLocationConstraint(executionVertex.getLocationConstraint()) + .withPreferredLocations(getPreferredLocationBasedOnState(executionVertex)).build(); + } + + private static Collection getPreferredLocationBasedOnState(final ExecutionVertex executionVertex) { + return executionVertex + .getPreferredLocationBasedOnState() + .map(Collections::singleton) + .orElse(Collections.emptySet()); + } + + private ExecutionVertexSchedulingRequirementsMapper() { + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/InternalTaskFailuresListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/InternalTaskFailuresListener.java new file mode 100644 index 0000000000000000000000000000000000000000..6dbda3bfb106c361db209fb3883bcf03aef1a2bf --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/InternalTaskFailuresListener.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; +import org.apache.flink.runtime.executiongraph.ExecutionGraph; +import org.apache.flink.runtime.jobmaster.JobMasterGateway; +import org.apache.flink.runtime.taskmanager.TaskExecutionState; + +/** + * This interface enables subscribing to Task failures that are detected from the JobMaster side + * (e.g., from within the {@link ExecutionGraph}). + * In contrast, there are also failures that are detected by the TaskManager, which are communicated + * via {@link JobMasterGateway#updateTaskExecutionState(TaskExecutionState)}. + */ +public interface InternalTaskFailuresListener { + + void notifyFailed(ExecutionAttemptID attemptId, Throwable t); + +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/LegacyScheduler.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/LegacyScheduler.java index e33b6b677977b0328f9cd617932402fc5d173fb2..bc5c45091f73d8e779bb99cc375406cbefad61e7 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/LegacyScheduler.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/LegacyScheduler.java @@ -19,116 +19,30 @@ package org.apache.flink.runtime.scheduler; -import org.apache.flink.api.common.JobID; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.time.Time; -import org.apache.flink.configuration.CheckpointingOptions; import org.apache.flink.configuration.Configuration; -import org.apache.flink.core.io.InputSplit; -import org.apache.flink.queryablestate.KvStateID; -import org.apache.flink.runtime.JobException; -import org.apache.flink.runtime.accumulators.AccumulatorSnapshot; import org.apache.flink.runtime.blob.BlobWriter; -import org.apache.flink.runtime.checkpoint.CheckpointCoordinator; -import org.apache.flink.runtime.checkpoint.CheckpointMetrics; import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory; -import org.apache.flink.runtime.checkpoint.CompletedCheckpoint; -import org.apache.flink.runtime.checkpoint.TaskStateSnapshot; -import org.apache.flink.runtime.client.JobExecutionException; -import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor; -import org.apache.flink.runtime.concurrent.FutureUtils; -import org.apache.flink.runtime.execution.ExecutionState; -import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; -import org.apache.flink.runtime.executiongraph.Execution; -import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; import org.apache.flink.runtime.executiongraph.ExecutionGraph; -import org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder; -import org.apache.flink.runtime.executiongraph.ExecutionGraphException; -import org.apache.flink.runtime.executiongraph.ExecutionJobVertex; -import org.apache.flink.runtime.executiongraph.IntermediateResult; -import org.apache.flink.runtime.executiongraph.JobStatusListener; -import org.apache.flink.runtime.executiongraph.restart.RestartStrategy; import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory; -import org.apache.flink.runtime.executiongraph.restart.RestartStrategyResolving; import org.apache.flink.runtime.io.network.partition.PartitionTracker; -import org.apache.flink.runtime.io.network.partition.ResultPartitionID; -import org.apache.flink.runtime.jobgraph.IntermediateDataSetID; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.jobgraph.JobStatus; -import org.apache.flink.runtime.jobgraph.JobVertexID; -import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; -import org.apache.flink.runtime.jobmanager.PartitionProducerDisposedException; -import org.apache.flink.runtime.jobmaster.SerializedInputSplit; import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; -import org.apache.flink.runtime.messages.FlinkJobNotFoundException; -import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint; -import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint; -import org.apache.flink.runtime.messages.webmonitor.JobDetails; import org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup; -import org.apache.flink.runtime.query.KvStateLocation; -import org.apache.flink.runtime.query.KvStateLocationRegistry; -import org.apache.flink.runtime.query.UnknownKvStateLocation; import org.apache.flink.runtime.rest.handler.legacy.backpressure.BackPressureStatsTracker; -import org.apache.flink.runtime.rest.handler.legacy.backpressure.OperatorBackPressureStats; import org.apache.flink.runtime.shuffle.ShuffleMaster; -import org.apache.flink.runtime.state.KeyGroupRange; -import org.apache.flink.runtime.taskmanager.TaskExecutionState; -import org.apache.flink.runtime.taskmanager.TaskManagerLocation; -import org.apache.flink.runtime.webmonitor.WebMonitorUtils; -import org.apache.flink.util.FlinkException; -import org.apache.flink.util.InstantiationUtil; -import org.apache.flink.util.function.FunctionUtils; import org.slf4j.Logger; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.concurrent.Executor; import java.util.concurrent.ScheduledExecutorService; -import static org.apache.flink.util.Preconditions.checkNotNull; - /** * A scheduler that delegates to the scheduling logic in the {@link ExecutionGraph}. * * @see ExecutionGraph#scheduleForExecution() */ -public class LegacyScheduler implements SchedulerNG { - - private final Logger log; - - private final JobGraph jobGraph; - - private final ExecutionGraph executionGraph; - - private final BackPressureStatsTracker backPressureStatsTracker; - - private final Executor ioExecutor; - - private final Configuration jobMasterConfiguration; - - private final SlotProvider slotProvider; - - private final ScheduledExecutorService futureExecutor; - - private final ClassLoader userCodeLoader; - - private final CheckpointRecoveryFactory checkpointRecoveryFactory; - - private final Time rpcTimeout; - - private final RestartStrategy restartStrategy; - - private final BlobWriter blobWriter; - - private final Time slotRequestTimeout; - - private ComponentMainThreadExecutor mainThreadExecutor = new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor( - "LegacyScheduler is not initialized with proper main thread executor. " + - "Call to LegacyScheduler.setMainThreadExecutor(...) required."); +public class LegacyScheduler extends SchedulerBase { public LegacyScheduler( final Logger log, @@ -148,116 +62,28 @@ public class LegacyScheduler implements SchedulerNG { final ShuffleMaster shuffleMaster, final PartitionTracker partitionTracker) throws Exception { - this.log = checkNotNull(log); - this.jobGraph = checkNotNull(jobGraph); - this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker); - this.ioExecutor = checkNotNull(ioExecutor); - this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration); - this.slotProvider = checkNotNull(slotProvider); - this.futureExecutor = checkNotNull(futureExecutor); - this.userCodeLoader = checkNotNull(userCodeLoader); - this.checkpointRecoveryFactory = checkNotNull(checkpointRecoveryFactory); - this.rpcTimeout = checkNotNull(rpcTimeout); - - final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration = - jobGraph.getSerializedExecutionConfig() - .deserializeValue(userCodeLoader) - .getRestartStrategy(); - - this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration, - restartStrategyFactory, - jobGraph.isCheckpointingEnabled()); - - log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobGraph.getName(), jobGraph.getJobID()); - - this.blobWriter = checkNotNull(blobWriter); - this.slotRequestTimeout = checkNotNull(slotRequestTimeout); - - this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup, checkNotNull(shuffleMaster), checkNotNull(partitionTracker)); - } - - private ExecutionGraph createAndRestoreExecutionGraph( - JobManagerJobMetricGroup currentJobManagerJobMetricGroup, - ShuffleMaster shuffleMaster, - PartitionTracker partitionTracker) throws Exception { - - ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup, shuffleMaster, partitionTracker); - - final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator(); - - if (checkpointCoordinator != null) { - // check whether we find a valid checkpoint - if (!checkpointCoordinator.restoreLatestCheckpointedState( - newExecutionGraph.getAllVertices(), - false, - false)) { - - // check whether we can restore from a savepoint - tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings()); - } - } - - return newExecutionGraph; - } - - private ExecutionGraph createExecutionGraph( - JobManagerJobMetricGroup currentJobManagerJobMetricGroup, - ShuffleMaster shuffleMaster, - final PartitionTracker partitionTracker) throws JobExecutionException, JobException { - return ExecutionGraphBuilder.buildGraph( - null, + super( + log, jobGraph, - jobMasterConfiguration, - futureExecutor, + backPressureStatsTracker, ioExecutor, + jobMasterConfiguration, slotProvider, + futureExecutor, userCodeLoader, checkpointRecoveryFactory, rpcTimeout, - restartStrategy, - currentJobManagerJobMetricGroup, + restartStrategyFactory, blobWriter, + jobManagerJobMetricGroup, slotRequestTimeout, - log, shuffleMaster, partitionTracker); } - /** - * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}. - * - * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored - * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from - * @throws Exception if the {@link ExecutionGraph} could not be restored - */ - private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception { - if (savepointRestoreSettings.restoreSavepoint()) { - final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator(); - if (checkpointCoordinator != null) { - checkpointCoordinator.restoreSavepoint( - savepointRestoreSettings.getRestorePath(), - savepointRestoreSettings.allowNonRestoredState(), - executionGraphToRestore.getAllVertices(), - userCodeLoader); - } - } - } - @Override - public void setMainThreadExecutor(final ComponentMainThreadExecutor mainThreadExecutor) { - this.mainThreadExecutor = checkNotNull(mainThreadExecutor); - executionGraph.start(mainThreadExecutor); - } - - @Override - public void registerJobStatusListener(final JobStatusListener jobStatusListener) { - executionGraph.registerJobStatusListener(jobStatusListener); - } - - @Override - public void startScheduling() { - mainThreadExecutor.assertRunningInMainThread(); - + protected void startSchedulingInternal() { + final ExecutionGraph executionGraph = getExecutionGraph(); try { executionGraph.scheduleForExecution(); } @@ -265,381 +91,4 @@ public class LegacyScheduler implements SchedulerNG { executionGraph.failGlobal(t); } } - - @Override - public void suspend(Throwable cause) { - mainThreadExecutor.assertRunningInMainThread(); - executionGraph.suspend(cause); - } - - @Override - public void cancel() { - mainThreadExecutor.assertRunningInMainThread(); - executionGraph.cancel(); - } - - @Override - public CompletableFuture getTerminationFuture() { - return executionGraph.getTerminationFuture().thenApply(FunctionUtils.nullFn()); - } - - @Override - public boolean updateTaskExecutionState(final TaskExecutionState taskExecutionState) { - mainThreadExecutor.assertRunningInMainThread(); - return executionGraph.updateState(taskExecutionState); - } - - @Override - public SerializedInputSplit requestNextInputSplit(JobVertexID vertexID, ExecutionAttemptID executionAttempt) throws IOException { - mainThreadExecutor.assertRunningInMainThread(); - - final Execution execution = executionGraph.getRegisteredExecutions().get(executionAttempt); - if (execution == null) { - // can happen when JobManager had already unregistered this execution upon on task failure, - // but TaskManager get some delay to aware of that situation - if (log.isDebugEnabled()) { - log.debug("Can not find Execution for attempt {}.", executionAttempt); - } - // but we should TaskManager be aware of this - throw new IllegalArgumentException("Can not find Execution for attempt " + executionAttempt); - } - - final ExecutionJobVertex vertex = executionGraph.getJobVertex(vertexID); - if (vertex == null) { - throw new IllegalArgumentException("Cannot find execution vertex for vertex ID " + vertexID); - } - - if (vertex.getSplitAssigner() == null) { - throw new IllegalStateException("No InputSplitAssigner for vertex ID " + vertexID); - } - - final InputSplit nextInputSplit = execution.getNextInputSplit(); - - if (log.isDebugEnabled()) { - log.debug("Send next input split {}.", nextInputSplit); - } - - try { - final byte[] serializedInputSplit = InstantiationUtil.serializeObject(nextInputSplit); - return new SerializedInputSplit(serializedInputSplit); - } catch (Exception ex) { - IOException reason = new IOException("Could not serialize the next input split of class " + - nextInputSplit.getClass() + ".", ex); - vertex.fail(reason); - throw reason; - } - } - - @Override - public ExecutionState requestPartitionState( - final IntermediateDataSetID intermediateResultId, - final ResultPartitionID resultPartitionId) throws PartitionProducerDisposedException { - - mainThreadExecutor.assertRunningInMainThread(); - - final Execution execution = executionGraph.getRegisteredExecutions().get(resultPartitionId.getProducerId()); - if (execution != null) { - return execution.getState(); - } - else { - final IntermediateResult intermediateResult = - executionGraph.getAllIntermediateResults().get(intermediateResultId); - - if (intermediateResult != null) { - // Try to find the producing execution - Execution producerExecution = intermediateResult - .getPartitionById(resultPartitionId.getPartitionId()) - .getProducer() - .getCurrentExecutionAttempt(); - - if (producerExecution.getAttemptId().equals(resultPartitionId.getProducerId())) { - return producerExecution.getState(); - } else { - throw new PartitionProducerDisposedException(resultPartitionId); - } - } else { - throw new IllegalArgumentException("Intermediate data set with ID " - + intermediateResultId + " not found."); - } - } - } - - @Override - public void scheduleOrUpdateConsumers(final ResultPartitionID partitionID) { - mainThreadExecutor.assertRunningInMainThread(); - - try { - executionGraph.scheduleOrUpdateConsumers(partitionID); - } catch (ExecutionGraphException e) { - throw new RuntimeException(e); - } - } - - @Override - public ArchivedExecutionGraph requestJob() { - mainThreadExecutor.assertRunningInMainThread(); - return ArchivedExecutionGraph.createFrom(executionGraph); - } - - @Override - public JobStatus requestJobStatus() { - return executionGraph.getState(); - } - - @Override - public JobDetails requestJobDetails() { - mainThreadExecutor.assertRunningInMainThread(); - return WebMonitorUtils.createDetailsForJob(executionGraph); - } - - @Override - public KvStateLocation requestKvStateLocation(final JobID jobId, final String registrationName) throws UnknownKvStateLocation, FlinkJobNotFoundException { - mainThreadExecutor.assertRunningInMainThread(); - - // sanity check for the correct JobID - if (jobGraph.getJobID().equals(jobId)) { - if (log.isDebugEnabled()) { - log.debug("Lookup key-value state for job {} with registration " + - "name {}.", jobGraph.getJobID(), registrationName); - } - - final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry(); - final KvStateLocation location = registry.getKvStateLocation(registrationName); - if (location != null) { - return location; - } else { - throw new UnknownKvStateLocation(registrationName); - } - } else { - if (log.isDebugEnabled()) { - log.debug("Request of key-value state location for unknown job {} received.", jobId); - } - throw new FlinkJobNotFoundException(jobId); - } - } - - @Override - public void notifyKvStateRegistered(final JobID jobId, final JobVertexID jobVertexId, final KeyGroupRange keyGroupRange, final String registrationName, final KvStateID kvStateId, final InetSocketAddress kvStateServerAddress) throws FlinkJobNotFoundException { - mainThreadExecutor.assertRunningInMainThread(); - - if (jobGraph.getJobID().equals(jobId)) { - if (log.isDebugEnabled()) { - log.debug("Key value state registered for job {} under name {}.", - jobGraph.getJobID(), registrationName); - } - - try { - executionGraph.getKvStateLocationRegistry().notifyKvStateRegistered( - jobVertexId, keyGroupRange, registrationName, kvStateId, kvStateServerAddress); - } catch (Exception e) { - throw new RuntimeException(e); - } - } else { - throw new FlinkJobNotFoundException(jobId); - } - } - - @Override - public void notifyKvStateUnregistered(final JobID jobId, final JobVertexID jobVertexId, final KeyGroupRange keyGroupRange, final String registrationName) throws FlinkJobNotFoundException { - mainThreadExecutor.assertRunningInMainThread(); - - if (jobGraph.getJobID().equals(jobId)) { - if (log.isDebugEnabled()) { - log.debug("Key value state unregistered for job {} under name {}.", - jobGraph.getJobID(), registrationName); - } - - try { - executionGraph.getKvStateLocationRegistry().notifyKvStateUnregistered( - jobVertexId, keyGroupRange, registrationName); - } catch (Exception e) { - throw new RuntimeException(e); - } - } else { - throw new FlinkJobNotFoundException(jobId); - } - } - - @Override - public void updateAccumulators(final AccumulatorSnapshot accumulatorSnapshot) { - mainThreadExecutor.assertRunningInMainThread(); - - executionGraph.updateAccumulators(accumulatorSnapshot); - } - - @Override - public Optional requestOperatorBackPressureStats(final JobVertexID jobVertexId) throws FlinkException { - final ExecutionJobVertex jobVertex = executionGraph.getJobVertex(jobVertexId); - if (jobVertex == null) { - throw new FlinkException("JobVertexID not found " + - jobVertexId); - } - - return backPressureStatsTracker.getOperatorBackPressureStats(jobVertex); - } - - @Override - public CompletableFuture triggerSavepoint(final String targetDirectory, final boolean cancelJob) { - mainThreadExecutor.assertRunningInMainThread(); - - final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); - if (checkpointCoordinator == null) { - throw new IllegalStateException( - String.format("Job %s is not a streaming job.", jobGraph.getJobID())); - } else if (targetDirectory == null && !checkpointCoordinator.getCheckpointStorage().hasDefaultSavepointLocation()) { - log.info("Trying to cancel job {} with savepoint, but no savepoint directory configured.", jobGraph.getJobID()); - - throw new IllegalStateException( - "No savepoint directory configured. You can either specify a directory " + - "while cancelling via -s :targetDirectory or configure a cluster-wide " + - "default via key '" + CheckpointingOptions.SAVEPOINT_DIRECTORY.key() + "'."); - } - - if (cancelJob) { - checkpointCoordinator.stopCheckpointScheduler(); - } - - return checkpointCoordinator - .triggerSavepoint(System.currentTimeMillis(), targetDirectory) - .thenApply(CompletedCheckpoint::getExternalPointer) - .handleAsync((path, throwable) -> { - if (throwable != null) { - if (cancelJob) { - startCheckpointScheduler(checkpointCoordinator); - } - throw new CompletionException(throwable); - } else if (cancelJob) { - log.info("Savepoint stored in {}. Now cancelling {}.", path, jobGraph.getJobID()); - cancel(); - } - return path; - }, mainThreadExecutor); - } - - private void startCheckpointScheduler(final CheckpointCoordinator checkpointCoordinator) { - mainThreadExecutor.assertRunningInMainThread(); - - if (checkpointCoordinator.isPeriodicCheckpointingConfigured()) { - try { - checkpointCoordinator.startCheckpointScheduler(); - } catch (IllegalStateException ignored) { - // Concurrent shut down of the coordinator - } - } - } - - @Override - public void acknowledgeCheckpoint(final JobID jobID, final ExecutionAttemptID executionAttemptID, final long checkpointId, final CheckpointMetrics checkpointMetrics, final TaskStateSnapshot checkpointState) { - mainThreadExecutor.assertRunningInMainThread(); - - final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); - final AcknowledgeCheckpoint ackMessage = new AcknowledgeCheckpoint( - jobID, - executionAttemptID, - checkpointId, - checkpointMetrics, - checkpointState); - - final String taskManagerLocationInfo = retrieveTaskManagerLocation(executionAttemptID); - - if (checkpointCoordinator != null) { - ioExecutor.execute(() -> { - try { - checkpointCoordinator.receiveAcknowledgeMessage(ackMessage, taskManagerLocationInfo); - } catch (Throwable t) { - log.warn("Error while processing checkpoint acknowledgement message", t); - } - }); - } else { - String errorMessage = "Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator"; - if (executionGraph.getState() == JobStatus.RUNNING) { - log.error(errorMessage, jobGraph.getJobID()); - } else { - log.debug(errorMessage, jobGraph.getJobID()); - } - } - } - - @Override - public void declineCheckpoint(final DeclineCheckpoint decline) { - mainThreadExecutor.assertRunningInMainThread(); - - final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); - final String taskManagerLocationInfo = retrieveTaskManagerLocation(decline.getTaskExecutionId()); - - if (checkpointCoordinator != null) { - ioExecutor.execute(() -> { - try { - checkpointCoordinator.receiveDeclineMessage(decline, taskManagerLocationInfo); - } catch (Exception e) { - log.error("Error in CheckpointCoordinator while processing {}", decline, e); - } - }); - } else { - String errorMessage = "Received DeclineCheckpoint message for job {} with no CheckpointCoordinator"; - if (executionGraph.getState() == JobStatus.RUNNING) { - log.error(errorMessage, jobGraph.getJobID()); - } else { - log.debug(errorMessage, jobGraph.getJobID()); - } - } - } - - @Override - public CompletableFuture stopWithSavepoint(final String targetDirectory, final boolean advanceToEndOfEventTime) { - mainThreadExecutor.assertRunningInMainThread(); - - final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); - - if (checkpointCoordinator == null) { - return FutureUtils.completedExceptionally(new IllegalStateException( - String.format("Job %s is not a streaming job.", jobGraph.getJobID()))); - } - - if (targetDirectory == null && !checkpointCoordinator.getCheckpointStorage().hasDefaultSavepointLocation()) { - log.info("Trying to cancel job {} with savepoint, but no savepoint directory configured.", jobGraph.getJobID()); - - return FutureUtils.completedExceptionally(new IllegalStateException( - "No savepoint directory configured. You can either specify a directory " + - "while cancelling via -s :targetDirectory or configure a cluster-wide " + - "default via key '" + CheckpointingOptions.SAVEPOINT_DIRECTORY.key() + "'.")); - } - - // we stop the checkpoint coordinator so that we are guaranteed - // to have only the data of the synchronous savepoint committed. - // in case of failure, and if the job restarts, the coordinator - // will be restarted by the CheckpointCoordinatorDeActivator. - checkpointCoordinator.stopCheckpointScheduler(); - - final long now = System.currentTimeMillis(); - final CompletableFuture savepointFuture = checkpointCoordinator - .triggerSynchronousSavepoint(now, advanceToEndOfEventTime, targetDirectory) - .thenApply(CompletedCheckpoint::getExternalPointer); - - final CompletableFuture terminationFuture = executionGraph - .getTerminationFuture() - .handle((jobstatus, throwable) -> { - - if (throwable != null) { - log.info("Failed during stopping job {} with a savepoint. Reason: {}", jobGraph.getJobID(), throwable.getMessage()); - throw new CompletionException(throwable); - } else if (jobstatus != JobStatus.FINISHED) { - log.info("Failed during stopping job {} with a savepoint. Reason: Reached state {} instead of FINISHED.", jobGraph.getJobID(), jobstatus); - throw new CompletionException(new FlinkException("Reached state " + jobstatus + " instead of FINISHED.")); - } - return jobstatus; - }); - - return savepointFuture.thenCompose((path) -> - terminationFuture.thenApply((jobStatus -> path))); - } - - private String retrieveTaskManagerLocation(ExecutionAttemptID executionAttemptID) { - final Optional currentExecution = Optional.ofNullable(executionGraph.getRegisteredExecutions().get(executionAttemptID)); - - return currentExecution - .map(Execution::getAssignedResourceLocation) - .map(TaskManagerLocation::toString) - .orElse("Unknown location"); - } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java new file mode 100644 index 0000000000000000000000000000000000000000..1b4f62edcf1cae31459fe0b50c67ac6fb19afacf --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerBase.java @@ -0,0 +1,753 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.io.InputSplit; +import org.apache.flink.queryablestate.KvStateID; +import org.apache.flink.runtime.JobException; +import org.apache.flink.runtime.accumulators.AccumulatorSnapshot; +import org.apache.flink.runtime.blob.BlobWriter; +import org.apache.flink.runtime.checkpoint.CheckpointCoordinator; +import org.apache.flink.runtime.checkpoint.CheckpointMetrics; +import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory; +import org.apache.flink.runtime.checkpoint.CompletedCheckpoint; +import org.apache.flink.runtime.checkpoint.TaskStateSnapshot; +import org.apache.flink.runtime.client.JobExecutionException; +import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor; +import org.apache.flink.runtime.concurrent.FutureUtils; +import org.apache.flink.runtime.execution.ExecutionState; +import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; +import org.apache.flink.runtime.executiongraph.Execution; +import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; +import org.apache.flink.runtime.executiongraph.ExecutionGraph; +import org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder; +import org.apache.flink.runtime.executiongraph.ExecutionGraphException; +import org.apache.flink.runtime.executiongraph.ExecutionJobVertex; +import org.apache.flink.runtime.executiongraph.ExecutionVertex; +import org.apache.flink.runtime.executiongraph.IntermediateResult; +import org.apache.flink.runtime.executiongraph.JobStatusListener; +import org.apache.flink.runtime.executiongraph.failover.adapter.DefaultFailoverTopology; +import org.apache.flink.runtime.executiongraph.failover.flip1.FailoverTopology; +import org.apache.flink.runtime.executiongraph.restart.RestartStrategy; +import org.apache.flink.runtime.executiongraph.restart.RestartStrategyFactory; +import org.apache.flink.runtime.executiongraph.restart.RestartStrategyResolving; +import org.apache.flink.runtime.io.network.partition.PartitionTracker; +import org.apache.flink.runtime.io.network.partition.ResultPartitionID; +import org.apache.flink.runtime.jobgraph.IntermediateDataSetID; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; +import org.apache.flink.runtime.jobgraph.JobVertexID; +import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; +import org.apache.flink.runtime.jobmanager.PartitionProducerDisposedException; +import org.apache.flink.runtime.jobmaster.SerializedInputSplit; +import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; +import org.apache.flink.runtime.messages.FlinkJobNotFoundException; +import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint; +import org.apache.flink.runtime.messages.checkpoint.DeclineCheckpoint; +import org.apache.flink.runtime.messages.webmonitor.JobDetails; +import org.apache.flink.runtime.metrics.groups.JobManagerJobMetricGroup; +import org.apache.flink.runtime.query.KvStateLocation; +import org.apache.flink.runtime.query.KvStateLocationRegistry; +import org.apache.flink.runtime.query.UnknownKvStateLocation; +import org.apache.flink.runtime.rest.handler.legacy.backpressure.BackPressureStatsTracker; +import org.apache.flink.runtime.rest.handler.legacy.backpressure.OperatorBackPressureStats; +import org.apache.flink.runtime.scheduler.adapter.ExecutionGraphToSchedulingTopologyAdapter; +import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID; +import org.apache.flink.runtime.scheduler.strategy.SchedulingTopology; +import org.apache.flink.runtime.shuffle.ShuffleMaster; +import org.apache.flink.runtime.state.KeyGroupRange; +import org.apache.flink.runtime.taskmanager.TaskExecutionState; +import org.apache.flink.runtime.taskmanager.TaskManagerLocation; +import org.apache.flink.runtime.webmonitor.WebMonitorUtils; +import org.apache.flink.util.FlinkException; +import org.apache.flink.util.InstantiationUtil; +import org.apache.flink.util.function.FunctionUtils; + +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.Executor; +import java.util.concurrent.ScheduledExecutorService; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Base class which can be used to implement {@link SchedulerNG}. + */ +public abstract class SchedulerBase implements SchedulerNG { + + private final Logger log; + + private final JobGraph jobGraph; + + private final ExecutionGraph executionGraph; + + private final SchedulingTopology schedulingTopology; + + private final FailoverTopology failoverTopology; + + private final InputsLocationsRetriever inputsLocationsRetriever; + + private final BackPressureStatsTracker backPressureStatsTracker; + + private final Executor ioExecutor; + + private final Configuration jobMasterConfiguration; + + private final SlotProvider slotProvider; + + private final ScheduledExecutorService futureExecutor; + + private final ClassLoader userCodeLoader; + + private final CheckpointRecoveryFactory checkpointRecoveryFactory; + + private final Time rpcTimeout; + + private final RestartStrategy restartStrategy; + + private final BlobWriter blobWriter; + + private final Time slotRequestTimeout; + + + private ComponentMainThreadExecutor mainThreadExecutor = new ComponentMainThreadExecutor.DummyComponentMainThreadExecutor( + "SchedulerBase is not initialized with proper main thread executor. " + + "Call to SchedulerBase.setMainThreadExecutor(...) required."); + + public SchedulerBase( + final Logger log, + final JobGraph jobGraph, + final BackPressureStatsTracker backPressureStatsTracker, + final Executor ioExecutor, + final Configuration jobMasterConfiguration, + final SlotProvider slotProvider, + final ScheduledExecutorService futureExecutor, + final ClassLoader userCodeLoader, + final CheckpointRecoveryFactory checkpointRecoveryFactory, + final Time rpcTimeout, + final RestartStrategyFactory restartStrategyFactory, + final BlobWriter blobWriter, + final JobManagerJobMetricGroup jobManagerJobMetricGroup, + final Time slotRequestTimeout, + final ShuffleMaster shuffleMaster, + final PartitionTracker partitionTracker) throws Exception { + + this.log = checkNotNull(log); + this.jobGraph = checkNotNull(jobGraph); + this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker); + this.ioExecutor = checkNotNull(ioExecutor); + this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration); + this.slotProvider = checkNotNull(slotProvider); + this.futureExecutor = checkNotNull(futureExecutor); + this.userCodeLoader = checkNotNull(userCodeLoader); + this.checkpointRecoveryFactory = checkNotNull(checkpointRecoveryFactory); + this.rpcTimeout = checkNotNull(rpcTimeout); + + final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration = + jobGraph.getSerializedExecutionConfig() + .deserializeValue(userCodeLoader) + .getRestartStrategy(); + + this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration, + restartStrategyFactory, + jobGraph.isCheckpointingEnabled()); + + log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobGraph.getName(), jobGraph.getJobID()); + + this.blobWriter = checkNotNull(blobWriter); + this.slotRequestTimeout = checkNotNull(slotRequestTimeout); + + this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup, checkNotNull(shuffleMaster), checkNotNull(partitionTracker)); + this.schedulingTopology = new ExecutionGraphToSchedulingTopologyAdapter(executionGraph); + this.failoverTopology = new DefaultFailoverTopology(executionGraph); + this.inputsLocationsRetriever = new ExecutionGraphToInputsLocationsRetrieverAdapter(executionGraph); + } + + private ExecutionGraph createAndRestoreExecutionGraph( + JobManagerJobMetricGroup currentJobManagerJobMetricGroup, + ShuffleMaster shuffleMaster, + PartitionTracker partitionTracker) throws Exception { + + ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup, shuffleMaster, partitionTracker); + + final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator(); + + if (checkpointCoordinator != null) { + // check whether we find a valid checkpoint + if (!checkpointCoordinator.restoreLatestCheckpointedState( + newExecutionGraph.getAllVertices(), + false, + false)) { + + // check whether we can restore from a savepoint + tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings()); + } + } + + return newExecutionGraph; + } + + private ExecutionGraph createExecutionGraph( + JobManagerJobMetricGroup currentJobManagerJobMetricGroup, + ShuffleMaster shuffleMaster, + final PartitionTracker partitionTracker) throws JobExecutionException, JobException { + return ExecutionGraphBuilder.buildGraph( + null, + jobGraph, + jobMasterConfiguration, + futureExecutor, + ioExecutor, + slotProvider, + userCodeLoader, + checkpointRecoveryFactory, + rpcTimeout, + restartStrategy, + currentJobManagerJobMetricGroup, + blobWriter, + slotRequestTimeout, + log, + shuffleMaster, + partitionTracker); + } + + /** + * @deprecated Direct access to the execution graph by scheduler implementations is discouraged + * because currently the execution graph has various features and responsibilities that a + * scheduler should not be concerned about. The following specialized abstractions to the + * execution graph and accessors should be preferred over direct access: + *
    + *
  • {@link #getSchedulingTopology()} + *
  • {@link #getFailoverTopology()} + *
  • {@link #getInputsLocationsRetriever()} + *
  • {@link #getExecutionVertex(ExecutionVertexID)} + *
  • {@link #getExecutionVertexId(ExecutionAttemptID)} + *
  • {@link #getExecutionVertexIdOrThrow(ExecutionAttemptID)} + *
+ * Currently, only {@link LegacyScheduler} requires direct access to the execution graph. + */ + @Deprecated + protected ExecutionGraph getExecutionGraph() { + return executionGraph; + } + + /** + * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}. + * + * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored + * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from + * @throws Exception if the {@link ExecutionGraph} could not be restored + */ + private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception { + if (savepointRestoreSettings.restoreSavepoint()) { + final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator(); + if (checkpointCoordinator != null) { + checkpointCoordinator.restoreSavepoint( + savepointRestoreSettings.getRestorePath(), + savepointRestoreSettings.allowNonRestoredState(), + executionGraphToRestore.getAllVertices(), + userCodeLoader); + } + } + } + + protected void resetForNewExecutionIfInTerminalState(final Collection verticesToDeploy) { + verticesToDeploy.forEach(executionVertexId -> getExecutionVertex(executionVertexId) + .resetForNewExecutionIfInTerminalState()); + } + + protected void transitionToScheduled(final Collection verticesToDeploy) { + verticesToDeploy.forEach(executionVertexId -> getExecutionVertex(executionVertexId) + .getCurrentExecutionAttempt() + .transitionState(ExecutionState.SCHEDULED)); + } + + protected ComponentMainThreadExecutor getMainThreadExecutor() { + return mainThreadExecutor; + } + + protected void failJob(Throwable cause) { + executionGraph.failJob(cause); + } + + protected final FailoverTopology getFailoverTopology() { + return failoverTopology; + } + + protected final SchedulingTopology getSchedulingTopology() { + return schedulingTopology; + } + + protected final InputsLocationsRetriever getInputsLocationsRetriever() { + return inputsLocationsRetriever; + } + + protected final void prepareExecutionGraphForNgScheduling() { + executionGraph.enableNgScheduling(new UpdateSchedulerNgOnInternalTaskFailuresListener(this, jobGraph.getJobID())); + executionGraph.transitionToRunning(); + } + + protected Optional getExecutionVertexId(final ExecutionAttemptID executionAttemptId) { + return Optional.ofNullable(executionGraph.getRegisteredExecutions().get(executionAttemptId)) + .map(this::getExecutionVertexId); + } + + protected ExecutionVertexID getExecutionVertexIdOrThrow(final ExecutionAttemptID executionAttemptId) { + return getExecutionVertexId(executionAttemptId) + .orElseThrow(() -> new IllegalStateException("Cannot find execution " + executionAttemptId)); + } + + private ExecutionVertexID getExecutionVertexId(final Execution execution) { + return execution.getVertex().getID(); + } + + protected ExecutionVertex getExecutionVertex(final ExecutionVertexID executionVertexId) { + return executionGraph.getAllVertices().get(executionVertexId.getJobVertexId()).getTaskVertices()[executionVertexId.getSubtaskIndex()]; + } + + protected JobGraph getJobGraph() { + return jobGraph; + } + + // ------------------------------------------------------------------------ + // SchedulerNG + // ------------------------------------------------------------------------ + + @Override + public void setMainThreadExecutor(final ComponentMainThreadExecutor mainThreadExecutor) { + this.mainThreadExecutor = checkNotNull(mainThreadExecutor); + executionGraph.start(mainThreadExecutor); + } + + @Override + public void registerJobStatusListener(final JobStatusListener jobStatusListener) { + executionGraph.registerJobStatusListener(jobStatusListener); + } + + @Override + public final void startScheduling() { + mainThreadExecutor.assertRunningInMainThread(); + startSchedulingInternal(); + } + + protected abstract void startSchedulingInternal(); + + @Override + public void suspend(Throwable cause) { + mainThreadExecutor.assertRunningInMainThread(); + executionGraph.suspend(cause); + } + + @Override + public void cancel() { + mainThreadExecutor.assertRunningInMainThread(); + executionGraph.cancel(); + } + + @Override + public CompletableFuture getTerminationFuture() { + return executionGraph.getTerminationFuture().thenApply(FunctionUtils.nullFn()); + } + + @Override + public final boolean updateTaskExecutionState(final TaskExecutionState taskExecutionState) { + final Optional executionVertexId = getExecutionVertexId(taskExecutionState.getID()); + if (executionVertexId.isPresent()) { + executionGraph.updateState(taskExecutionState); + updateTaskExecutionStateInternal(executionVertexId.get(), taskExecutionState); + return true; + } + return false; + } + + protected void updateTaskExecutionStateInternal(final ExecutionVertexID executionVertexId, final TaskExecutionState taskExecutionState) { + } + + @Override + public SerializedInputSplit requestNextInputSplit(JobVertexID vertexID, ExecutionAttemptID executionAttempt) throws IOException { + mainThreadExecutor.assertRunningInMainThread(); + + final Execution execution = executionGraph.getRegisteredExecutions().get(executionAttempt); + if (execution == null) { + // can happen when JobManager had already unregistered this execution upon on task failure, + // but TaskManager get some delay to aware of that situation + if (log.isDebugEnabled()) { + log.debug("Can not find Execution for attempt {}.", executionAttempt); + } + // but we should TaskManager be aware of this + throw new IllegalArgumentException("Can not find Execution for attempt " + executionAttempt); + } + + final ExecutionJobVertex vertex = executionGraph.getJobVertex(vertexID); + if (vertex == null) { + throw new IllegalArgumentException("Cannot find execution vertex for vertex ID " + vertexID); + } + + if (vertex.getSplitAssigner() == null) { + throw new IllegalStateException("No InputSplitAssigner for vertex ID " + vertexID); + } + + final InputSplit nextInputSplit = execution.getNextInputSplit(); + + if (log.isDebugEnabled()) { + log.debug("Send next input split {}.", nextInputSplit); + } + + try { + final byte[] serializedInputSplit = InstantiationUtil.serializeObject(nextInputSplit); + return new SerializedInputSplit(serializedInputSplit); + } catch (Exception ex) { + IOException reason = new IOException("Could not serialize the next input split of class " + + nextInputSplit.getClass() + ".", ex); + vertex.fail(reason); + throw reason; + } + } + + @Override + public ExecutionState requestPartitionState( + final IntermediateDataSetID intermediateResultId, + final ResultPartitionID resultPartitionId) throws PartitionProducerDisposedException { + + mainThreadExecutor.assertRunningInMainThread(); + + final Execution execution = executionGraph.getRegisteredExecutions().get(resultPartitionId.getProducerId()); + if (execution != null) { + return execution.getState(); + } + else { + final IntermediateResult intermediateResult = + executionGraph.getAllIntermediateResults().get(intermediateResultId); + + if (intermediateResult != null) { + // Try to find the producing execution + Execution producerExecution = intermediateResult + .getPartitionById(resultPartitionId.getPartitionId()) + .getProducer() + .getCurrentExecutionAttempt(); + + if (producerExecution.getAttemptId().equals(resultPartitionId.getProducerId())) { + return producerExecution.getState(); + } else { + throw new PartitionProducerDisposedException(resultPartitionId); + } + } else { + throw new IllegalArgumentException("Intermediate data set with ID " + + intermediateResultId + " not found."); + } + } + } + + @Override + public final void scheduleOrUpdateConsumers(final ResultPartitionID partitionId) { + mainThreadExecutor.assertRunningInMainThread(); + + try { + executionGraph.scheduleOrUpdateConsumers(partitionId); + } catch (ExecutionGraphException e) { + throw new RuntimeException(e); + } + + final ExecutionVertexID producerVertexId = getExecutionVertexIdOrThrow(partitionId.getProducerId()); + scheduleOrUpdateConsumersInternal(producerVertexId, partitionId); + } + + protected void scheduleOrUpdateConsumersInternal(ExecutionVertexID producerVertexId, ResultPartitionID resultPartitionId) { + } + + @Override + public ArchivedExecutionGraph requestJob() { + mainThreadExecutor.assertRunningInMainThread(); + return ArchivedExecutionGraph.createFrom(executionGraph); + } + + @Override + public JobStatus requestJobStatus() { + return executionGraph.getState(); + } + + @Override + public JobDetails requestJobDetails() { + mainThreadExecutor.assertRunningInMainThread(); + return WebMonitorUtils.createDetailsForJob(executionGraph); + } + + @Override + public KvStateLocation requestKvStateLocation(final JobID jobId, final String registrationName) throws UnknownKvStateLocation, FlinkJobNotFoundException { + mainThreadExecutor.assertRunningInMainThread(); + + // sanity check for the correct JobID + if (jobGraph.getJobID().equals(jobId)) { + if (log.isDebugEnabled()) { + log.debug("Lookup key-value state for job {} with registration " + + "name {}.", jobGraph.getJobID(), registrationName); + } + + final KvStateLocationRegistry registry = executionGraph.getKvStateLocationRegistry(); + final KvStateLocation location = registry.getKvStateLocation(registrationName); + if (location != null) { + return location; + } else { + throw new UnknownKvStateLocation(registrationName); + } + } else { + if (log.isDebugEnabled()) { + log.debug("Request of key-value state location for unknown job {} received.", jobId); + } + throw new FlinkJobNotFoundException(jobId); + } + } + + @Override + public void notifyKvStateRegistered(final JobID jobId, final JobVertexID jobVertexId, final KeyGroupRange keyGroupRange, final String registrationName, final KvStateID kvStateId, final InetSocketAddress kvStateServerAddress) throws FlinkJobNotFoundException { + mainThreadExecutor.assertRunningInMainThread(); + + if (jobGraph.getJobID().equals(jobId)) { + if (log.isDebugEnabled()) { + log.debug("Key value state registered for job {} under name {}.", + jobGraph.getJobID(), registrationName); + } + + try { + executionGraph.getKvStateLocationRegistry().notifyKvStateRegistered( + jobVertexId, keyGroupRange, registrationName, kvStateId, kvStateServerAddress); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + throw new FlinkJobNotFoundException(jobId); + } + } + + @Override + public void notifyKvStateUnregistered(final JobID jobId, final JobVertexID jobVertexId, final KeyGroupRange keyGroupRange, final String registrationName) throws FlinkJobNotFoundException { + mainThreadExecutor.assertRunningInMainThread(); + + if (jobGraph.getJobID().equals(jobId)) { + if (log.isDebugEnabled()) { + log.debug("Key value state unregistered for job {} under name {}.", + jobGraph.getJobID(), registrationName); + } + + try { + executionGraph.getKvStateLocationRegistry().notifyKvStateUnregistered( + jobVertexId, keyGroupRange, registrationName); + } catch (Exception e) { + throw new RuntimeException(e); + } + } else { + throw new FlinkJobNotFoundException(jobId); + } + } + + @Override + public void updateAccumulators(final AccumulatorSnapshot accumulatorSnapshot) { + mainThreadExecutor.assertRunningInMainThread(); + + executionGraph.updateAccumulators(accumulatorSnapshot); + } + + @Override + public Optional requestOperatorBackPressureStats(final JobVertexID jobVertexId) throws FlinkException { + final ExecutionJobVertex jobVertex = executionGraph.getJobVertex(jobVertexId); + if (jobVertex == null) { + throw new FlinkException("JobVertexID not found " + + jobVertexId); + } + + return backPressureStatsTracker.getOperatorBackPressureStats(jobVertex); + } + + @Override + public CompletableFuture triggerSavepoint(final String targetDirectory, final boolean cancelJob) { + mainThreadExecutor.assertRunningInMainThread(); + + final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); + if (checkpointCoordinator == null) { + throw new IllegalStateException( + String.format("Job %s is not a streaming job.", jobGraph.getJobID())); + } else if (targetDirectory == null && !checkpointCoordinator.getCheckpointStorage().hasDefaultSavepointLocation()) { + log.info("Trying to cancel job {} with savepoint, but no savepoint directory configured.", jobGraph.getJobID()); + + throw new IllegalStateException( + "No savepoint directory configured. You can either specify a directory " + + "while cancelling via -s :targetDirectory or configure a cluster-wide " + + "default via key '" + CheckpointingOptions.SAVEPOINT_DIRECTORY.key() + "'."); + } + + if (cancelJob) { + checkpointCoordinator.stopCheckpointScheduler(); + } + + return checkpointCoordinator + .triggerSavepoint(System.currentTimeMillis(), targetDirectory) + .thenApply(CompletedCheckpoint::getExternalPointer) + .handleAsync((path, throwable) -> { + if (throwable != null) { + if (cancelJob) { + startCheckpointScheduler(checkpointCoordinator); + } + throw new CompletionException(throwable); + } else if (cancelJob) { + log.info("Savepoint stored in {}. Now cancelling {}.", path, jobGraph.getJobID()); + cancel(); + } + return path; + }, mainThreadExecutor); + } + + private void startCheckpointScheduler(final CheckpointCoordinator checkpointCoordinator) { + mainThreadExecutor.assertRunningInMainThread(); + + if (checkpointCoordinator.isPeriodicCheckpointingConfigured()) { + try { + checkpointCoordinator.startCheckpointScheduler(); + } catch (IllegalStateException ignored) { + // Concurrent shut down of the coordinator + } + } + } + + @Override + public void acknowledgeCheckpoint(final JobID jobID, final ExecutionAttemptID executionAttemptID, final long checkpointId, final CheckpointMetrics checkpointMetrics, final TaskStateSnapshot checkpointState) { + mainThreadExecutor.assertRunningInMainThread(); + + final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); + final AcknowledgeCheckpoint ackMessage = new AcknowledgeCheckpoint( + jobID, + executionAttemptID, + checkpointId, + checkpointMetrics, + checkpointState); + + final String taskManagerLocationInfo = retrieveTaskManagerLocation(executionAttemptID); + + if (checkpointCoordinator != null) { + ioExecutor.execute(() -> { + try { + checkpointCoordinator.receiveAcknowledgeMessage(ackMessage, taskManagerLocationInfo); + } catch (Throwable t) { + log.warn("Error while processing checkpoint acknowledgement message", t); + } + }); + } else { + String errorMessage = "Received AcknowledgeCheckpoint message for job {} with no CheckpointCoordinator"; + if (executionGraph.getState() == JobStatus.RUNNING) { + log.error(errorMessage, jobGraph.getJobID()); + } else { + log.debug(errorMessage, jobGraph.getJobID()); + } + } + } + + @Override + public void declineCheckpoint(final DeclineCheckpoint decline) { + mainThreadExecutor.assertRunningInMainThread(); + + final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); + final String taskManagerLocationInfo = retrieveTaskManagerLocation(decline.getTaskExecutionId()); + + if (checkpointCoordinator != null) { + ioExecutor.execute(() -> { + try { + checkpointCoordinator.receiveDeclineMessage(decline, taskManagerLocationInfo); + } catch (Exception e) { + log.error("Error in CheckpointCoordinator while processing {}", decline, e); + } + }); + } else { + String errorMessage = "Received DeclineCheckpoint message for job {} with no CheckpointCoordinator"; + if (executionGraph.getState() == JobStatus.RUNNING) { + log.error(errorMessage, jobGraph.getJobID()); + } else { + log.debug(errorMessage, jobGraph.getJobID()); + } + } + } + + @Override + public CompletableFuture stopWithSavepoint(final String targetDirectory, final boolean advanceToEndOfEventTime) { + mainThreadExecutor.assertRunningInMainThread(); + + final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator(); + + if (checkpointCoordinator == null) { + return FutureUtils.completedExceptionally(new IllegalStateException( + String.format("Job %s is not a streaming job.", jobGraph.getJobID()))); + } + + if (targetDirectory == null && !checkpointCoordinator.getCheckpointStorage().hasDefaultSavepointLocation()) { + log.info("Trying to cancel job {} with savepoint, but no savepoint directory configured.", jobGraph.getJobID()); + + return FutureUtils.completedExceptionally(new IllegalStateException( + "No savepoint directory configured. You can either specify a directory " + + "while cancelling via -s :targetDirectory or configure a cluster-wide " + + "default via key '" + CheckpointingOptions.SAVEPOINT_DIRECTORY.key() + "'.")); + } + + // we stop the checkpoint coordinator so that we are guaranteed + // to have only the data of the synchronous savepoint committed. + // in case of failure, and if the job restarts, the coordinator + // will be restarted by the CheckpointCoordinatorDeActivator. + checkpointCoordinator.stopCheckpointScheduler(); + + final long now = System.currentTimeMillis(); + final CompletableFuture savepointFuture = checkpointCoordinator + .triggerSynchronousSavepoint(now, advanceToEndOfEventTime, targetDirectory) + .thenApply(CompletedCheckpoint::getExternalPointer); + + final CompletableFuture terminationFuture = executionGraph + .getTerminationFuture() + .handle((jobstatus, throwable) -> { + + if (throwable != null) { + log.info("Failed during stopping job {} with a savepoint. Reason: {}", jobGraph.getJobID(), throwable.getMessage()); + throw new CompletionException(throwable); + } else if (jobstatus != JobStatus.FINISHED) { + log.info("Failed during stopping job {} with a savepoint. Reason: Reached state {} instead of FINISHED.", jobGraph.getJobID(), jobstatus); + throw new CompletionException(new FlinkException("Reached state " + jobstatus + " instead of FINISHED.")); + } + return jobstatus; + }); + + return savepointFuture.thenCompose((path) -> + terminationFuture.thenApply((jobStatus -> path))); + } + + private String retrieveTaskManagerLocation(ExecutionAttemptID executionAttemptID) { + final Optional currentExecution = Optional.ofNullable(executionGraph.getRegisteredExecutions().get(executionAttemptID)); + + return currentExecution + .map(Execution::getAssignedResourceLocation) + .map(TaskManagerLocation::toString) + .orElse("Unknown location"); + } + +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerOperations.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerOperations.java index 50d3f873232fe983fb511bb52d3edcfac4027d8e..3256a171e4eb1a60220016bfecb948fe6795a024 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerOperations.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SchedulerOperations.java @@ -29,6 +29,7 @@ public interface SchedulerOperations { /** * Allocate slots and deploy the vertex when slots are returned. + * Only vertices in CREATED state will be accepted. Errors will happen if scheduling Non-CREATED vertices. * * @param executionVertexDeploymentOptions The tasks to be deployed and deployment options */ diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/UpdateSchedulerNgOnInternalTaskFailuresListener.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/UpdateSchedulerNgOnInternalTaskFailuresListener.java new file mode 100644 index 0000000000000000000000000000000000000000..f37e4ecddb19d46f65e42cf75304788906f93bb0 --- /dev/null +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/UpdateSchedulerNgOnInternalTaskFailuresListener.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.runtime.execution.ExecutionState; +import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; +import org.apache.flink.runtime.taskmanager.TaskExecutionState; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Calls {@link SchedulerNG#updateTaskExecutionState(TaskExecutionState)} on task failure. + */ +class UpdateSchedulerNgOnInternalTaskFailuresListener implements InternalTaskFailuresListener { + + private final SchedulerNG schedulerNg; + + private final JobID jobId; + + public UpdateSchedulerNgOnInternalTaskFailuresListener( + final SchedulerNG schedulerNg, + final JobID jobId) { + + this.schedulerNg = checkNotNull(schedulerNg); + this.jobId = checkNotNull(jobId); + } + + @Override + public void notifyFailed(final ExecutionAttemptID attemptId, final Throwable t) { + schedulerNg.updateTaskExecutionState(new TaskExecutionState( + jobId, + attemptId, + ExecutionState.FAILED, + t)); + } +} diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adapter/ExecutionGraphToSchedulingTopologyAdapter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adapter/ExecutionGraphToSchedulingTopologyAdapter.java index 9b377d82b18eb5e6793199123996a5dc7862c1dd..94c835c5a578f0d2907111ea2bca7b8f9b412919 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adapter/ExecutionGraphToSchedulingTopologyAdapter.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/adapter/ExecutionGraphToSchedulingTopologyAdapter.java @@ -107,7 +107,7 @@ public class ExecutionGraphToSchedulingTopologyAdapter implements SchedulingTopo List producedPartitions) { DefaultSchedulingExecutionVertex schedulingVertex = new DefaultSchedulingExecutionVertex( - new ExecutionVertexID(vertex.getJobvertexId(), vertex.getParallelSubtaskIndex()), + vertex.getID(), producedPartitions, new ExecutionStateSupplier(vertex), vertex.getInputDependencyConstraint()); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/strategy/LazyFromSourcesSchedulingStrategy.java b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/strategy/LazyFromSourcesSchedulingStrategy.java index 55a77c8e632110c0aa7659e31d44de66089b5c40..bc4ebb65c54af9f494a70089e8c4ee59b389090c 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/strategy/LazyFromSourcesSchedulingStrategy.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/strategy/LazyFromSourcesSchedulingStrategy.java @@ -102,6 +102,7 @@ public class LazyFromSourcesSchedulingStrategy implements SchedulingStrategy { final Set verticesToSchedule = schedulingTopology.getVertexOrThrow(executionVertexId) .getProducedResultPartitions() .stream() + .filter(partition -> partition.getPartitionType().isBlocking()) .flatMap(partition -> inputConstraintChecker.markSchedulingResultPartitionFinished(partition).stream()) .flatMap(partition -> partition.getConsumers().stream()) .collect(Collectors.toSet()); diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java index b1238ec8e8da70b2bff6004d0f8d9d47a210eab5..f0db4cd82b2d371a72c1a12cbf6f650cfbc876e3 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskExecutor.java @@ -157,8 +157,6 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway { /** The task manager configuration. */ private final TaskManagerConfiguration taskManagerConfiguration; - private final HeartbeatServices heartbeatServices; - /** The fatal error handler to use in case of a fatal error. */ private final FatalErrorHandler fatalErrorHandler; @@ -207,10 +205,10 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway { private FileCache fileCache; /** The heartbeat manager for job manager in the task manager. */ - private HeartbeatManager jobManagerHeartbeatManager; + private final HeartbeatManager jobManagerHeartbeatManager; /** The heartbeat manager for resource manager in the task manager. */ - private HeartbeatManager resourceManagerHeartbeatManager; + private final HeartbeatManager resourceManagerHeartbeatManager; private final PartitionTable partitionTable; @@ -249,7 +247,6 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway { checkArgument(taskManagerConfiguration.getNumberSlots() > 0, "The number of slots has to be larger than 0."); this.taskManagerConfiguration = checkNotNull(taskManagerConfiguration); - this.heartbeatServices = checkNotNull(heartbeatServices); this.taskExecutorServices = checkNotNull(taskExecutorServices); this.haServices = checkNotNull(haServices); this.fatalErrorHandler = checkNotNull(fatalErrorHandler); @@ -278,6 +275,26 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway { this.stackTraceSampleService = new StackTraceSampleService(rpcService.getScheduledExecutor()); this.taskCompletionTracker = new TaskCompletionTracker(); + + final ResourceID resourceId = taskExecutorServices.getTaskManagerLocation().getResourceID(); + this.jobManagerHeartbeatManager = createJobManagerHeartbeatManager(heartbeatServices, resourceId); + this.resourceManagerHeartbeatManager = createResourceManagerHeartbeatManager(heartbeatServices, resourceId); + } + + private HeartbeatManager createResourceManagerHeartbeatManager(HeartbeatServices heartbeatServices, ResourceID resourceId) { + return heartbeatServices.createHeartbeatManager( + resourceId, + new ResourceManagerHeartbeatListener(), + getMainThreadExecutor(), + log); + } + + private HeartbeatManager createJobManagerHeartbeatManager(HeartbeatServices heartbeatServices, ResourceID resourceId) { + return heartbeatServices.createHeartbeatManager( + resourceId, + new JobManagerHeartbeatListener(), + getMainThreadExecutor(), + log); } @Override @@ -304,8 +321,6 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway { private void startTaskExecutorServices() throws Exception { try { - startHeartbeatServices(); - // start by connecting to the ResourceManager resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener()); @@ -412,38 +427,9 @@ public class TaskExecutor extends RpcEndpoint implements TaskExecutorGateway { // it will call close() recursively from the parent to children taskManagerMetricGroup.close(); - stopHeartbeatServices(); - ExceptionUtils.tryRethrowException(exception); } - private void startHeartbeatServices() { - final ResourceID resourceId = taskExecutorServices.getTaskManagerLocation().getResourceID(); - jobManagerHeartbeatManager = heartbeatServices.createHeartbeatManager( - resourceId, - new JobManagerHeartbeatListener(), - getMainThreadExecutor(), - log); - - resourceManagerHeartbeatManager = heartbeatServices.createHeartbeatManager( - resourceId, - new ResourceManagerHeartbeatListener(), - getMainThreadExecutor(), - log); - } - - private void stopHeartbeatServices() { - if (jobManagerHeartbeatManager != null) { - jobManagerHeartbeatManager.stop(); - jobManagerHeartbeatManager = null; - } - - if (resourceManagerHeartbeatManager != null) { - resourceManagerHeartbeatManager.stop(); - resourceManagerHeartbeatManager = null; - } - } - // ====================================================================== // RPC methods // ====================================================================== diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java index b73d4971b1e2330ae72f26092dbae6a7627bba6d..af71bbdee4146c2b349a4b9f9229714a4ad9526b 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerConfiguration.java @@ -31,13 +31,14 @@ import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders import org.apache.flink.runtime.registration.RetryingRegistrationConfiguration; import org.apache.flink.runtime.taskmanager.TaskManagerRuntimeInfo; import org.apache.flink.util.Preconditions; +import org.apache.flink.util.TimeUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.annotation.Nullable; -import scala.concurrent.duration.Duration; +import java.time.Duration; /** * Configuration object for {@link TaskExecutor}. @@ -185,9 +186,8 @@ public class TaskManagerConfiguration implements TaskManagerRuntimeInfo { final String[] tmpDirPaths = ConfigurationUtils.parseTempDirectories(configuration); final Time timeout; - try { - timeout = Time.milliseconds(AkkaUtils.getTimeout(configuration).toMillis()); + timeout = AkkaUtils.getTimeoutAsTime(configuration); } catch (Exception e) { throw new IllegalArgumentException( "Invalid format for '" + AkkaOptions.ASK_TIMEOUT.key() + @@ -196,56 +196,39 @@ public class TaskManagerConfiguration implements TaskManagerRuntimeInfo { LOG.info("Messages have a max timeout of " + timeout); - final Time finiteRegistrationDuration; - + Time finiteRegistrationDuration; try { - Duration maxRegistrationDuration = Duration.create(configuration.getString(TaskManagerOptions.REGISTRATION_TIMEOUT)); - if (maxRegistrationDuration.isFinite()) { - finiteRegistrationDuration = Time.milliseconds(maxRegistrationDuration.toMillis()); - } else { - finiteRegistrationDuration = null; - } - } catch (NumberFormatException e) { - throw new IllegalArgumentException("Invalid format for parameter " + - TaskManagerOptions.REGISTRATION_TIMEOUT.key(), e); + Duration maxRegistrationDuration = TimeUtils.parseDuration(configuration.getString(TaskManagerOptions.REGISTRATION_TIMEOUT)); + finiteRegistrationDuration = Time.milliseconds(maxRegistrationDuration.toMillis()); + } catch (IllegalArgumentException e) { + LOG.warn("Invalid format for parameter {}. Set the timeout to be infinite.", + TaskManagerOptions.REGISTRATION_TIMEOUT.key()); + finiteRegistrationDuration = null; } final Time initialRegistrationPause; try { - Duration pause = Duration.create(configuration.getString(TaskManagerOptions.INITIAL_REGISTRATION_BACKOFF)); - if (pause.isFinite()) { - initialRegistrationPause = Time.milliseconds(pause.toMillis()); - } else { - throw new IllegalArgumentException("The initial registration pause must be finite: " + pause); - } - } catch (NumberFormatException e) { + Duration pause = TimeUtils.parseDuration(configuration.getString(TaskManagerOptions.INITIAL_REGISTRATION_BACKOFF)); + initialRegistrationPause = Time.milliseconds(pause.toMillis()); + } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Invalid format for parameter " + TaskManagerOptions.INITIAL_REGISTRATION_BACKOFF.key(), e); } final Time maxRegistrationPause; try { - Duration pause = Duration.create(configuration.getString( - TaskManagerOptions.REGISTRATION_MAX_BACKOFF)); - if (pause.isFinite()) { - maxRegistrationPause = Time.milliseconds(pause.toMillis()); - } else { - throw new IllegalArgumentException("The maximum registration pause must be finite: " + pause); - } - } catch (NumberFormatException e) { + Duration pause = TimeUtils.parseDuration(configuration.getString(TaskManagerOptions.REGISTRATION_MAX_BACKOFF)); + maxRegistrationPause = Time.milliseconds(pause.toMillis()); + } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Invalid format for parameter " + TaskManagerOptions.INITIAL_REGISTRATION_BACKOFF.key(), e); } final Time refusedRegistrationPause; try { - Duration pause = Duration.create(configuration.getString(TaskManagerOptions.REFUSED_REGISTRATION_BACKOFF)); - if (pause.isFinite()) { - refusedRegistrationPause = Time.milliseconds(pause.toMillis()); - } else { - throw new IllegalArgumentException("The refused registration pause must be finite: " + pause); - } - } catch (NumberFormatException e) { + Duration pause = TimeUtils.parseDuration(configuration.getString(TaskManagerOptions.REFUSED_REGISTRATION_BACKOFF)); + refusedRegistrationPause = Time.milliseconds(pause.toMillis()); + } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Invalid format for parameter " + TaskManagerOptions.INITIAL_REGISTRATION_BACKOFF.key(), e); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java index 72cc2c9b218ce0c35c8d6bd6f2d1d2f122fcf26e..6493decef99352770cf3c83adf14d0a7797bfbaf 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunner.java @@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory; import java.lang.reflect.UndeclaredThrowableException; import java.net.InetAddress; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.concurrent.Callable; @@ -426,7 +427,7 @@ public class TaskManagerRunner implements FatalErrorHandler, AutoCloseableAsync final Configuration configuration, final HighAvailabilityServices haServices) throws LeaderRetrievalException { - final Time lookupTimeout = Time.milliseconds(AkkaUtils.getLookupTimeout(configuration).toMillis()); + final Duration lookupTimeout = AkkaUtils.getLookupTimeout(configuration); final InetAddress taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress( haServices.getResourceManagerLeaderRetriever(), diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java index 3db847dd34c3d4490c4c447160e9b12786376d57..752deabed2c4043a3c8b0b0909db4f28b8fabee5 100755 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServices.java @@ -433,7 +433,7 @@ public class TaskManagerServices { final long managedMemorySize = getManagedMemoryFromHeapAndManaged(config, heapAndManagedMemory); ConfigurationParserUtils.checkConfigParameter(managedMemorySize < heapAndManagedMemory, managedMemorySize, - TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), + TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.key(), "Managed memory size too large for " + (networkReservedMemory >> 20) + " MB network buffer memory and a total of " + totalJavaMemorySizeMB + " MB JVM memory"); @@ -460,20 +460,20 @@ public class TaskManagerServices { * All values are in bytes. */ public static long getManagedMemoryFromHeapAndManaged(Configuration config, long heapAndManagedMemory) { - if (config.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) { + if (config.contains(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE)) { // take the configured absolute value - final String sizeValue = config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE); + final String sizeValue = config.getString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE); try { return MemorySize.parse(sizeValue, MEGA_BYTES).getBytes(); } catch (IllegalArgumentException e) { throw new IllegalConfigurationException( - "Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e); + "Could not read " + TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.key(), e); } } else { // calculate managed memory size via fraction - final float fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION); + final float fraction = config.getFloat(TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION); return (long) (fraction * heapAndManagedMemory); } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java index 64f5a30bcb71e08088f2cbdc679b193f28db7819..ea6032f389bd9494c10f5bdb7b5aed8feccd8cb5 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskexecutor/TaskManagerServicesConfiguration.java @@ -67,7 +67,7 @@ public class TaskManagerServicesConfiguration { /** * Managed memory (in megabytes). * - * @see TaskManagerOptions#MANAGED_MEMORY_SIZE + * @see TaskManagerOptions#LEGACY_MANAGED_MEMORY_SIZE */ private final long configuredMemory; @@ -201,7 +201,7 @@ public class TaskManagerServicesConfiguration { * * @return managed memory or a default value (currently -1) if not configured * - * @see TaskManagerOptions#MANAGED_MEMORY_SIZE + * @see TaskManagerOptions#LEGACY_MANAGED_MEMORY_SIZE */ long getConfiguredMemory() { return configuredMemory; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java index 5d2cfd95c4da03a495c2796893f1cdc219cd2e48..26ba5de036cba1d9ed374e45a1d666bf9859d298 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/InputGateWithMetrics.java @@ -67,12 +67,12 @@ public class InputGateWithMetrics extends InputGate { @Override public Optional getNext() throws IOException, InterruptedException { - return updateMetrics(inputGate.getNext()); + return inputGate.getNext().map(this::updateMetrics); } @Override public Optional pollNext() throws IOException, InterruptedException { - return updateMetrics(inputGate.pollNext()); + return inputGate.pollNext().map(this::updateMetrics); } @Override @@ -85,8 +85,8 @@ public class InputGateWithMetrics extends InputGate { inputGate.close(); } - private Optional updateMetrics(Optional bufferOrEvent) { - bufferOrEvent.ifPresent(b -> numBytesIn.inc(b.getSize())); + private BufferOrEvent updateMetrics(BufferOrEvent bufferOrEvent) { + numBytesIn.inc(bufferOrEvent.getSize()); return bufferOrEvent; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NettyShuffleEnvironmentConfiguration.java b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NettyShuffleEnvironmentConfiguration.java index 6d4ca693bdd9ed31e08ff044c660ac5421c001f6..a6eb90f6f65338db1b9fe250abd178621164c690 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NettyShuffleEnvironmentConfiguration.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/taskmanager/NettyShuffleEnvironmentConfiguration.java @@ -231,8 +231,8 @@ public class NettyShuffleEnvironmentConfiguration { * *

The following configuration parameters are involved: *

    - *
  • {@link TaskManagerOptions#MANAGED_MEMORY_SIZE},
    • - *
  • {@link TaskManagerOptions#MANAGED_MEMORY_FRACTION},
    • + *
  • {@link TaskManagerOptions#LEGACY_MANAGED_MEMORY_SIZE},
    • + *
  • {@link TaskManagerOptions#LEGACY_MANAGED_MEMORY_FRACTION},
    • *
  • {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
    • *
  • {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN},
    • *
  • {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX}, and
    • diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ConfigurationParserUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ConfigurationParserUtils.java index 948d4c83c15ea4aa390892951189680d2afee865..0d3788c09c7c7b985ab231c32d79358c6ce93959 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ConfigurationParserUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ConfigurationParserUtils.java @@ -43,21 +43,23 @@ public class ConfigurationParserUtils { */ public static long getManagedMemorySize(Configuration configuration) { long managedMemorySize; - String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue(); - if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) { + String managedMemorySizeDefaultVal = TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.defaultValue(); + if (!configuration.getString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) { try { managedMemorySize = MemorySize.parse( - configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes(); + configuration.getString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes(); } catch (IllegalArgumentException e) { - throw new IllegalConfigurationException("Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e); + throw new IllegalConfigurationException("Could not read " + TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE + + .key(), e); } } else { managedMemorySize = Long.valueOf(managedMemorySizeDefaultVal); } checkConfigParameter(configuration.getString( - TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) || managedMemorySize > 0, - managedMemorySize, TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), + TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.defaultValue()) || managedMemorySize > 0, + managedMemorySize, TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.key(), "MemoryManager needs at least one MB of memory. " + "If you leave this config parameter empty, the system automatically pick a fraction of the available memory."); @@ -71,10 +73,10 @@ public class ConfigurationParserUtils { * @return fraction of managed memory */ public static float getManagedMemoryFraction(Configuration configuration) { - float managedMemoryFraction = configuration.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION); + float managedMemoryFraction = configuration.getFloat(TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION); checkConfigParameter(managedMemoryFraction > 0.0f && managedMemoryFraction < 1.0f, managedMemoryFraction, - TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(), + TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.key(), "MemoryManager fraction of the free memory must be between 0.0 and 1.0"); return managedMemoryFraction; diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderConnectionInfo.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderConnectionInfo.java index 2c94c43826e289d7872c2bb1cd38cae3592809ed..c58691989d0a1dde88817320ab02aa7ac6ac1e64 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderConnectionInfo.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderConnectionInfo.java @@ -18,74 +18,34 @@ package org.apache.flink.runtime.util; -import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.util.FlinkException; - -import akka.actor.Address; - -import java.net.MalformedURLException; import java.util.UUID; /** * Wrapper class for a pair of connection address and leader session ID. */ public class LeaderConnectionInfo { + private final UUID leaderSessionId; private final String address; - private final UUID leaderSessionID; - - private final String hostname; - - private final int port; - - public LeaderConnectionInfo(String address, UUID leaderSessionID) throws FlinkException { + public LeaderConnectionInfo(UUID leaderSessionId, String address) { + this.leaderSessionId = leaderSessionId; this.address = address; - this.leaderSessionID = leaderSessionID; - - final Address akkaAddress; - // this only works as long as the address is Akka based - try { - akkaAddress = AkkaUtils.getAddressFromAkkaURL(address); - } catch (MalformedURLException e) { - throw new FlinkException("Could not extract the hostname from the given address \'" + - address + "\'.", e); - } - - if (akkaAddress.host().isDefined()) { - hostname = akkaAddress.host().get(); - } else { - hostname = "localhost"; - } + } - if (akkaAddress.port().isDefined()) { - port = (int) akkaAddress.port().get(); - } else { - port = -1; - } + public UUID getLeaderSessionId() { + return leaderSessionId; } public String getAddress() { return address; } - public UUID getLeaderSessionID() { - return leaderSessionID; - } - - public String getHostname() { - return hostname; - } - - public int getPort() { - return port; - } - @Override public String toString() { return "LeaderConnectionInfo{" + - "address='" + address + '\'' + - ", leaderSessionID=" + leaderSessionID + + "leaderSessionId=" + leaderSessionId + + ", address='" + address + '\'' + '}'; } } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java index e0dca79ff1d1cf2fc0b79756b692fddc0e7e529b..ad746a52b56474505c45d7407fa68aa618480622 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/LeaderRetrievalUtils.java @@ -18,24 +18,19 @@ package org.apache.flink.runtime.util; -import org.apache.flink.api.common.time.Time; -import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalException; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.runtime.net.ConnectionUtils; -import org.apache.flink.util.FlinkException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.net.InetAddress; +import java.time.Duration; import java.util.UUID; - -import scala.concurrent.Await; -import scala.concurrent.Future; -import scala.concurrent.Promise; -import scala.concurrent.duration.FiniteDuration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; /** * Utility class to work with {@link LeaderRetrievalService} class. @@ -57,36 +52,17 @@ public class LeaderRetrievalUtils { */ public static LeaderConnectionInfo retrieveLeaderConnectionInfo( LeaderRetrievalService leaderRetrievalService, - Time timeout) throws LeaderRetrievalException { - return retrieveLeaderConnectionInfo(leaderRetrievalService, FutureUtils.toFiniteDuration(timeout)); - } + Duration timeout) throws LeaderRetrievalException { - /** - * Retrieves the leader akka url and the current leader session ID. The values are stored in a - * {@link LeaderConnectionInfo} instance. - * - * @param leaderRetrievalService Leader retrieval service to retrieve the leader connection - * information - * @param timeout Timeout when to give up looking for the leader - * @return LeaderConnectionInfo containing the leader's akka URL and the current leader session - * ID - * @throws LeaderRetrievalException - */ - public static LeaderConnectionInfo retrieveLeaderConnectionInfo( - LeaderRetrievalService leaderRetrievalService, - FiniteDuration timeout - ) throws LeaderRetrievalException { LeaderConnectionInfoListener listener = new LeaderConnectionInfoListener(); try { leaderRetrievalService.start(listener); - Future connectionInfoFuture = listener.getLeaderConnectionInfoFuture(); - - return Await.result(connectionInfoFuture, timeout); + return listener.getLeaderConnectionInfoFuture().get(timeout.toMillis(), TimeUnit.MILLISECONDS); } catch (Exception e) { throw new LeaderRetrievalException("Could not retrieve the leader address and leader " + - "session ID.", e); + "session ID.", e); } finally { try { leaderRetrievalService.stop(); @@ -98,28 +74,23 @@ public class LeaderRetrievalUtils { public static InetAddress findConnectingAddress( LeaderRetrievalService leaderRetrievalService, - Time timeout) throws LeaderRetrievalException { - return findConnectingAddress(leaderRetrievalService, new FiniteDuration(timeout.getSize(), timeout.getUnit())); - } + Duration timeout) throws LeaderRetrievalException { - public static InetAddress findConnectingAddress( - LeaderRetrievalService leaderRetrievalService, - FiniteDuration timeout) throws LeaderRetrievalException { ConnectionUtils.LeaderConnectingAddressListener listener = new ConnectionUtils.LeaderConnectingAddressListener(); try { leaderRetrievalService.start(listener); LOG.info("Trying to select the network interface and address to use " + - "by connecting to the leading JobManager."); + "by connecting to the leading JobManager."); LOG.info("TaskManager will try to connect for " + timeout + - " before falling back to heuristics"); + " before falling back to heuristics"); return listener.findConnectingAddress(timeout); } catch (Exception e) { throw new LeaderRetrievalException("Could not find the connecting address by " + - "connecting to the current leader.", e); + "connecting to the current leader.", e); } finally { try { leaderRetrievalService.stop(); @@ -134,35 +105,28 @@ public class LeaderRetrievalUtils { * leader's akka URL and the current leader session ID. */ public static class LeaderConnectionInfoListener implements LeaderRetrievalListener { - private final Promise connectionInfo = new scala.concurrent.impl.Promise.DefaultPromise<>(); + private final CompletableFuture connectionInfoFuture = new CompletableFuture<>(); - public Future getLeaderConnectionInfoFuture() { - return connectionInfo.future(); + public CompletableFuture getLeaderConnectionInfoFuture() { + return connectionInfoFuture; } @Override public void notifyLeaderAddress(String leaderAddress, UUID leaderSessionID) { - if (leaderAddress != null && !leaderAddress.equals("") && !connectionInfo.isCompleted()) { - try { - final LeaderConnectionInfo leaderConnectionInfo = new LeaderConnectionInfo(leaderAddress, leaderSessionID); - connectionInfo.success(leaderConnectionInfo); - } catch (FlinkException e) { - connectionInfo.failure(e); - } - + if (leaderAddress != null && !leaderAddress.equals("") && !connectionInfoFuture.isDone()) { + final LeaderConnectionInfo leaderConnectionInfo = new LeaderConnectionInfo(leaderSessionID, leaderAddress); + connectionInfoFuture.complete(leaderConnectionInfo); } } @Override public void handleError(Exception exception) { - if (!connectionInfo.isCompleted()) { - connectionInfo.failure(exception); - } + connectionInfoFuture.completeExceptionally(exception); } } - + // ------------------------------------------------------------------------ - + /** * Private constructor to prevent instantiation. */ diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java index 084d021c9cc945a64198ac180783333e95651e28..f29f3351cdf69ea7927d09be381dbbae68f76de9 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/util/ZooKeeperUtils.java @@ -172,11 +172,10 @@ public class ZooKeeperUtils { * @param client The {@link CuratorFramework} ZooKeeper client to use * @param configuration {@link Configuration} object containing the configuration values * @return {@link ZooKeeperLeaderRetrievalService} instance. - * @throws Exception */ public static ZooKeeperLeaderRetrievalService createLeaderRetrievalService( final CuratorFramework client, - final Configuration configuration) throws Exception { + final Configuration configuration) { return createLeaderRetrievalService(client, configuration, ""); } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java index 1c31edb1efd7a3e579d9f628a5699d4ce3d228cb..6b1ff00e8c007309cbd1447d61f1eca6d0479f80 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorEndpoint.java @@ -710,7 +710,7 @@ public class WebMonitorEndpoint extends RestServerEndp @Override public void grantLeadership(final UUID leaderSessionID) { log.info("{} was granted leadership with leaderSessionID={}", getRestBaseUrl(), leaderSessionID); - leaderElectionService.confirmLeaderSessionID(leaderSessionID); + leaderElectionService.confirmLeadership(leaderSessionID, getRestBaseUrl()); } @Override @@ -719,7 +719,7 @@ public class WebMonitorEndpoint extends RestServerEndp } @Override - public String getAddress() { + public String getDescription() { return getRestBaseUrl(); } diff --git a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala index 531f04e7328d452f071d3e1f66baa464ac1d73f3..7070df46e7f8e908475f73fe98ce45e8ad166d41 100755 --- a/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala +++ b/flink-runtime/src/main/scala/org/apache/flink/runtime/akka/AkkaUtils.scala @@ -20,7 +20,8 @@ package org.apache.flink.runtime.akka import java.io.IOException import java.net._ -import java.util.concurrent.{Callable, CompletableFuture, TimeUnit} +import java.time +import java.util.concurrent.{Callable, CompletableFuture} import akka.actor._ import akka.pattern.{ask => akkaAsk} @@ -31,6 +32,7 @@ import org.apache.flink.runtime.clusterframework.BootstrapTools.{FixedThreadPool import org.apache.flink.runtime.concurrent.FutureUtils import org.apache.flink.runtime.net.SSLUtils import org.apache.flink.util.NetUtils +import org.apache.flink.util.TimeUtils import org.apache.flink.util.function.FunctionUtils import org.jboss.netty.channel.ChannelException import org.jboss.netty.logging.{InternalLoggerFactory, Slf4JLoggerFactory} @@ -49,8 +51,6 @@ import scala.language.postfixOps object AkkaUtils { val LOG: Logger = LoggerFactory.getLogger(AkkaUtils.getClass) - val INF_TIMEOUT: FiniteDuration = 21474835 seconds - val FLINK_ACTOR_SYSTEM_NAME = "flink" def getFlinkActorSystemName = { @@ -364,12 +364,12 @@ object AkkaUtils { } private def validateHeartbeat(pauseParamName: String, - pauseValue: String, + pauseValue: time.Duration, intervalParamName: String, - intervalValue: String): Unit = { - if (Duration.apply(pauseValue).lteq(Duration.apply(intervalValue))) { + intervalValue: time.Duration): Unit = { + if (pauseValue.compareTo(intervalValue) <= 0) { throw new IllegalConfigurationException( - "%s [%s] must greater then %s [%s]", + "%s [%s] must greater than %s [%s]", pauseParamName, pauseValue, intervalParamName, @@ -397,27 +397,34 @@ object AkkaUtils { val normalizedExternalHostname = NetUtils.unresolvedHostToNormalizedString(externalHostname) - val akkaAskTimeout = Duration(configuration.getString(AkkaOptions.ASK_TIMEOUT)) + val akkaAskTimeout = getTimeout(configuration) - val startupTimeout = configuration.getString( - AkkaOptions.STARTUP_TIMEOUT, - (akkaAskTimeout * 10).toString) + val startupTimeout = TimeUtils.getStringInMillis( + TimeUtils.parseDuration( + configuration.getString( + AkkaOptions.STARTUP_TIMEOUT, + TimeUtils.getStringInMillis(akkaAskTimeout.multipliedBy(10L))))) - val transportHeartbeatInterval = configuration.getString( - AkkaOptions.TRANSPORT_HEARTBEAT_INTERVAL) + val transportHeartbeatIntervalDuration = TimeUtils.parseDuration( + configuration.getString(AkkaOptions.TRANSPORT_HEARTBEAT_INTERVAL)) - val transportHeartbeatPause = configuration.getString( - AkkaOptions.TRANSPORT_HEARTBEAT_PAUSE) + val transportHeartbeatPauseDuration = TimeUtils.parseDuration( + configuration.getString(AkkaOptions.TRANSPORT_HEARTBEAT_PAUSE)) validateHeartbeat( AkkaOptions.TRANSPORT_HEARTBEAT_PAUSE.key(), - transportHeartbeatPause, + transportHeartbeatPauseDuration, AkkaOptions.TRANSPORT_HEARTBEAT_INTERVAL.key(), - transportHeartbeatInterval) + transportHeartbeatIntervalDuration) + + val transportHeartbeatInterval = TimeUtils.getStringInMillis(transportHeartbeatIntervalDuration) + + val transportHeartbeatPause = TimeUtils.getStringInMillis(transportHeartbeatPauseDuration) val transportThreshold = configuration.getDouble(AkkaOptions.TRANSPORT_THRESHOLD) - val akkaTCPTimeout = configuration.getString(AkkaOptions.TCP_TIMEOUT) + val akkaTCPTimeout = TimeUtils.getStringInMillis( + TimeUtils.parseDuration(configuration.getString(AkkaOptions.TCP_TIMEOUT))) val akkaFramesize = configuration.getString(AkkaOptions.FRAMESIZE) @@ -728,15 +735,13 @@ object AkkaUtils { } } - def getTimeout(config: Configuration): FiniteDuration = { - val duration = Duration(config.getString(AkkaOptions.ASK_TIMEOUT)) - - new FiniteDuration(duration.toMillis, TimeUnit.MILLISECONDS) + def getTimeout(config: Configuration): time.Duration = { + TimeUtils.parseDuration(config.getString(AkkaOptions.ASK_TIMEOUT)) } def getTimeoutAsTime(config: Configuration): Time = { try { - val duration = Duration(config.getString(AkkaOptions.ASK_TIMEOUT)) + val duration = getTimeout(config) Time.milliseconds(duration.toMillis) } catch { @@ -746,38 +751,17 @@ object AkkaUtils { } def getDefaultTimeout: Time = { - val duration = Duration(AkkaOptions.ASK_TIMEOUT.defaultValue()) + val duration = TimeUtils.parseDuration(AkkaOptions.ASK_TIMEOUT.defaultValue()) Time.milliseconds(duration.toMillis) } - def getDefaultTimeoutAsFiniteDuration: FiniteDuration = { - val timeout = getDefaultTimeout - - new FiniteDuration(timeout.toMilliseconds, TimeUnit.MILLISECONDS) - } - - def getLookupTimeout(config: Configuration): FiniteDuration = { - val duration = Duration(config.getString(AkkaOptions.LOOKUP_TIMEOUT)) - - new FiniteDuration(duration.toMillis, TimeUnit.MILLISECONDS) - } - - def getDefaultLookupTimeout: FiniteDuration = { - val duration = Duration(AkkaOptions.LOOKUP_TIMEOUT.defaultValue()) - new FiniteDuration(duration.toMillis, TimeUnit.MILLISECONDS) + def getLookupTimeout(config: Configuration): time.Duration = { + TimeUtils.parseDuration(config.getString(AkkaOptions.LOOKUP_TIMEOUT)) } - def getClientTimeout(config: Configuration): FiniteDuration = { - val duration = Duration(config.getString(AkkaOptions.CLIENT_TIMEOUT)) - - new FiniteDuration(duration.toMillis, TimeUnit.MILLISECONDS) - } - - def getDefaultClientTimeout: FiniteDuration = { - val duration = Duration(AkkaOptions.CLIENT_TIMEOUT.defaultValue()) - - new FiniteDuration(duration.toMillis, TimeUnit.MILLISECONDS) + def getClientTimeout(config: Configuration): time.Duration = { + TimeUtils.parseDuration(config.getString(AkkaOptions.CLIENT_TIMEOUT)) } /** Returns the address of the given [[ActorSystem]]. The [[Address]] object contains diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/FailoverStrategyCheckpointCoordinatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/FailoverStrategyCheckpointCoordinatorTest.java index 1d7771608ff791eddb7b8e0edfcf3324265e11eb..17a5bccead2a7ecdf3b3d2a86843355d20cd918d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/FailoverStrategyCheckpointCoordinatorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/FailoverStrategyCheckpointCoordinatorTest.java @@ -28,7 +28,6 @@ import org.apache.flink.runtime.executiongraph.Execution; import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; import org.apache.flink.runtime.executiongraph.ExecutionVertex; import org.apache.flink.runtime.executiongraph.failover.AdaptedRestartPipelinedRegionStrategyNG; -import org.apache.flink.runtime.executiongraph.failover.FailoverRegion; import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy; import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; import org.apache.flink.runtime.state.SharedStateRegistry; @@ -65,7 +64,7 @@ public class FailoverStrategyCheckpointCoordinatorTest extends TestLogger { /** * Tests that {@link CheckpointCoordinator#abortPendingCheckpoints(CheckpointException)} - * called by {@link AdaptedRestartPipelinedRegionStrategyNG} or {@link FailoverRegion} could handle + * called by {@link AdaptedRestartPipelinedRegionStrategyNG} could handle * the {@code currentPeriodicTrigger} null situation well. */ @Test diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java index 6f3c60b5fce18fbca41a55bcce74a518fefdd4bc..4ed1050253f0a75d32b1d27eba0af95bbb081a67 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/checkpoint/StandaloneCompletedCheckpointStoreTest.java @@ -18,8 +18,10 @@ package org.apache.flink.runtime.checkpoint; +import org.apache.flink.api.common.JobID; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.state.SharedStateRegistry; + import org.junit.Test; import java.io.IOException; @@ -27,6 +29,8 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -41,7 +45,7 @@ public class StandaloneCompletedCheckpointStoreTest extends CompletedCheckpointS @Override protected CompletedCheckpointStore createCompletedCheckpoints( - int maxNumberOfCheckpointsToRetain) throws Exception { + int maxNumberOfCheckpointsToRetain) throws Exception { return new StandaloneCompletedCheckpointStore(maxNumberOfCheckpointsToRetain); } @@ -86,7 +90,7 @@ public class StandaloneCompletedCheckpointStoreTest extends CompletedCheckpointS assertTrue(checkpoint.isDiscarded()); verifyCheckpointDiscarded(taskStates); } - + /** * Tests that the checkpoint does not exist in the store when we fail to add * it into the store (i.e., there exists an exception thrown by the method). @@ -96,16 +100,16 @@ public class StandaloneCompletedCheckpointStoreTest extends CompletedCheckpointS final int numCheckpointsToRetain = 1; CompletedCheckpointStore store = createCompletedCheckpoints(numCheckpointsToRetain); - + for (long i = 0; i <= numCheckpointsToRetain; ++i) { CompletedCheckpoint checkpointToAdd = mock(CompletedCheckpoint.class); doReturn(i).when(checkpointToAdd).getCheckpointID(); doReturn(Collections.emptyMap()).when(checkpointToAdd).getOperatorStates(); doThrow(new IOException()).when(checkpointToAdd).discardOnSubsume(); - + try { store.addCheckpoint(checkpointToAdd); - + // The checkpoint should be in the store if we successfully add it into the store. List addedCheckpoints = store.getAllCheckpoints(); assertTrue(addedCheckpoints.contains(checkpointToAdd)); @@ -116,4 +120,60 @@ public class StandaloneCompletedCheckpointStoreTest extends CompletedCheckpointS } } } + + @Test + public void testPreferCheckpointWithoutSavepoint() throws Exception { + StandaloneCompletedCheckpointStore store = new StandaloneCompletedCheckpointStore(5); + JobID jobId = new JobID(); + store.addCheckpoint(checkpoint(jobId, 1L)); + store.addCheckpoint(checkpoint(jobId, 2L)); + store.addCheckpoint(checkpoint(jobId, 3L)); + + CompletedCheckpoint latestCheckpoint = store.getLatestCheckpoint(true); + + assertThat(latestCheckpoint.getCheckpointID(), equalTo(3L)); + } + + @Test + public void testPreferCheckpointWithSavepoint() throws Exception { + StandaloneCompletedCheckpointStore store = new StandaloneCompletedCheckpointStore(5); + JobID jobId = new JobID(); + store.addCheckpoint(checkpoint(jobId, 1L)); + store.addCheckpoint(savepoint(jobId, 2L)); + store.addCheckpoint(savepoint(jobId, 3L)); + + CompletedCheckpoint latestCheckpoint = store.getLatestCheckpoint(true); + + assertThat(latestCheckpoint.getCheckpointID(), equalTo(1L)); + } + + @Test + public void testPreferCheckpointWithOnlySavepoint() throws Exception { + StandaloneCompletedCheckpointStore store = new StandaloneCompletedCheckpointStore(5); + JobID jobId = new JobID(); + store.addCheckpoint(savepoint(jobId, 1L)); + store.addCheckpoint(savepoint(jobId, 2L)); + + CompletedCheckpoint latestCheckpoint = store.getLatestCheckpoint(true); + + assertThat(latestCheckpoint.getCheckpointID(), equalTo(2L)); + } + + private static CompletedCheckpoint checkpoint(JobID jobId, long checkpointId) { + return new TestCompletedCheckpoint( + jobId, + checkpointId, + checkpointId, + Collections.emptyMap(), + CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE)); + } + + private static CompletedCheckpoint savepoint(JobID jobId, long checkpointId) { + return new TestCompletedCheckpoint( + jobId, + checkpointId, + checkpointId, + Collections.emptyMap(), + CheckpointProperties.forSavepoint()); + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/TaskExecutorResourceUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/TaskExecutorResourceUtilsTest.java new file mode 100644 index 0000000000000000000000000000000000000000..8577d8f1c83479aabbb403210116cc5a1912c083 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/clusterframework/TaskExecutorResourceUtilsTest.java @@ -0,0 +1,719 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.clusterframework; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.MemorySize; +import org.apache.flink.configuration.NettyShuffleEnvironmentOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.util.TestLogger; + +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +/** + * Tests for {@link TaskExecutorResourceUtils}. + */ +public class TaskExecutorResourceUtilsTest extends TestLogger { + + private static final MemorySize TASK_HEAP_SIZE = MemorySize.parse("100m"); + private static final MemorySize MANAGED_MEM_SIZE = MemorySize.parse("200m"); + private static final MemorySize TOTAL_FLINK_MEM_SIZE = MemorySize.parse("800m"); + private static final MemorySize TOTAL_PROCESS_MEM_SIZE = MemorySize.parse("1g"); + + private static final TaskExecutorResourceSpec TM_RESOURCE_SPEC = new TaskExecutorResourceSpec( + MemorySize.parse("1m"), + MemorySize.parse("2m"), + MemorySize.parse("3m"), + MemorySize.parse("4m"), + MemorySize.parse("5m"), + MemorySize.parse("6m"), + MemorySize.parse("7m"), + MemorySize.parse("8m")); + + @Test + public void testGenerateDynamicConfigurations() { + String dynamicConfigsStr = TaskExecutorResourceUtils.generateDynamicConfigsStr(TM_RESOURCE_SPEC); + Map configs = new HashMap<>(); + String[] configStrs = dynamicConfigsStr.split(" "); + assertThat(configStrs.length % 2, is(0)); + for (int i = 0; i < configStrs.length; ++i) { + String configStr = configStrs[i]; + if (i % 2 == 0) { + assertThat(configStr, is("-D")); + } else { + String[] configKV = configStr.split("="); + assertThat(configKV.length, is(2)); + configs.put(configKV[0], configKV[1]); + } + } + + assertThat(MemorySize.parse(configs.get(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getFrameworkHeapSize())); + assertThat(MemorySize.parse(configs.get(TaskManagerOptions.TASK_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getTaskHeapSize())); + assertThat(MemorySize.parse(configs.get(TaskManagerOptions.TASK_OFF_HEAP_MEMORY.key())), is(TM_RESOURCE_SPEC.getTaskOffHeapSize())); + assertThat(MemorySize.parse(configs.get(TaskManagerOptions.SHUFFLE_MEMORY_MAX.key())), is(TM_RESOURCE_SPEC.getShuffleMemSize())); + assertThat(MemorySize.parse(configs.get(TaskManagerOptions.SHUFFLE_MEMORY_MIN.key())), is(TM_RESOURCE_SPEC.getShuffleMemSize())); + assertThat(MemorySize.parse(configs.get(TaskManagerOptions.MANAGED_MEMORY_SIZE.key())), is(TM_RESOURCE_SPEC.getManagedMemorySize())); + assertThat(MemorySize.parse(configs.get(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_SIZE.key())), is(TM_RESOURCE_SPEC.getOffHeapManagedMemorySize())); + } + + @Test + public void testGenerateJvmParameters() throws Exception { + String jvmParamsStr = TaskExecutorResourceUtils.generateJvmParametersStr(TM_RESOURCE_SPEC); + MemorySize heapSizeMax = null; + MemorySize heapSizeMin = null; + MemorySize directSize = null; + MemorySize metaspaceSize = null; + for (String paramStr : jvmParamsStr.split(" ")) { + if (paramStr.startsWith("-Xmx")) { + heapSizeMax = MemorySize.parse(paramStr.substring("-Xmx".length())); + } else if (paramStr.startsWith("-Xms")) { + heapSizeMin = MemorySize.parse(paramStr.substring("-Xms".length())); + } else if (paramStr.startsWith("-XX:MaxDirectMemorySize=")) { + directSize = MemorySize.parse(paramStr.substring("-XX:MaxDirectMemorySize=".length())); + } else if (paramStr.startsWith("-XX:MaxMetaspaceSize=")) { + metaspaceSize = MemorySize.parse(paramStr.substring("-XX:MaxMetaspaceSize=".length())); + } else { + throw new Exception("Unknown JVM parameter: " + paramStr); + } + } + + assertThat(heapSizeMax, is(TM_RESOURCE_SPEC.getFrameworkHeapSize().add(TM_RESOURCE_SPEC.getTaskHeapSize()).add(TM_RESOURCE_SPEC.getOnHeapManagedMemorySize()))); + assertThat(heapSizeMin, is(heapSizeMax)); + assertThat(directSize, is(TM_RESOURCE_SPEC.getTaskOffHeapSize().add(TM_RESOURCE_SPEC.getShuffleMemSize()))); + assertThat(metaspaceSize, is(TM_RESOURCE_SPEC.getJvmMetaspaceSize())); + } + + @Test public void testConfigFrameworkHeapMemory() { + final MemorySize frameworkHeapSize = MemorySize.parse("100m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, frameworkHeapSize.getMebiBytes() + "m"); + + validateInAllConfigurations(conf, taskExecutorResourceSpec -> assertThat(taskExecutorResourceSpec.getFrameworkHeapSize(), is(frameworkHeapSize))); + } + + @Test + public void testConfigTaskHeapMemory() { + final MemorySize taskHeapSize = MemorySize.parse("50m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TASK_HEAP_MEMORY, taskHeapSize.getMebiBytes() + "m"); + + // validate in configurations without explicit task heap memory size, + // to avoid checking against overwritten task heap memory size + validateInConfigurationsWithoutExplicitTaskHeapMem(conf, taskExecutorResourceSpec -> assertThat(taskExecutorResourceSpec.getTaskHeapSize(), is(taskHeapSize))); + } + + @Test + public void testConfigTaskOffheapMemory() { + final MemorySize taskOffHeapSize = MemorySize.parse("50m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, taskOffHeapSize.getMebiBytes() + "m"); + + validateInAllConfigurations(conf, taskExecutorResourceSpec -> assertThat(taskExecutorResourceSpec.getTaskOffHeapSize(), is(taskOffHeapSize))); + } + + @Test + public void testConfigShuffleMemoryRange() { + final MemorySize shuffleMin = MemorySize.parse("50m"); + final MemorySize shuffleMax = MemorySize.parse("200m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.SHUFFLE_MEMORY_MAX, shuffleMax.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.SHUFFLE_MEMORY_MIN, shuffleMin.getMebiBytes() + "m"); + + validateInAllConfigurations(conf, taskExecutorResourceSpec -> { + assertThat(taskExecutorResourceSpec.getShuffleMemSize().getBytes(), greaterThanOrEqualTo(shuffleMin.getBytes())); + assertThat(taskExecutorResourceSpec.getShuffleMemSize().getBytes(), lessThanOrEqualTo(shuffleMax.getBytes())); + }); + } + + @Test + public void testConfigShuffleMemoryRangeFailure() { + final MemorySize shuffleMin = MemorySize.parse("200m"); + final MemorySize shuffleMax = MemorySize.parse("50m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.SHUFFLE_MEMORY_MAX, shuffleMax.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.SHUFFLE_MEMORY_MIN, shuffleMin.getMebiBytes() + "m"); + + validateFailInAllConfigurations(conf); + } + + @Test + public void testConfigShuffleMemoryFraction() { + final MemorySize shuffleMin = MemorySize.parse("0m"); + final MemorySize shuffleMax = MemorySize.parse("1t"); + final float fraction = 0.2f; + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.SHUFFLE_MEMORY_MAX, shuffleMax.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.SHUFFLE_MEMORY_MIN, shuffleMin.getMebiBytes() + "m"); + conf.setFloat(TaskManagerOptions.SHUFFLE_MEMORY_FRACTION, fraction); + + // validate in configurations without explicit total flink/process memory, otherwise explicit configured + // shuffle memory fraction might conflict with total flink/process memory minus other memory sizes + validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorResourceSpec -> + assertThat(taskExecutorResourceSpec.getShuffleMemSize(), is(taskExecutorResourceSpec.getTotalFlinkMemorySize().multiply(fraction)))); + } + + @Test + public void testConfigShuffleMemoryFractionFailure() { + Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.SHUFFLE_MEMORY_FRACTION, -0.1f); + validateFailInAllConfigurations(conf); + + conf.setFloat(TaskManagerOptions.SHUFFLE_MEMORY_FRACTION, 1.0f); + validateFailInAllConfigurations(conf); + } + + @Test + public void testConfigShuffleMemoryLegacyRangeFraction() { + final MemorySize shuffleMin = MemorySize.parse("50m"); + final MemorySize shuffleMax = MemorySize.parse("200m"); + final float fraction = 0.2f; + + @SuppressWarnings("deprecation") + final ConfigOption legacyOptionMin = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN; + @SuppressWarnings("deprecation") + final ConfigOption legacyOptionMax = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX; + @SuppressWarnings("deprecation") + final ConfigOption legacyOptionFraction = NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION; + + Configuration conf = new Configuration(); + conf.setString(legacyOptionMin, shuffleMin.getMebiBytes() + "m"); + conf.setString(legacyOptionMax, shuffleMax.getMebiBytes() + "m"); + + validateInAllConfigurations(conf, taskExecutorResourceSpec -> { + assertThat(taskExecutorResourceSpec.getShuffleMemSize().getBytes(), greaterThanOrEqualTo(shuffleMin.getBytes())); + assertThat(taskExecutorResourceSpec.getShuffleMemSize().getBytes(), lessThanOrEqualTo(shuffleMax.getBytes())); + }); + + conf.setString(legacyOptionMin, "0m"); + conf.setString(legacyOptionMax, "1t"); + conf.setFloat(legacyOptionFraction, fraction); + + validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorResourceSpec -> + assertThat(taskExecutorResourceSpec.getShuffleMemSize(), is(taskExecutorResourceSpec.getTotalFlinkMemorySize().multiply(fraction)))); + } + + @Test + public void testConfigShuffleMemoryLegacyNumOfBuffers() { + final MemorySize pageSize = MemorySize.parse("32k"); + final int numOfBuffers = 1024; + final MemorySize shuffleSize = pageSize.multiply(numOfBuffers); + + @SuppressWarnings("deprecation") + final ConfigOption legacyOption = NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS; + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.MEMORY_SEGMENT_SIZE, pageSize.getKibiBytes() + "k"); + conf.setInteger(legacyOption, numOfBuffers); + + // validate in configurations without explicit total flink/process memory, otherwise explicit configured + // shuffle memory size might conflict with total flink/process memory minus other memory sizes + validateInConfigWithExplicitTaskHeapAndManagedMem(conf, taskExecutorResourceSpec -> + assertThat(taskExecutorResourceSpec.getShuffleMemSize(), is(shuffleSize))); + } + + @Test + public void testConfigManagedMemorySize() { + final MemorySize managedMemSize = MemorySize.parse("100m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSize.getMebiBytes() + "m"); + + // validate in configurations without explicit managed memory size, + // to avoid checking against overwritten managed memory size + validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorResourceSpec -> assertThat(taskExecutorResourceSpec.getManagedMemorySize(), is(managedMemSize))); + } + + @Test + public void testConfigManagedMemoryLegacySize() { + final MemorySize managedMemSize = MemorySize.parse("100m"); + + @SuppressWarnings("deprecation") + final ConfigOption legacyOption = TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE; + + Configuration conf = new Configuration(); + conf.setString(legacyOption, managedMemSize.getMebiBytes() + "m"); + + // validate in configurations without explicit managed memory size, + // to avoid checking against overwritten managed memory size + validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorResourceSpec -> assertThat(taskExecutorResourceSpec.getManagedMemorySize(), is(managedMemSize))); + } + + @Test + public void testConfigManagedMemoryFraction() { + final float fraction = 0.5f; + + Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, fraction); + + // managed memory fraction is only used when managed memory size is not explicitly configured + validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorResourceSpec -> + assertThat(taskExecutorResourceSpec.getManagedMemorySize(), is(taskExecutorResourceSpec.getTotalFlinkMemorySize().multiply(fraction)))); + } + + @Test + public void testConfigManagedMemoryFractionFailure() { + final Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, -0.1f); + validateFailInConfigurationsWithoutExplicitManagedMem(conf); + + conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, 1.0f); + validateFailInConfigurationsWithoutExplicitManagedMem(conf); + } + + @Test + public void testConfigManagedMemoryLegacyFraction() { + final float fraction = 0.5f; + + @SuppressWarnings("deprecation") + final ConfigOption legacyOption = TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION; + + Configuration conf = new Configuration(); + conf.setFloat(legacyOption, fraction); + + // managed memory fraction is only used when managed memory size is not explicitly configured + validateInConfigurationsWithoutExplicitManagedMem(conf, taskExecutorResourceSpec -> + assertThat(taskExecutorResourceSpec.getManagedMemorySize(), is(taskExecutorResourceSpec.getTotalFlinkMemorySize().multiply(fraction)))); + } + + @Test + public void testConfigOffHeapManagedMemorySize() { + final MemorySize offHeapSize = MemorySize.parse("20m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_SIZE, offHeapSize.getMebiBytes() + "m"); + + validateInAllConfigurations(conf, taskExecutorResourceSpec -> { + assertThat(taskExecutorResourceSpec.getOffHeapManagedMemorySize(), is(offHeapSize)); + assertThat(taskExecutorResourceSpec.getOnHeapManagedMemorySize(), is(taskExecutorResourceSpec.getManagedMemorySize().subtract(taskExecutorResourceSpec.getOffHeapManagedMemorySize()))); + }); + } + + @Test + public void testConfigOffHeapManagedMemorySizeFailure() { + final MemorySize offHeapSize = MemorySize.parse("1t"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_SIZE, offHeapSize.getMebiBytes() + "m"); + + validateFailInAllConfigurations(conf); + } + + @Test + public void testConfigOffHeapManagedMemoryFraction() { + final float fraction = 0.5f; + + Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_FRACTION, fraction); + + validateInAllConfigurations(conf, taskExecutorResourceSpec -> { + assertThat(taskExecutorResourceSpec.getOffHeapManagedMemorySize(), is(taskExecutorResourceSpec.getManagedMemorySize().multiply(fraction))); + assertThat(taskExecutorResourceSpec.getOnHeapManagedMemorySize(), is(taskExecutorResourceSpec.getManagedMemorySize().subtract(taskExecutorResourceSpec.getOffHeapManagedMemorySize()))); + }); + } + + @Test + public void testConfigOffHeapManagedMemoryFractionFailure() { + Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_FRACTION, 1.1f); + validateFailInAllConfigurations(conf); + } + + @Test + public void testConfigOffHeapManagedMemoryLegacyOffHeap() { + @SuppressWarnings("deprecation") + final ConfigOption legacyOption = TaskManagerOptions.MEMORY_OFF_HEAP; + + // negative off-heap managed memory fraction means not configured, if off-heap managed memory size is also not configured, + // legacy 'taskmanager.memory.off-heap' will be used to set managed memory to either all on-heap or all off-heap + Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_FRACTION, -1.0f); + + conf.setBoolean(legacyOption, true); + validateInAllConfigurations(conf, taskExecutorResourceSpec -> { + assertThat(taskExecutorResourceSpec.getOffHeapManagedMemorySize(), is(taskExecutorResourceSpec.getManagedMemorySize())); + assertThat(taskExecutorResourceSpec.getOnHeapManagedMemorySize(), is(new MemorySize(0L))); + }); + } + + @Test + public void testConfigOffHeapManagedMemoryLegacyOnHeap() { + @SuppressWarnings("deprecation") + final ConfigOption legacyOption = TaskManagerOptions.MEMORY_OFF_HEAP; + + // negative off-heap managed memory fraction means not configured, if off-heap managed memory size is also not configured, + // legacy 'taskmanager.memory.off-heap' will be used to set managed memory to either all on-heap or all off-heap + Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_OFFHEAP_FRACTION, -1.0f); + + conf.setBoolean(legacyOption, false); + validateInAllConfigurations(conf, taskExecutorResourceSpec -> { + assertThat(taskExecutorResourceSpec.getOnHeapManagedMemorySize(), is(taskExecutorResourceSpec.getManagedMemorySize())); + assertThat(taskExecutorResourceSpec.getOffHeapManagedMemorySize(), is(new MemorySize(0L))); + }); + } + + @Test + public void testConfigJvmMetaspaceSize() { + final MemorySize jvmMetaspaceSize = MemorySize.parse("50m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.JVM_METASPACE, jvmMetaspaceSize.getMebiBytes() + "m"); + + validateInAllConfigurations(conf, taskExecutorResourceSpec -> assertThat(taskExecutorResourceSpec.getJvmMetaspaceSize(), is(jvmMetaspaceSize))); + } + + @Test + public void testConfigJvmOverheadRange() { + final MemorySize minSize = MemorySize.parse("50m"); + final MemorySize maxSize = MemorySize.parse("200m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.JVM_OVERHEAD_MAX, maxSize.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.JVM_OVERHEAD_MIN, minSize.getMebiBytes() + "m"); + + validateInAllConfigurations(conf, taskExecutorResourceSpec -> { + assertThat(taskExecutorResourceSpec.getJvmOverheadSize().getBytes(), + greaterThanOrEqualTo(minSize.getBytes())); + assertThat(taskExecutorResourceSpec.getJvmOverheadSize().getBytes(), lessThanOrEqualTo(maxSize.getBytes())); + }); + } + + @Test + public void testConfigJvmOverheadRangeFailure() { + final MemorySize minSize = MemorySize.parse("200m"); + final MemorySize maxSize = MemorySize.parse("50m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.JVM_OVERHEAD_MAX, maxSize.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.JVM_OVERHEAD_MIN, minSize.getMebiBytes() + "m"); + + validateFailInAllConfigurations(conf); + } + + @Test + public void testConfigJvmOverheadFraction() { + final MemorySize minSize = MemorySize.parse("0m"); + final MemorySize maxSize = MemorySize.parse("1t"); + final float fraction = 0.2f; + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.JVM_OVERHEAD_MAX, maxSize.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.JVM_OVERHEAD_MIN, minSize.getMebiBytes() + "m"); + conf.setFloat(TaskManagerOptions.JVM_OVERHEAD_FRACTION, fraction); + + validateInAllConfigurations(conf, taskExecutorResourceSpec -> + assertThat(taskExecutorResourceSpec.getJvmOverheadSize(), is(taskExecutorResourceSpec.getTotalProcessMemorySize().multiply(fraction)))); + } + + @Test + public void testConfigJvmOverheadFractionFailureNegative() { + final Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.JVM_OVERHEAD_FRACTION, -0.1f); + validateFailInConfigurationsWithoutExplicitManagedMem(conf); + } + + @Test + public void testConfigJvmOverheadFractionFailureNoLessThanOne() { + final Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.JVM_OVERHEAD_FRACTION, 1.0f); + validateFailInConfigurationsWithoutExplicitManagedMem(conf); + } + + @Test + public void testConfigTotalFlinkMemory() { + final MemorySize totalFlinkMemorySize = MemorySize.parse("1g"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemorySize.getMebiBytes() + "m"); + + TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(conf); + assertThat(taskExecutorResourceSpec.getTotalFlinkMemorySize(), is(totalFlinkMemorySize)); + } + + @Test + public void testFlinkInternalMemorySizeAddUpFailure() { + final MemorySize totalFlinkMemory = MemorySize.parse("499m"); + final MemorySize frameworkHeap = MemorySize.parse("100m"); + final MemorySize taskHeap = MemorySize.parse("100m"); + final MemorySize taskOffHeap = MemorySize.parse("100m"); + final MemorySize shuffle = MemorySize.parse("100m"); + final MemorySize managed = MemorySize.parse("100m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.FRAMEWORK_HEAP_MEMORY, frameworkHeap.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.TASK_HEAP_MEMORY, taskHeap.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.TASK_OFF_HEAP_MEMORY, taskOffHeap.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.SHUFFLE_MEMORY_MIN, shuffle.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.SHUFFLE_MEMORY_MAX, shuffle.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managed.getMebiBytes() + "m"); + + validateFail(conf); + } + + @Test + public void testConfigTotalFlinkMemoryLegacyMB() { + final MemorySize totalFlinkMemorySize = MemorySize.parse("1g"); + + @SuppressWarnings("deprecation") + final ConfigOption legacyOption = TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB; + + Configuration conf = new Configuration(); + conf.setInteger(legacyOption, totalFlinkMemorySize.getMebiBytes()); + + TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(conf); + assertThat(taskExecutorResourceSpec.getTotalFlinkMemorySize(), is(totalFlinkMemorySize)); + } + + @Test + public void testConfigTotalProcessMemorySize() { + final MemorySize totalProcessMemorySize = MemorySize.parse("1g"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TOTAL_PROCESS_MEMORY, totalProcessMemorySize.getMebiBytes() + "m"); + + TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(conf); + assertThat(taskExecutorResourceSpec.getTotalProcessMemorySize(), is(totalProcessMemorySize)); + } + + @Test + public void testFlinkInternalMemoryFractionAddUpFailure() { + final float shuffleFraction = 0.6f; + final float managedFraction = 0.6f; + + Configuration conf = new Configuration(); + conf.setFloat(TaskManagerOptions.SHUFFLE_MEMORY_FRACTION, shuffleFraction); + conf.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedFraction); + + // if managed memory size is explicitly configured, then managed memory fraction will be ignored + validateFailInConfigurationsWithoutExplicitManagedMem(conf); + } + + @Test + public void testConfigTotalFlinkMemoryLegacySize() { + final MemorySize totalFlinkMemorySize = MemorySize.parse("1g"); + + @SuppressWarnings("deprecation") + final ConfigOption legacyOption = TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY; + + Configuration conf = new Configuration(); + conf.setString(legacyOption, totalFlinkMemorySize.getMebiBytes() + "m"); + + TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(conf); + assertThat(taskExecutorResourceSpec.getTotalFlinkMemorySize(), is(totalFlinkMemorySize)); + } + + @Test + public void testConfigTotalProcessMemoryAddUpFailure() { + final MemorySize totalProcessMemory = MemorySize.parse("699m"); + final MemorySize totalFlinkMemory = MemorySize.parse("500m"); + final MemorySize jvmMetaspace = MemorySize.parse("100m"); + final MemorySize jvmOverhead = MemorySize.parse("100m"); + + Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TOTAL_PROCESS_MEMORY, totalProcessMemory.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.TOTAL_FLINK_MEMORY, totalFlinkMemory.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.JVM_METASPACE, jvmMetaspace.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.JVM_OVERHEAD_MIN, jvmOverhead.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.JVM_OVERHEAD_MAX, jvmOverhead.getMebiBytes() + "m"); + + validateFail(conf); + } + + private void validateInAllConfigurations(final Configuration customConfig, Consumer validateFunc) { + validateInConfigWithExplicitTaskHeapAndManagedMem(customConfig, validateFunc); + validateInConfigWithExplicitTotalFlinkMem(customConfig, validateFunc); + validateInConfigWithExplicitTotalFlinkAndTaskHeapMem(customConfig, validateFunc); + validateInConfigWithExplicitTotalFlinkAndManagedMem(customConfig, validateFunc); + validateInConfigWithExplicitTotalProcessMem(customConfig, validateFunc); + } + + private void validateFailInAllConfigurations(final Configuration customConfig) { + validateFailInConfigWithExplicitTaskHeapAndManagedMem(customConfig); + validateFailInConfigWithExplicitTotalFlinkMem(customConfig); + validateFailInConfigWithExplicitTotalFlinkAndTaskHeapMem(customConfig); + validateFailInConfigWithExplicitTotalFlinkAndManagedMem(customConfig); + validateFailInConfigWithExplicitTotalProcessMem(customConfig); + } + + private void validateInConfigurationsWithoutExplicitTaskHeapMem(final Configuration customConfig, Consumer validateFunc) { + validateInConfigWithExplicitTotalFlinkMem(customConfig, validateFunc); + validateInConfigWithExplicitTotalFlinkAndManagedMem(customConfig, validateFunc); + validateInConfigWithExplicitTotalProcessMem(customConfig, validateFunc); + } + + private void validateInConfigurationsWithoutExplicitManagedMem(final Configuration customConfig, Consumer validateFunc) { + validateInConfigWithExplicitTotalFlinkMem(customConfig, validateFunc); + validateInConfigWithExplicitTotalFlinkAndTaskHeapMem(customConfig, validateFunc); + validateInConfigWithExplicitTotalProcessMem(customConfig, validateFunc); + } + + private void validateFailInConfigurationsWithoutExplicitManagedMem(final Configuration customConfig) { + validateFailInConfigWithExplicitTotalFlinkMem(customConfig); + validateFailInConfigWithExplicitTotalFlinkAndTaskHeapMem(customConfig); + validateFailInConfigWithExplicitTotalProcessMem(customConfig); + } + + private void validateInConfigWithExplicitTaskHeapAndManagedMem( + final Configuration customConfig, Consumer validateFunc) { + log.info("Validating in configuration with explicit task heap and managed memory size."); + final Configuration config = configWithExplicitTaskHeapAndManageMem(); + config.addAll(customConfig); + TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(config); + assertThat(taskExecutorResourceSpec.getTaskHeapSize(), is(TASK_HEAP_SIZE)); + assertThat(taskExecutorResourceSpec.getManagedMemorySize(), is(MANAGED_MEM_SIZE)); + validateFunc.accept(taskExecutorResourceSpec); + } + + private void validateFailInConfigWithExplicitTaskHeapAndManagedMem(final Configuration customConfig) { + log.info("Validating failing in configuration with explicit task heap and managed memory size."); + final Configuration config = configWithExplicitTaskHeapAndManageMem(); + config.addAll(customConfig); + validateFail(config); + } + + private void validateInConfigWithExplicitTotalFlinkMem( + final Configuration customConfig, Consumer validateFunc) { + log.info("Validating in configuration with explicit total flink memory size."); + final Configuration config = configWithExplicitTotalFlinkMem(); + config.addAll(customConfig); + TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(config); + assertThat(taskExecutorResourceSpec.getTotalFlinkMemorySize(), is(TOTAL_FLINK_MEM_SIZE)); + validateFunc.accept(taskExecutorResourceSpec); + } + + private void validateFailInConfigWithExplicitTotalFlinkMem(final Configuration customConfig) { + log.info("Validating failing in configuration with explicit total flink memory size."); + final Configuration config = configWithExplicitTotalFlinkMem(); + config.addAll(customConfig); + validateFail(config); + } + + private void validateInConfigWithExplicitTotalFlinkAndTaskHeapMem( + final Configuration customConfig, Consumer validateFunc) { + log.info("Validating in configuration with explicit total flink and task heap memory size."); + final Configuration config = configWithExplicitTotalFlinkAndTaskHeapMem(); + config.addAll(customConfig); + TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(config); + assertThat(taskExecutorResourceSpec.getTotalFlinkMemorySize(), is(TOTAL_FLINK_MEM_SIZE)); + assertThat(taskExecutorResourceSpec.getTaskHeapSize(), is(TASK_HEAP_SIZE)); + validateFunc.accept(taskExecutorResourceSpec); + } + + private void validateFailInConfigWithExplicitTotalFlinkAndTaskHeapMem(final Configuration customConfig) { + log.info("Validating failing in configuration with explicit total flink and task heap memory size."); + final Configuration config = configWithExplicitTotalFlinkAndTaskHeapMem(); + config.addAll(customConfig); + validateFail(config); + } + + private void validateInConfigWithExplicitTotalFlinkAndManagedMem( + final Configuration customConfig, Consumer validateFunc) { + log.info("Validating in configuration with explicit total flink and managed memory size."); + final Configuration config = configWithExplicitTotalFlinkAndManagedMem(); + config.addAll(customConfig); + TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(config); + assertThat(taskExecutorResourceSpec.getTotalFlinkMemorySize(), is(TOTAL_FLINK_MEM_SIZE)); + assertThat(taskExecutorResourceSpec.getManagedMemorySize(), is(MANAGED_MEM_SIZE)); + validateFunc.accept(taskExecutorResourceSpec); + } + + private void validateFailInConfigWithExplicitTotalFlinkAndManagedMem(final Configuration customConfig) { + log.info("Validating failing in configuration with explicit total flink and managed memory size."); + final Configuration config = configWithExplicitTotalFlinkAndManagedMem(); + config.addAll(customConfig); + validateFail(config); + } + + private void validateInConfigWithExplicitTotalProcessMem( + final Configuration customConfig, Consumer validateFunc) { + log.info("Validating in configuration with explicit total process memory size."); + final Configuration config = configWithExplicitTotalProcessMem(); + config.addAll(customConfig); + TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(config); + assertThat(taskExecutorResourceSpec.getTotalProcessMemorySize(), is(TOTAL_PROCESS_MEM_SIZE)); + validateFunc.accept(taskExecutorResourceSpec); + } + + private void validateFailInConfigWithExplicitTotalProcessMem(final Configuration customConfig) { + log.info("Validating failing in configuration with explicit total process memory size."); + final Configuration config = configWithExplicitTotalProcessMem(); + config.addAll(customConfig); + validateFail(config); + } + + private void validateFail(final Configuration config) { + try { + TaskExecutorResourceUtils.resourceSpecFromConfig(config); + fail("Configuration did not fail as expected."); + } catch (Throwable t) { + // expected + } + } + + private static Configuration configWithExplicitTaskHeapAndManageMem() { + final Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TASK_HEAP_MEMORY, TASK_HEAP_SIZE.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, MANAGED_MEM_SIZE.getMebiBytes() + "m"); + return conf; + } + + private static Configuration configWithExplicitTotalFlinkMem() { + final Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TOTAL_FLINK_MEMORY, TOTAL_FLINK_MEM_SIZE.getMebiBytes() + "m"); + return conf; + } + + private static Configuration configWithExplicitTotalFlinkAndTaskHeapMem() { + final Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TOTAL_FLINK_MEMORY, TOTAL_FLINK_MEM_SIZE.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.TASK_HEAP_MEMORY, TASK_HEAP_SIZE.getMebiBytes() + "m"); + return conf; + } + + private static Configuration configWithExplicitTotalFlinkAndManagedMem() { + final Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TOTAL_FLINK_MEMORY, TOTAL_FLINK_MEM_SIZE.getMebiBytes() + "m"); + conf.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, MANAGED_MEM_SIZE.getMebiBytes() + "m"); + return conf; + } + + private static Configuration configWithExplicitTotalProcessMem() { + final Configuration conf = new Configuration(); + conf.setString(TaskManagerOptions.TOTAL_PROCESS_MEMORY, TOTAL_PROCESS_MEM_SIZE.getMebiBytes() + "m"); + return conf; + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/FutureUtilsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/FutureUtilsTest.java index 95ee97a9b20de641342d15b30970a257d06ca3ca..e75abb938b1c2ebae8eb5a37314c8ea2b9be5a57 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/FutureUtilsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/concurrent/FutureUtilsTest.java @@ -48,6 +48,7 @@ import java.util.function.BiFunction; import java.util.function.Supplier; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.emptyArray; @@ -765,4 +766,48 @@ public class FutureUtilsTest extends TestLogger { return exception != null; } } + + @Test + public void testForwardNormal() throws Exception { + final CompletableFuture source = new CompletableFuture<>(); + final CompletableFuture target = new CompletableFuture<>(); + + FutureUtils.forward(source, target); + + assertThat(target.isDone(), is(source.isDone())); + + source.complete("foobar"); + + assertThat(target.isDone(), is(source.isDone())); + assertThat(target.get(), is(equalTo(source.get()))); + } + + @Test + public void testForwardExceptionally() { + final CompletableFuture source = new CompletableFuture<>(); + final CompletableFuture target = new CompletableFuture<>(); + + FutureUtils.forward(source, target); + + assertThat(target.isDone(), is(source.isDone())); + + source.completeExceptionally(new FlinkException("foobar")); + + assertThat(target.isDone(), is(source.isDone())); + + Throwable targetException = getThrowable(target); + Throwable actualException = getThrowable(source); + + assertThat(targetException, is(equalTo(actualException))); + } + + private static Throwable getThrowable(CompletableFuture completableFuture) { + try { + completableFuture.join(); + } catch (CompletionException e) { + return e.getCause(); + } + + throw new AssertionError("Future has not been completed exceptionally."); + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherHATest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherHATest.java index 16840c0b700e5a5e4134d9a1c92978fad96889c0..b25c60e7e638071233d736c0ea29e7fb475712e8 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherHATest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherHATest.java @@ -42,7 +42,7 @@ import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.rpc.TestingRpcService; import org.apache.flink.runtime.testtasks.NoOpInvokable; -import org.apache.flink.runtime.testutils.InMemoryJobGraphStore; +import org.apache.flink.runtime.testutils.TestingJobGraphStore; import org.apache.flink.runtime.util.TestingFatalErrorHandler; import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; import org.apache.flink.util.ExceptionUtils; @@ -207,13 +207,14 @@ public class DispatcherHATest extends TestLogger { */ @Test public void testJobRecoveryWhenChangingLeadership() throws Exception { - final InMemoryJobGraphStore submittedJobGraphStore = new InMemoryJobGraphStore(); - final CompletableFuture recoveredJobFuture = new CompletableFuture<>(); - submittedJobGraphStore.setRecoverJobGraphFunction((jobID, jobIDSubmittedJobGraphMap) -> { - recoveredJobFuture.complete(jobID); - return jobIDSubmittedJobGraphMap.get(jobID); - }); + final TestingJobGraphStore submittedJobGraphStore = TestingJobGraphStore.newBuilder() + .setRecoverJobGraphFunction( + (jobID, jobIDSubmittedJobGraphMap) -> { + recoveredJobFuture.complete(jobID); + return jobIDSubmittedJobGraphMap.get(jobID); + }) + .build(); final TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService(); @@ -292,7 +293,7 @@ public class DispatcherHATest extends TestLogger { @Nonnull private TestingJobManagerRunnerFactory createTestingJobManagerRunnerFactory() { - return new TestingJobManagerRunnerFactory(new CompletableFuture<>(), new CompletableFuture<>(), CompletableFuture.completedFuture(null)); + return new TestingJobManagerRunnerFactory(); } private HATestingDispatcher createDispatcher(HighAvailabilityServices haServices) throws Exception { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherResourceCleanupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherResourceCleanupTest.java index 182f11d8f5f0260bed8d2768156d9ba7c269ea4c..b0979a64fc669ac929b295dcf467b968ed3255d0 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherResourceCleanupTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherResourceCleanupTest.java @@ -29,22 +29,32 @@ import org.apache.flink.runtime.blob.BlobStore; import org.apache.flink.runtime.blob.PermanentBlobKey; import org.apache.flink.runtime.blob.TestingBlobStore; import org.apache.flink.runtime.blob.TestingBlobStoreBuilder; +import org.apache.flink.runtime.client.DuplicateJobSubmissionException; import org.apache.flink.runtime.client.JobSubmissionException; import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; import org.apache.flink.runtime.heartbeat.HeartbeatServices; +import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.highavailability.RunningJobsRegistry; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobgraph.JobVertex; +import org.apache.flink.runtime.jobmanager.JobGraphStore; +import org.apache.flink.runtime.jobmaster.JobManagerRunner; +import org.apache.flink.runtime.jobmaster.JobManagerSharedServices; import org.apache.flink.runtime.jobmaster.JobNotFinishedException; +import org.apache.flink.runtime.jobmaster.TestingJobManagerRunner; +import org.apache.flink.runtime.jobmaster.factories.JobManagerJobMetricGroupFactory; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway; import org.apache.flink.runtime.rest.handler.legacy.utils.ArchivedExecutionGraphBuilder; +import org.apache.flink.runtime.rpc.FatalErrorHandler; +import org.apache.flink.runtime.rpc.RpcService; import org.apache.flink.runtime.rpc.TestingRpcService; import org.apache.flink.runtime.testtasks.NoOpInvokable; +import org.apache.flink.runtime.testutils.TestingJobGraphStore; import org.apache.flink.runtime.util.TestingFatalErrorHandler; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; @@ -70,8 +80,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -119,14 +127,10 @@ public class DispatcherResourceCleanupTest extends TestLogger { private File blobFile; - private AtomicReference> failJobMasterCreationWith; - private CompletableFuture storedHABlobFuture; private CompletableFuture deleteAllHABlobsFuture; - private CompletableFuture resultFuture; private CompletableFuture cleanupJobFuture; - private CompletableFuture terminationFuture; - private FaultyJobGraphStore submittedJobGraphStore; + private TestingJobGraphStore jobGraphStore; @BeforeClass public static void setupClass() { @@ -150,8 +154,8 @@ public class DispatcherResourceCleanupTest extends TestLogger { clearedJobLatch = new OneShotLatch(); runningJobsRegistry = new SingleRunningJobsRegistry(jobId, clearedJobLatch); highAvailabilityServices.setRunningJobsRegistry(runningJobsRegistry); - submittedJobGraphStore = new FaultyJobGraphStore(); - highAvailabilityServices.setJobGraphStore(submittedJobGraphStore); + jobGraphStore = TestingJobGraphStore.newBuilder().build(); + highAvailabilityServices.setJobGraphStore(jobGraphStore); storedHABlobFuture = new CompletableFuture<>(); deleteAllHABlobsFuture = new CompletableFuture<>(); @@ -163,7 +167,6 @@ public class DispatcherResourceCleanupTest extends TestLogger { .createTestingBlobStore(); cleanupJobFuture = new CompletableFuture<>(); - terminationFuture = new CompletableFuture<>(); blobServer = new TestingBlobServer(configuration, testingBlobStore, cleanupJobFuture); @@ -172,12 +175,27 @@ public class DispatcherResourceCleanupTest extends TestLogger { jobGraph.addUserJarBlobKey(permanentBlobKey); blobFile = blobServer.getStorageLocation(jobId, permanentBlobKey); - resultFuture = new CompletableFuture<>(); + assertThat(blobFile.exists(), is(true)); + + // verify that we stored the blob also in the BlobStore + assertThat(storedHABlobFuture.get(), equalTo(permanentBlobKey)); fatalErrorHandler = new TestingFatalErrorHandler(); + } + + private TestingJobManagerRunnerFactory startDispatcherAndSubmitJob() throws Exception { + return startDispatcherAndSubmitJob(0); + } - failJobMasterCreationWith = new AtomicReference<>(); + private TestingJobManagerRunnerFactory startDispatcherAndSubmitJob(int numBlockingJobManagerRunners) throws Exception { + final TestingJobManagerRunnerFactory testingJobManagerRunnerFactoryNG = new TestingJobManagerRunnerFactory(numBlockingJobManagerRunners); + startDispatcher(testingJobManagerRunnerFactoryNG); + submitJob(); + return testingJobManagerRunnerFactoryNG; + } + + private void startDispatcher(JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception { TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway(); dispatcher = new TestingDispatcher( rpcService, @@ -190,7 +208,7 @@ public class DispatcherResourceCleanupTest extends TestLogger { UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), null, new MemoryArchivedExecutionGraphStore(), - new TestingJobManagerRunnerFactory(new CompletableFuture<>(), resultFuture, terminationFuture, failJobMasterCreationWith), + jobManagerRunnerFactory, fatalErrorHandler); dispatcher.start(); @@ -198,11 +216,6 @@ public class DispatcherResourceCleanupTest extends TestLogger { dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class); dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get(); - - assertThat(blobFile.exists(), is(true)); - - // verify that we stored the blob also in the BlobStore - assertThat(storedHABlobFuture.get(), equalTo(permanentBlobKey)); } @After @@ -225,10 +238,10 @@ public class DispatcherResourceCleanupTest extends TestLogger { @Test public void testBlobServerCleanupWhenJobFinished() throws Exception { - submitJob(); + final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob(); // complete the job - finishJob(); + finishJob(jobManagerRunnerFactory.takeCreatedJobManagerRunner()); assertThatHABlobsHaveBeenRemoved(); } @@ -249,11 +262,11 @@ public class DispatcherResourceCleanupTest extends TestLogger { @Test public void testBlobServerCleanupWhenJobNotFinished() throws Exception { - submitJob(); + final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob(); // job not finished - resultFuture.completeExceptionally(new JobNotFinishedException(jobId)); - terminationFuture.complete(null); + final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner(); + testingJobManagerRunner.completeResultFutureExceptionally(new JobNotFinishedException(jobId)); assertThat(cleanupJobFuture.get(), equalTo(jobId)); @@ -275,7 +288,7 @@ public class DispatcherResourceCleanupTest extends TestLogger { */ @Test public void testBlobServerCleanupWhenJobSubmissionFails() throws Exception { - failJobMasterCreationWith.set(() -> new FlinkException("Test exception.")); + startDispatcher(new FailingJobManagerRunnerFactory(new FlinkException("Test exception"))); final CompletableFuture submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout); try { @@ -290,11 +303,9 @@ public class DispatcherResourceCleanupTest extends TestLogger { @Test public void testBlobServerCleanupWhenClosingDispatcher() throws Exception { - submitJob(); + startDispatcherAndSubmitJob(); - dispatcher.closeAsync(); - terminationFuture.complete(null); - dispatcher.getTerminationFuture().get(); + dispatcher.closeAsync().get(); assertThat(cleanupJobFuture.get(), equalTo(jobId)); @@ -317,13 +328,13 @@ public class DispatcherResourceCleanupTest extends TestLogger { */ @Test public void testRunningJobsRegistryCleanup() throws Exception { - submitJob(); + final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob(); runningJobsRegistry.setJobRunning(jobId); assertThat(runningJobsRegistry.contains(jobId), is(true)); - resultFuture.complete(new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).setJobID(jobId).build()); - terminationFuture.complete(null); + final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner(); + testingJobManagerRunner.completeResultFuture(new ArchivedExecutionGraphBuilder().setState(JobStatus.FINISHED).setJobID(jobId).build()); // wait for the clearing clearedJobLatch.await(); @@ -337,10 +348,11 @@ public class DispatcherResourceCleanupTest extends TestLogger { */ @Test public void testJobSubmissionUnderSameJobId() throws Exception { - submitJob(); + final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob(1); runningJobsRegistry.setJobRunning(jobId); - resultFuture.completeExceptionally(new JobNotFinishedException(jobId)); + final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner(); + testingJobManagerRunner.completeResultFutureExceptionally(new JobNotFinishedException(jobId)); final CompletableFuture submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout); @@ -351,7 +363,7 @@ public class DispatcherResourceCleanupTest extends TestLogger { } catch (TimeoutException ignored) { // expected } finally { - terminationFuture.complete(null); + testingJobManagerRunner.completeTerminationFuture(); } assertThat(submissionFuture.get(), equalTo(Acknowledge.get())); @@ -363,29 +375,28 @@ public class DispatcherResourceCleanupTest extends TestLogger { */ @Test public void testDuplicateJobSubmissionDoesNotDeleteJobMetaData() throws Exception { - submitJob(); + final TestingJobManagerRunnerFactory testingJobManagerRunnerFactoryNG = startDispatcherAndSubmitJob(); final CompletableFuture submissionFuture = dispatcherGateway.submitJob(jobGraph, timeout); try { try { submissionFuture.get(); - fail("Expected a JobSubmissionFailure."); + fail("Expected a DuplicateJobSubmissionFailure."); } catch (ExecutionException ee) { - assertThat(ExceptionUtils.findThrowable(ee, JobSubmissionException.class).isPresent(), is(true)); + assertThat(ExceptionUtils.findThrowable(ee, DuplicateJobSubmissionException.class).isPresent(), is(true)); } assertThatHABlobsHaveNotBeenRemoved(); } finally { - finishJob(); + finishJob(testingJobManagerRunnerFactoryNG.takeCreatedJobManagerRunner()); } assertThatHABlobsHaveBeenRemoved(); } - private void finishJob() { - resultFuture.complete(new ArchivedExecutionGraphBuilder().setJobID(jobId).setState(JobStatus.FINISHED).build()); - terminationFuture.complete(null); + private void finishJob(TestingJobManagerRunner takeCreatedJobManagerRunner) { + takeCreatedJobManagerRunner.completeResultFuture(new ArchivedExecutionGraphBuilder().setJobID(jobId).setState(JobStatus.FINISHED).build()); } private void assertThatHABlobsHaveNotBeenRemoved() { @@ -400,7 +411,7 @@ public class DispatcherResourceCleanupTest extends TestLogger { */ @Test public void testJobRecoveryWithPendingTermination() throws Exception { - submitJob(); + final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob(1); runningJobsRegistry.setJobRunning(jobId); dispatcherLeaderElectionService.notLeader(); @@ -413,7 +424,7 @@ public class DispatcherResourceCleanupTest extends TestLogger { } catch (TimeoutException ignored) { // expected } finally { - terminationFuture.complete(null); + jobManagerRunnerFactory.takeCreatedJobManagerRunner().completeTerminationFuture(); } assertThat(leaderFuture.get(), equalTo(leaderSessionId)); @@ -475,16 +486,23 @@ public class DispatcherResourceCleanupTest extends TestLogger { @Test public void testHABlobsAreNotRemovedIfHAJobGraphRemovalFails() throws Exception { - submittedJobGraphStore.setRemovalFailure(new Exception("Failed to Remove future")); - submitJob(); + final JobGraphStore jobGraphStore = TestingJobGraphStore.newBuilder() + .setRemoveJobGraphConsumer( + ignored -> { + throw new Exception("Failed to Remove future"); + }) + .build(); + + highAvailabilityServices.setJobGraphStore(jobGraphStore); + final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob(); ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder() .setJobID(jobId) .setState(JobStatus.CANCELED) .build(); - resultFuture.complete(executionGraph); - terminationFuture.complete(null); + final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner(); + testingJobManagerRunner.completeResultFuture(executionGraph); assertThat(cleanupJobFuture.get(), equalTo(jobId)); assertThat(deleteAllHABlobsFuture.isDone(), is(false)); @@ -492,15 +510,15 @@ public class DispatcherResourceCleanupTest extends TestLogger { @Test public void testHABlobsAreRemovedIfHAJobGraphRemovalSucceeds() throws Exception { - submitJob(); + final TestingJobManagerRunnerFactory jobManagerRunnerFactory = startDispatcherAndSubmitJob(); ArchivedExecutionGraph executionGraph = new ArchivedExecutionGraphBuilder() .setJobID(jobId) .setState(JobStatus.CANCELED) .build(); - resultFuture.complete(executionGraph); - terminationFuture.complete(null); + final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner(); + testingJobManagerRunner.completeResultFuture(executionGraph); assertThat(cleanupJobFuture.get(), equalTo(jobId)); assertThat(deleteAllHABlobsFuture.get(), equalTo(jobId)); @@ -531,4 +549,17 @@ public class DispatcherResourceCleanupTest extends TestLogger { return result; } } + + private class FailingJobManagerRunnerFactory implements JobManagerRunnerFactory { + private final Exception testException; + + public FailingJobManagerRunnerFactory(FlinkException testException) { + this.testException = testException; + } + + @Override + public JobManagerRunner createJobManagerRunner(JobGraph jobGraph, Configuration configuration, RpcService rpcService, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, JobManagerSharedServices jobManagerServices, JobManagerJobMetricGroupFactory jobManagerJobMetricGroupFactory, FatalErrorHandler fatalErrorHandler) throws Exception { + throw testException; + } + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java index dfeaac73772296f3d9fb80a2459b54341fe9b2af..46620142f154fd9692da3c331d5807281dd08c3b 100755 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/DispatcherTest.java @@ -44,6 +44,7 @@ import org.apache.flink.runtime.jobmaster.JobManagerRunner; import org.apache.flink.runtime.jobmaster.JobManagerSharedServices; import org.apache.flink.runtime.jobmaster.JobNotFinishedException; import org.apache.flink.runtime.jobmaster.JobResult; +import org.apache.flink.runtime.jobmaster.TestingJobManagerRunner; import org.apache.flink.runtime.jobmaster.factories.JobManagerJobMetricGroupFactory; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; @@ -62,7 +63,7 @@ import org.apache.flink.runtime.state.CheckpointStorageLocation; import org.apache.flink.runtime.state.CompletedCheckpointStorageLocation; import org.apache.flink.runtime.state.StateBackend; import org.apache.flink.runtime.testtasks.NoOpInvokable; -import org.apache.flink.runtime.testutils.InMemoryJobGraphStore; +import org.apache.flink.runtime.testutils.TestingJobGraphStore; import org.apache.flink.runtime.util.TestingFatalErrorHandler; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.FlinkException; @@ -134,7 +135,7 @@ public class DispatcherTest extends TestLogger { private TestingFatalErrorHandler fatalErrorHandler; - private FaultyJobGraphStore submittedJobGraphStore; + private TestingJobGraphStore jobGraphStore; private TestingLeaderElectionService dispatcherLeaderElectionService; @@ -178,14 +179,14 @@ public class DispatcherTest extends TestLogger { fatalErrorHandler = new TestingFatalErrorHandler(); heartbeatServices = new HeartbeatServices(1000L, 10000L); - submittedJobGraphStore = new FaultyJobGraphStore(); + jobGraphStore = TestingJobGraphStore.newBuilder().build(); dispatcherLeaderElectionService = new TestingLeaderElectionService(); jobMasterLeaderElectionService = new TestingLeaderElectionService(); haServices = new TestingHighAvailabilityServices(); haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService); - haServices.setJobGraphStore(submittedJobGraphStore); + haServices.setJobGraphStore(jobGraphStore); haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService); haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory()); haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService()); @@ -313,14 +314,17 @@ public class DispatcherTest extends TestLogger { */ @Test public void testLeaderElection() throws Exception { - dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch)); - CompletableFuture jobIdsFuture = new CompletableFuture<>(); - submittedJobGraphStore.setJobIdsFunction( - (Collection jobIds) -> { - jobIdsFuture.complete(null); - return jobIds; - }); + final JobGraphStore jobGraphStore = TestingJobGraphStore.newBuilder() + .setJobIdsFunction( + (Collection jobIds) -> { + jobIdsFuture.complete(null); + return jobIds; + }) + .build(); + + haServices.setJobGraphStore(jobGraphStore); + dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch)); electDispatcher(); @@ -343,16 +347,16 @@ public class DispatcherTest extends TestLogger { dispatcherGateway.submitJob(jobGraph, TIMEOUT).get(); jobMasterLeaderElectionService.isLeader(UUID.randomUUID()).get(); - final JobGraph jobGraph = submittedJobGraphStore.recoverJobGraph(TEST_JOB_ID); + final JobGraph jobGraph = jobGraphStore.recoverJobGraph(TEST_JOB_ID); // pretend that other Dispatcher has removed job from submittedJobGraphStore - submittedJobGraphStore.removeJobGraph(TEST_JOB_ID); + jobGraphStore.removeJobGraph(TEST_JOB_ID); dispatcher.onRemovedJobGraph(TEST_JOB_ID); assertThat(dispatcherGateway.listJobs(TIMEOUT).get(), empty()); // pretend that other Dispatcher has added a job to submittedJobGraphStore runningJobsRegistry.clearJob(TEST_JOB_ID); - submittedJobGraphStore.putJobGraph(jobGraph); + jobGraphStore.putJobGraph(jobGraph); dispatcher.onAddedJobGraph(TEST_JOB_ID); createdJobManagerRunnerLatch.await(); assertThat(dispatcherGateway.listJobs(TIMEOUT).get(), hasSize(1)); @@ -360,14 +364,20 @@ public class DispatcherTest extends TestLogger { @Test public void testOnAddedJobGraphRecoveryFailure() throws Exception { - dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch)); - final FlinkException expectedFailure = new FlinkException("Expected failure"); - submittedJobGraphStore.setRecoveryFailure(expectedFailure); + final JobGraphStore jobGraphStore = TestingJobGraphStore.newBuilder() + .setRecoverJobGraphFunction( + (ignoredA, ignoredB) -> { + throw expectedFailure; + }) + .build(); + + haServices.setJobGraphStore(jobGraphStore); + dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch)); dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get(); - submittedJobGraphStore.putJobGraph(jobGraph); + jobGraphStore.putJobGraph(jobGraph); dispatcher.onAddedJobGraph(TEST_JOB_ID); final CompletableFuture errorFuture = fatalErrorHandler.getErrorFuture(); @@ -385,7 +395,7 @@ public class DispatcherTest extends TestLogger { dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get(); - submittedJobGraphStore.putJobGraph(jobGraph); + jobGraphStore.putJobGraph(jobGraph); runningJobsRegistry.setJobFinished(TEST_JOB_ID); dispatcher.onAddedJobGraph(TEST_JOB_ID); @@ -459,7 +469,7 @@ public class DispatcherTest extends TestLogger { dispatcherGateway.submitJob(jobGraph, TIMEOUT).get(); // check that the job has been persisted - assertThat(submittedJobGraphStore.getJobIds(), contains(jobGraph.getJobID())); + assertThat(jobGraphStore.getJobIds(), contains(jobGraph.getJobID())); jobMasterLeaderElectionService.isLeader(UUID.randomUUID()).get(); @@ -556,13 +566,16 @@ public class DispatcherTest extends TestLogger { */ @Test public void testFatalErrorAfterJobIdRecoveryFailure() throws Exception { - dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch)); - final FlinkException testException = new FlinkException("Test exception"); - submittedJobGraphStore.setJobIdsFunction( - (Collection jobIds) -> { - throw testException; - }); + final JobGraphStore jobGraphStore = TestingJobGraphStore.newBuilder() + .setJobIdsFunction( + (Collection jobIds) -> { + throw testException; + }) + .build(); + + haServices.setJobGraphStore(jobGraphStore); + dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch)); electDispatcher(); @@ -581,17 +594,19 @@ public class DispatcherTest extends TestLogger { @Test public void testFatalErrorAfterJobRecoveryFailure() throws Exception { final FlinkException testException = new FlinkException("Test exception"); + final TestingJobGraphStore jobGraphStore = TestingJobGraphStore.newBuilder() + .setRecoverJobGraphFunction( + (JobID jobId, Map jobGraphs) -> { + throw testException; + }) + .build(); + haServices.setJobGraphStore(jobGraphStore); dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch)); dispatcher.waitUntilStarted(); - submittedJobGraphStore.putJobGraph(jobGraph); - - submittedJobGraphStore.setRecoverJobGraphFunction( - (JobID jobId, Map jobGraphs) -> { - throw testException; - }); + jobGraphStore.putJobGraph(jobGraph); electDispatcher(); @@ -617,7 +632,7 @@ public class DispatcherTest extends TestLogger { final JobGraph failingJobGraph = createFailingJobGraph(testException); - submittedJobGraphStore.putJobGraph(failingJobGraph); + jobGraphStore.putJobGraph(failingJobGraph); electDispatcher(); @@ -707,7 +722,7 @@ public class DispatcherTest extends TestLogger { @Test public void testPersistedJobGraphWhenDispatcherIsShutDown() throws Exception { - final InMemoryJobGraphStore submittedJobGraphStore = new InMemoryJobGraphStore(); + final TestingJobGraphStore submittedJobGraphStore = TestingJobGraphStore.newBuilder().build(); haServices.setJobGraphStore(submittedJobGraphStore); dispatcher = createAndStartDispatcher(heartbeatServices, haServices, DefaultJobManagerRunnerFactory.INSTANCE); @@ -754,14 +769,14 @@ public class DispatcherTest extends TestLogger { } @Test - public void testShutDownClusterShouldTerminateDispatcher() throws Exception { + public void testShutDownClusterShouldCompleteShutDownFuture() throws Exception { dispatcher = createAndStartDispatcher(heartbeatServices, haServices, DefaultJobManagerRunnerFactory.INSTANCE); dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get(); final DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class); dispatcherGateway.shutDownCluster().get(); - dispatcher.getTerminationFuture().get(); + dispatcher.getShutDownFuture().get(); } private final class BlockingJobManagerRunnerFactory extends TestingJobManagerRunnerFactory { @@ -770,13 +785,11 @@ public class DispatcherTest extends TestLogger { private final ThrowingRunnable jobManagerRunnerCreationLatch; BlockingJobManagerRunnerFactory(@Nonnull ThrowingRunnable jobManagerRunnerCreationLatch) { - super(new CompletableFuture<>(), new CompletableFuture<>(), CompletableFuture.completedFuture(null)); - this.jobManagerRunnerCreationLatch = jobManagerRunnerCreationLatch; } @Override - public JobManagerRunner createJobManagerRunner(JobGraph jobGraph, Configuration configuration, RpcService rpcService, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, JobManagerSharedServices jobManagerSharedServices, JobManagerJobMetricGroupFactory jobManagerJobMetricGroupFactory, FatalErrorHandler fatalErrorHandler) throws Exception { + public TestingJobManagerRunner createJobManagerRunner(JobGraph jobGraph, Configuration configuration, RpcService rpcService, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, JobManagerSharedServices jobManagerSharedServices, JobManagerJobMetricGroupFactory jobManagerJobMetricGroupFactory, FatalErrorHandler fatalErrorHandler) throws Exception { jobManagerRunnerCreationLatch.run(); return super.createJobManagerRunner(jobGraph, configuration, rpcService, highAvailabilityServices, heartbeatServices, jobManagerSharedServices, jobManagerJobMetricGroupFactory, fatalErrorHandler); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStoreTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStoreTest.java index cc116dd771477b48423527b002947c13fad35526..b3b31ac7572e95c1fdfc199c66586199e4054a77 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStoreTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FileArchivedExecutionGraphStoreTest.java @@ -54,7 +54,9 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; /** * Tests for the {@link FileArchivedExecutionGraphStore}. @@ -136,7 +138,7 @@ public class FileArchivedExecutionGraphStoreTest extends TestLogger { final int numberExecutionGraphs = 10; final Collection executionGraphs = generateTerminalExecutionGraphs(numberExecutionGraphs); - final Collection jobDetails = executionGraphs.stream().map(WebMonitorUtils::createDetailsForJob).collect(Collectors.toList()); + final Collection jobDetails = generateJobDetails(executionGraphs); final File rootDir = temporaryFolder.newFolder(); @@ -165,6 +167,7 @@ public class FileArchivedExecutionGraphStoreTest extends TestLogger { try (final FileArchivedExecutionGraphStore executionGraphStore = new FileArchivedExecutionGraphStore( rootDir, expirationTime, + Integer.MAX_VALUE, 10000L, scheduledExecutor, manualTicker)) { @@ -227,6 +230,7 @@ public class FileArchivedExecutionGraphStoreTest extends TestLogger { try (final FileArchivedExecutionGraphStore executionGraphStore = new FileArchivedExecutionGraphStore( rootDir, Time.hours(1L), + Integer.MAX_VALUE, 100L << 10, TestingUtils.defaultScheduledExecutor(), Ticker.systemTicker())) { @@ -259,6 +263,47 @@ public class FileArchivedExecutionGraphStoreTest extends TestLogger { } } + /** + * Tests that the size of {@link FileArchivedExecutionGraphStore} is no more than the configured max capacity + * and the old execution graphs will be purged if the total added number exceeds the max capacity. + */ + @Test + public void testMaximumCapacity() throws IOException { + final File rootDir = temporaryFolder.newFolder(); + + final int maxCapacity = 10; + final int numberExecutionGraphs = 10; + + final Collection oldExecutionGraphs = generateTerminalExecutionGraphs(numberExecutionGraphs); + final Collection newExecutionGraphs = generateTerminalExecutionGraphs(numberExecutionGraphs); + + final Collection jobDetails = generateJobDetails(newExecutionGraphs); + + try (final FileArchivedExecutionGraphStore executionGraphStore = new FileArchivedExecutionGraphStore( + rootDir, + Time.hours(1L), + maxCapacity, + 10000L, + TestingUtils.defaultScheduledExecutor(), + Ticker.systemTicker())) { + + for (ArchivedExecutionGraph executionGraph : oldExecutionGraphs) { + executionGraphStore.put(executionGraph); + // no more than the configured maximum capacity + assertTrue(executionGraphStore.size() <= maxCapacity); + } + + for (ArchivedExecutionGraph executionGraph : newExecutionGraphs) { + executionGraphStore.put(executionGraph); + // equals to the configured maximum capacity + assertEquals(maxCapacity, executionGraphStore.size()); + } + + // the older execution graphs are purged + assertThat(executionGraphStore.getAvailableJobDetails(), Matchers.containsInAnyOrder(jobDetails.toArray())); + } + } + private Collection generateTerminalExecutionGraphs(int number) { final Collection executionGraphs = new ArrayList<>(number); @@ -277,6 +322,7 @@ public class FileArchivedExecutionGraphStoreTest extends TestLogger { return new FileArchivedExecutionGraphStore( storageDirectory, Time.hours(1L), + Integer.MAX_VALUE, 10000L, TestingUtils.defaultScheduledExecutor(), Ticker.systemTicker()); @@ -318,4 +364,8 @@ public class FileArchivedExecutionGraphStoreTest extends TestLogger { private static Matcher matchesPartiallyWith(ArchivedExecutionGraph executionGraph) { return new PartialArchivedExecutionGraphMatcher(executionGraph); } + + private static Collection generateJobDetails(Collection executionGraphs) { + return executionGraphs.stream().map(WebMonitorUtils::createDetailsForJob).collect(Collectors.toList()); + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java index 22490fc25f793370731d3e93fd7e760eae0544e2..8f855298c3a2d2cedc30cf71e92c7e232a4483ea 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/MiniDispatcherTest.java @@ -30,6 +30,7 @@ import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobmaster.JobResult; +import org.apache.flink.runtime.jobmaster.TestingJobManagerRunner; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway; @@ -85,10 +86,6 @@ public class MiniDispatcherTest extends TestLogger { private final ArchivedExecutionGraphStore archivedExecutionGraphStore = new MemoryArchivedExecutionGraphStore(); - private CompletableFuture jobGraphFuture; - - private CompletableFuture resultFuture; - private TestingLeaderElectionService dispatcherLeaderElectionService; private TestingHighAvailabilityServices highAvailabilityServices; @@ -122,10 +119,7 @@ public class MiniDispatcherTest extends TestLogger { highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService); - jobGraphFuture = new CompletableFuture<>(); - resultFuture = new CompletableFuture<>(); - - testingJobManagerRunnerFactory = new TestingJobManagerRunnerFactory(jobGraphFuture, resultFuture, CompletableFuture.completedFuture(null)); + testingJobManagerRunnerFactory = new TestingJobManagerRunnerFactory(); } @After @@ -158,9 +152,9 @@ public class MiniDispatcherTest extends TestLogger { // wait until the Dispatcher is the leader dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get(); - final JobGraph actualJobGraph = jobGraphFuture.get(); + final TestingJobManagerRunner testingJobManagerRunner = testingJobManagerRunnerFactory.takeCreatedJobManagerRunner(); - assertThat(actualJobGraph.getJobID(), is(jobGraph.getJobID())); + assertThat(testingJobManagerRunner.getJobID(), is(jobGraph.getJobID())); } finally { RpcUtils.terminateRpcEndpoint(miniDispatcher, timeout); } @@ -181,12 +175,12 @@ public class MiniDispatcherTest extends TestLogger { dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get(); // wait until we have submitted the job - jobGraphFuture.get(); + final TestingJobManagerRunner testingJobManagerRunner = testingJobManagerRunnerFactory.takeCreatedJobManagerRunner(); - resultFuture.complete(archivedExecutionGraph); + testingJobManagerRunner.completeResultFuture(archivedExecutionGraph); // wait until we terminate - miniDispatcher.getJobTerminationFuture().get(); + miniDispatcher.getShutDownFuture().get(); } finally { RpcUtils.terminateRpcEndpoint(miniDispatcher, timeout); } @@ -207,9 +201,9 @@ public class MiniDispatcherTest extends TestLogger { dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get(); // wait until we have submitted the job - jobGraphFuture.get(); + final TestingJobManagerRunner testingJobManagerRunner = testingJobManagerRunnerFactory.takeCreatedJobManagerRunner(); - resultFuture.complete(archivedExecutionGraph); + testingJobManagerRunner.completeResultFuture(archivedExecutionGraph); assertFalse(miniDispatcher.getTerminationFuture().isDone()); @@ -235,17 +229,18 @@ public class MiniDispatcherTest extends TestLogger { return new MiniDispatcher( rpcService, UUID.randomUUID().toString(), - configuration, - highAvailabilityServices, - () -> CompletableFuture.completedFuture(resourceManagerGateway), - blobServer, - heartbeatServices, - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), - null, - archivedExecutionGraphStore, - testingJobManagerRunnerFactory, - testingFatalErrorHandler, - VoidHistoryServerArchivist.INSTANCE, + new DispatcherServices( + configuration, + highAvailabilityServices, + () -> CompletableFuture.completedFuture(resourceManagerGateway), + blobServer, + heartbeatServices, + UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), + archivedExecutionGraphStore, + testingFatalErrorHandler, + VoidHistoryServerArchivist.INSTANCE, + null, + testingJobManagerRunnerFactory), jobGraph, executionMode); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/TestingDispatcher.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/TestingDispatcher.java index 43fc96f258578c8b8ff37e98f47d8679a7132192..90b70532d819cc1c25d85c597c5a6c3d812ef8a0 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/TestingDispatcher.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/TestingDispatcher.java @@ -60,18 +60,19 @@ class TestingDispatcher extends Dispatcher { super( rpcService, endpointId, - configuration, - highAvailabilityServices, - highAvailabilityServices.getJobGraphStore(), - resourceManagerGatewayRetriever, - blobServer, - heartbeatServices, - jobManagerMetricGroup, - metricQueryServiceAddress, - archivedExecutionGraphStore, - jobManagerRunnerFactory, - fatalErrorHandler, - VoidHistoryServerArchivist.INSTANCE); + new DispatcherServices( + configuration, + highAvailabilityServices, + resourceManagerGatewayRetriever, + blobServer, + heartbeatServices, + jobManagerMetricGroup, + archivedExecutionGraphStore, + fatalErrorHandler, + VoidHistoryServerArchivist.INSTANCE, + metricQueryServiceAddress, + jobManagerRunnerFactory), + highAvailabilityServices.getJobGraphStore()); this.startFuture = new CompletableFuture<>(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/TestingJobManagerRunnerFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/TestingJobManagerRunnerFactory.java index c19038c5e9b6e8daff14ffac6f9bf9abe4cb2152..c6f34d72641424be4c6a6c1cde948e94092632c8 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/TestingJobManagerRunnerFactory.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/TestingJobManagerRunnerFactory.java @@ -19,75 +19,68 @@ package org.apache.flink.runtime.dispatcher; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.jobmaster.JobManagerRunner; import org.apache.flink.runtime.jobmaster.JobManagerSharedServices; +import org.apache.flink.runtime.jobmaster.TestingJobManagerRunner; import org.apache.flink.runtime.jobmaster.factories.JobManagerJobMetricGroupFactory; import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; +import javax.annotation.Nonnull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; /** - * {@link JobManagerRunnerFactory} implementation for - * testing purposes. + * Testing implementation of {@link JobManagerRunnerFactory} which returns a {@link TestingJobManagerRunner}. */ class TestingJobManagerRunnerFactory implements JobManagerRunnerFactory { - private final CompletableFuture jobGraphFuture; - private final CompletableFuture resultFuture; - private final CompletableFuture terminationFuture; - private final AtomicReference> failJobMasterCreationWith; + private final BlockingQueue createdJobManagerRunner = new ArrayBlockingQueue<>(16); - TestingJobManagerRunnerFactory( - CompletableFuture jobGraphFuture, - CompletableFuture resultFuture, - CompletableFuture terminationFuture) { - this(jobGraphFuture, resultFuture, terminationFuture, new AtomicReference<>()); + private int numBlockingJobManagerRunners; + + public TestingJobManagerRunnerFactory() { + this(0); } - TestingJobManagerRunnerFactory( - CompletableFuture jobGraphFuture, - CompletableFuture resultFuture, - CompletableFuture terminationFuture, - AtomicReference> failJobMasterCreationWith) { - this.jobGraphFuture = jobGraphFuture; - this.resultFuture = resultFuture; - this.terminationFuture = terminationFuture; - this.failJobMasterCreationWith = failJobMasterCreationWith; + public TestingJobManagerRunnerFactory(int numBlockingJobManagerRunners) { + this.numBlockingJobManagerRunners = numBlockingJobManagerRunners; } @Override - public JobManagerRunner createJobManagerRunner( + public TestingJobManagerRunner createJobManagerRunner( JobGraph jobGraph, Configuration configuration, RpcService rpcService, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, - JobManagerSharedServices jobManagerSharedServices, + JobManagerSharedServices jobManagerServices, JobManagerJobMetricGroupFactory jobManagerJobMetricGroupFactory, FatalErrorHandler fatalErrorHandler) throws Exception { - final Supplier exceptionSupplier = failJobMasterCreationWith.get(); + final TestingJobManagerRunner testingJobManagerRunner = createTestingJobManagerRunner(jobGraph); + createdJobManagerRunner.offer(testingJobManagerRunner); - if (exceptionSupplier != null) { - throw exceptionSupplier.get(); - } else { - jobGraphFuture.complete(jobGraph); + return testingJobManagerRunner; + } - final JobManagerRunner mock = mock(JobManagerRunner.class); - when(mock.getResultFuture()).thenReturn(resultFuture); - when(mock.closeAsync()).thenReturn(terminationFuture); - when(mock.getJobGraph()).thenReturn(jobGraph); + @Nonnull + private TestingJobManagerRunner createTestingJobManagerRunner(JobGraph jobGraph) { + final boolean blockingTermination; - return mock; + if (numBlockingJobManagerRunners > 0) { + numBlockingJobManagerRunners--; + blockingTermination = true; + } else { + blockingTermination = false; } + + return new TestingJobManagerRunner(jobGraph.getJobID(), blockingTermination); + } + + public TestingJobManagerRunner takeCreatedJobManagerRunner() throws InterruptedException { + return createdJobManagerRunner.take(); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/ZooKeeperHADispatcherTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/ZooKeeperHADispatcherTest.java index dd535160b7ffb1039a3d88b4edc835cf80658903..b40946c7431fb7e4ccbdb9120c52076be5542500 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/ZooKeeperHADispatcherTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/ZooKeeperHADispatcherTest.java @@ -24,7 +24,6 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.HighAvailabilityOptions; import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.blob.VoidBlobStore; -import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; @@ -33,6 +32,7 @@ import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.jobmanager.ZooKeeperJobGraphStore; import org.apache.flink.runtime.jobmaster.JobResult; +import org.apache.flink.runtime.jobmaster.TestingJobManagerRunner; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; @@ -62,6 +62,7 @@ import org.junit.rules.TestName; import javax.annotation.Nonnull; import java.io.IOException; +import java.time.Duration; import java.util.Collection; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -153,7 +154,7 @@ public class ZooKeeperHADispatcherTest extends TestLogger { final TestingDispatcher dispatcher = createDispatcher( testingHighAvailabilityServices, - new TestingJobManagerRunnerFactory(new CompletableFuture<>(), new CompletableFuture<>(), CompletableFuture.completedFuture(null))); + new TestingJobManagerRunnerFactory()); dispatcher.start(); @@ -218,15 +219,14 @@ public class ZooKeeperHADispatcherTest extends TestLogger { final TestingLeaderElectionService leaderElectionService2 = new TestingLeaderElectionService(); haServices2.setDispatcherLeaderElectionService(leaderElectionService2); - final CompletableFuture jobGraphFuture = new CompletableFuture<>(); - final CompletableFuture resultFuture = new CompletableFuture<>(); + final TestingJobManagerRunnerFactory jobManagerRunnerFactory = new TestingJobManagerRunnerFactory(); final TestingDispatcher dispatcher1 = createDispatcher( haServices1, - new TestingJobManagerRunnerFactory(jobGraphFuture, resultFuture, CompletableFuture.completedFuture(null))); + jobManagerRunnerFactory); final TestingDispatcher dispatcher2 = createDispatcher( haServices2, - new TestingJobManagerRunnerFactory(new CompletableFuture<>(), new CompletableFuture<>(), CompletableFuture.completedFuture(null))); + new TestingJobManagerRunnerFactory()); try { dispatcher1.start(); @@ -241,10 +241,10 @@ public class ZooKeeperHADispatcherTest extends TestLogger { final CompletableFuture jobResultFuture = dispatcherGateway1.requestJobResult(jobGraph.getJobID(), TIMEOUT); - jobGraphFuture.get(); + final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner(); // complete the job - resultFuture.complete(new ArchivedExecutionGraphBuilder().setJobID(jobGraph.getJobID()).setState(JobStatus.FINISHED).build()); + testingJobManagerRunner.completeResultFuture(new ArchivedExecutionGraphBuilder().setJobID(jobGraph.getJobID()).setState(JobStatus.FINISHED).build()); final JobResult jobResult = jobResultFuture.get(); @@ -281,32 +281,37 @@ public class ZooKeeperHADispatcherTest extends TestLogger { try { haServices = new ZooKeeperHaServices(curatorFramework, rpcService.getExecutor(), configuration, new VoidBlobStore()); - final CompletableFuture jobGraphFuture1 = new CompletableFuture<>(); + final TestingJobManagerRunnerFactory jobManagerRunnerFactory = new TestingJobManagerRunnerFactory(); dispatcher1 = createDispatcher( haServices, - new TestingJobManagerRunnerFactory(jobGraphFuture1, new CompletableFuture<>(), CompletableFuture.completedFuture(null))); - final CompletableFuture jobGraphFuture2 = new CompletableFuture<>(); + jobManagerRunnerFactory); dispatcher2 = createDispatcher( haServices, - new TestingJobManagerRunnerFactory(jobGraphFuture2, new CompletableFuture<>(), CompletableFuture.completedFuture(null))); + jobManagerRunnerFactory); dispatcher1.start(); dispatcher2.start(); - final LeaderConnectionInfo leaderConnectionInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo(haServices.getDispatcherLeaderRetriever(), TIMEOUT); + final LeaderConnectionInfo leaderConnectionInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo( + haServices.getDispatcherLeaderRetriever(), + Duration.ofMillis(TIMEOUT.toMilliseconds())); - final DispatcherGateway dispatcherGateway = rpcService.connect(leaderConnectionInfo.getAddress(), DispatcherId.fromUuid(leaderConnectionInfo.getLeaderSessionID()), DispatcherGateway.class).get(); + final DispatcherGateway dispatcherGateway = rpcService.connect(leaderConnectionInfo.getAddress(), DispatcherId.fromUuid(leaderConnectionInfo.getLeaderSessionId()), DispatcherGateway.class).get(); final JobGraph nonEmptyJobGraph = DispatcherHATest.createNonEmptyJobGraph(); dispatcherGateway.submitJob(nonEmptyJobGraph, TIMEOUT).get(); + // pop first job manager runner + jobManagerRunnerFactory.takeCreatedJobManagerRunner(); + if (dispatcher1.getAddress().equals(leaderConnectionInfo.getAddress())) { dispatcher1.closeAsync(); - assertThat(jobGraphFuture2.get().getJobID(), is(equalTo(nonEmptyJobGraph.getJobID()))); } else { dispatcher2.closeAsync(); - assertThat(jobGraphFuture1.get().getJobID(), is(equalTo(nonEmptyJobGraph.getJobID()))); } + + final TestingJobManagerRunner testingJobManagerRunner = jobManagerRunnerFactory.takeCreatedJobManagerRunner(); + assertThat(testingJobManagerRunner.getJobID(), is(equalTo(nonEmptyJobGraph.getJobID()))); } finally { if (dispatcher1 != null) { RpcUtils.terminateRpcEndpoint(dispatcher1, TIMEOUT); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/entrypoint/component/TestingDefaultDispatcherResourceManagerComponentFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/entrypoint/component/TestingDefaultDispatcherResourceManagerComponentFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..03fe15f87b352a4bbab651aa74939cc08efa6514 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/entrypoint/component/TestingDefaultDispatcherResourceManagerComponentFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.entrypoint.component; + +import org.apache.flink.runtime.clusterframework.types.ResourceID; +import org.apache.flink.runtime.dispatcher.runner.DispatcherRunnerFactory; +import org.apache.flink.runtime.resourcemanager.ResourceManagerFactory; +import org.apache.flink.runtime.rest.SessionRestEndpointFactory; + +/** + * Testing extension of the {@link DefaultDispatcherResourceManagerComponentFactory}. + */ +public class TestingDefaultDispatcherResourceManagerComponentFactory { + public static DispatcherResourceManagerComponentFactory createSessionComponentFactory( + DispatcherRunnerFactory dispatcherRunnerFactory, + ResourceManagerFactory resourceManagerFactory) { + return new DefaultDispatcherResourceManagerComponentFactory( + dispatcherRunnerFactory, + resourceManagerFactory, + SessionRestEndpointFactory.INSTANCE); + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ConcurrentFailoverStrategyExecutionGraphTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ConcurrentFailoverStrategyExecutionGraphTest.java deleted file mode 100644 index 3198dc19366042d2bce17c4933cf9fcb39fbcf61..0000000000000000000000000000000000000000 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ConcurrentFailoverStrategyExecutionGraphTest.java +++ /dev/null @@ -1,540 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.executiongraph; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.api.common.time.Time; -import org.apache.flink.runtime.blob.VoidBlobWriter; -import org.apache.flink.runtime.checkpoint.CheckpointCoordinator; -import org.apache.flink.runtime.checkpoint.CheckpointOptions; -import org.apache.flink.runtime.checkpoint.CheckpointRetentionPolicy; -import org.apache.flink.runtime.checkpoint.CheckpointStatsTracker; -import org.apache.flink.runtime.checkpoint.PendingCheckpoint; -import org.apache.flink.runtime.checkpoint.StandaloneCheckpointIDCounter; -import org.apache.flink.runtime.checkpoint.StandaloneCompletedCheckpointStore; -import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutor; -import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter; -import org.apache.flink.runtime.concurrent.FutureUtils; -import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor; -import org.apache.flink.runtime.execution.ExecutionState; -import org.apache.flink.runtime.execution.SuppressRestartsException; -import org.apache.flink.runtime.executiongraph.failover.FailoverRegion; -import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy.Factory; -import org.apache.flink.runtime.executiongraph.failover.RestartIndividualStrategy; -import org.apache.flink.runtime.executiongraph.failover.RestartPipelinedRegionStrategy; -import org.apache.flink.runtime.executiongraph.restart.RestartStrategy; -import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.jobgraph.JobStatus; -import org.apache.flink.runtime.jobgraph.JobVertex; -import org.apache.flink.runtime.jobgraph.JobVertexID; -import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; -import org.apache.flink.runtime.jobmanager.slots.TaskManagerGateway; -import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; -import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint; -import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; -import org.apache.flink.runtime.state.memory.MemoryStateBackend; -import org.apache.flink.runtime.testingUtils.TestingUtils; -import org.apache.flink.runtime.testtasks.NoOpInvokable; -import org.apache.flink.util.TestLogger; - -import org.junit.Test; - -import javax.annotation.Nonnull; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; - -import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilExecutionState; -import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilJobStatus; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * These tests make sure that global failover (restart all) always takes precedence over - * local recovery strategies. - * - *

    This test must be in the package it resides in, because it uses package-private methods - * from the ExecutionGraph classes. - */ -public class ConcurrentFailoverStrategyExecutionGraphTest extends TestLogger { - - private final ComponentMainThreadExecutor mainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread(); - - /** - * Tests that a cancellation concurrent to a local failover leads to a properly - * cancelled state. - */ - @Test - public void testCancelWhileInLocalFailover() throws Exception { - - // the logic in this test is as follows: - // - start a job - // - cause a task failure and delay the local recovery action via the manual executor - // - cancel the job to go into cancelling - // - resume in local recovery action - // - validate that this does in fact not start a new task, because the graph as a - // whole should now be cancelled already - - final JobID jid = new JobID(); - final int parallelism = 2; - - final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism); - - final ExecutionGraph graph = createSampleGraph( - jid, - TestRestartPipelinedRegionStrategy::new, - TestRestartStrategy.directExecuting(), - slotProvider, - parallelism); - - graph.start(mainThreadExecutor); - TestRestartPipelinedRegionStrategy strategy = (TestRestartPipelinedRegionStrategy) graph.getFailoverStrategy(); - - // This future is used to block the failover strategy execution until we complete it - final CompletableFuture blocker = new CompletableFuture<>(); - strategy.setBlockerFuture(blocker); - - final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next(); - final ExecutionVertex vertex1 = ejv.getTaskVertices()[0]; - final ExecutionVertex vertex2 = ejv.getTaskVertices()[1]; - - graph.scheduleForExecution(); - assertEquals(JobStatus.RUNNING, graph.getState()); - - // let one of the vertices fail - that triggers a local recovery action - vertex1.getCurrentExecutionAttempt().fail(new Exception("test failure")); - assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState()); - - // graph should still be running and the failover recovery action should be queued - assertEquals(JobStatus.RUNNING, graph.getState()); - - // now cancel the job - graph.cancel(); - - assertEquals(JobStatus.CANCELLING, graph.getState()); - assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState()); - assertEquals(ExecutionState.CANCELING, vertex2.getCurrentExecutionAttempt().getState()); - - // let the recovery action continue - blocker.complete(null); - - // now report that cancelling is complete for the other vertex - vertex2.getCurrentExecutionAttempt().completeCancelling(); - - assertEquals(JobStatus.CANCELED, graph.getTerminationFuture().get()); - assertTrue(vertex1.getCurrentExecutionAttempt().getState().isTerminal()); - assertTrue(vertex2.getCurrentExecutionAttempt().getState().isTerminal()); - - // make sure all slots are recycled - assertEquals(parallelism, slotProvider.getNumberOfAvailableSlots()); - } - - /** - * Tests that a terminal global failure concurrent to a local failover - * leads to a properly failed state. - */ - @Test - public void testGlobalFailureConcurrentToLocalFailover() throws Exception { - - // the logic in this test is as follows: - // - start a job - // - cause a task failure and delay the local recovery action via the manual executor - // - cause a global failure - // - resume in local recovery action - // - validate that this does in fact not start a new task, because the graph as a - // whole should now be terminally failed already - - final JobID jid = new JobID(); - final int parallelism = 2; - - final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism); - - final ExecutionGraph graph = createSampleGraph( - jid, - TestRestartPipelinedRegionStrategy::new, - TestRestartStrategy.directExecuting(), - slotProvider, - parallelism); - - graph.start(mainThreadExecutor); - TestRestartPipelinedRegionStrategy strategy = (TestRestartPipelinedRegionStrategy) graph.getFailoverStrategy(); - - // This future is used to block the failover strategy execution until we complete it - final CompletableFuture blocker = new CompletableFuture<>(); - strategy.setBlockerFuture(blocker); - - final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next(); - final ExecutionVertex vertex1 = ejv.getTaskVertices()[0]; - final ExecutionVertex vertex2 = ejv.getTaskVertices()[1]; - - graph.scheduleForExecution(); - assertEquals(JobStatus.RUNNING, graph.getState()); - - // let one of the vertices fail - that triggers a local recovery action - vertex1.getCurrentExecutionAttempt().fail(new Exception("test failure")); - assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState()); - - // graph should still be running and the failover recovery action should be queued - assertEquals(JobStatus.RUNNING, graph.getState()); - - // now cancel the job - graph.failGlobal(new SuppressRestartsException(new Exception("test exception"))); - - assertEquals(JobStatus.FAILING, graph.getState()); - assertEquals(ExecutionState.FAILED, vertex1.getCurrentExecutionAttempt().getState()); - assertEquals(ExecutionState.CANCELING, vertex2.getCurrentExecutionAttempt().getState()); - - // let the recovery action continue - blocker.complete(null); - - // now report that cancelling is complete for the other vertex - vertex2.getCurrentExecutionAttempt().completeCancelling(); - - assertEquals(JobStatus.FAILED, graph.getState()); - assertTrue(vertex1.getCurrentExecutionAttempt().getState().isTerminal()); - assertTrue(vertex2.getCurrentExecutionAttempt().getState().isTerminal()); - - // make sure all slots are recycled - assertEquals(parallelism, slotProvider.getNumberOfAvailableSlots()); - } - - /** - * Tests that a local failover does not try to trump a global failover. - */ - @Test - public void testGlobalRecoveryConcurrentToLocalRecovery() throws Exception { - - // the logic in this test is as follows: - // - start a job - // - cause a task failure and delay the local recovery action via the manual executor - // - cause a global failure that is recovering immediately - // - resume in local recovery action - // - validate that this does in fact not cause another task restart, because the global - // recovery should already have restarted the task graph - - final JobID jid = new JobID(); - final int parallelism = 2; - - final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism); - - final ExecutionGraph graph = createSampleGraph( - jid, - TestRestartPipelinedRegionStrategy::new, - new TestRestartStrategy(2, false), // twice restart, no delay - slotProvider, - parallelism); - - TestRestartPipelinedRegionStrategy strategy = (TestRestartPipelinedRegionStrategy) graph.getFailoverStrategy(); - - // This future is used to block the failover strategy execution until we complete it - CompletableFuture blocker = new CompletableFuture<>(); - strategy.setBlockerFuture(blocker); - - graph.start(mainThreadExecutor); - - final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next(); - final ExecutionVertex vertex1 = ejv.getTaskVertices()[0]; - final ExecutionVertex vertex2 = ejv.getTaskVertices()[1]; - - graph.scheduleForExecution(); - assertEquals(JobStatus.RUNNING, graph.getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex1).getState()); - - // let one of the vertices fail - that triggers a local recovery action - vertex2.getCurrentExecutionAttempt().fail(new Exception("test failure")); - assertEquals(ExecutionState.FAILED, vertex2.getCurrentExecutionAttempt().getState()); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(vertex2).getState()); - - // graph should still be running and the failover recovery action should be queued - assertEquals(JobStatus.RUNNING, graph.getState()); - - // now cancel the job - graph.failGlobal(new Exception("test exception")); - - assertEquals(JobStatus.FAILING, graph.getState()); - assertEquals(ExecutionState.FAILED, vertex2.getCurrentExecutionAttempt().getState()); - assertEquals(ExecutionState.CANCELING, vertex1.getCurrentExecutionAttempt().getState()); - - // now report that cancelling is complete for the other vertex - vertex1.getCurrentExecutionAttempt().completeCancelling(); - - waitUntilJobStatus(graph, JobStatus.RUNNING, 1000); - assertEquals(JobStatus.RUNNING, graph.getState()); - - waitUntilExecutionState(vertex1.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 1000); - waitUntilExecutionState(vertex2.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 1000); - vertex1.getCurrentExecutionAttempt().switchToRunning(); - vertex2.getCurrentExecutionAttempt().switchToRunning(); - assertEquals(ExecutionState.RUNNING, vertex1.getCurrentExecutionAttempt().getState()); - assertEquals(ExecutionState.RUNNING, vertex2.getCurrentExecutionAttempt().getState()); - - // let the recovery action continue - this should do nothing any more - blocker.complete(null); - - // validate that the graph is still peachy - assertEquals(JobStatus.RUNNING, graph.getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex1).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex2).getState()); - assertEquals(ExecutionState.RUNNING, vertex1.getCurrentExecutionAttempt().getState()); - assertEquals(ExecutionState.RUNNING, vertex2.getCurrentExecutionAttempt().getState()); - assertEquals(1, vertex1.getCurrentExecutionAttempt().getAttemptNumber()); - assertEquals(1, vertex2.getCurrentExecutionAttempt().getAttemptNumber()); - assertEquals(1, vertex1.getCopyOfPriorExecutionsList().size()); - assertEquals(1, vertex2.getCopyOfPriorExecutionsList().size()); - - // make sure all slots are in use - assertEquals(0, slotProvider.getNumberOfAvailableSlots()); - - blocker = new CompletableFuture<>(); - strategy.setBlockerFuture(blocker); - - // validate that a task failure then can be handled by the local recovery - vertex2.getCurrentExecutionAttempt().fail(new Exception("test failure")); - - // let the local recovery action continue - this should recover the vertex2 - blocker.complete(null); - - waitUntilExecutionState(vertex2.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 1000); - vertex2.getCurrentExecutionAttempt().switchToRunning(); - - // validate that the local recovery result - assertEquals(JobStatus.RUNNING, graph.getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex1).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(vertex2).getState()); - assertEquals(ExecutionState.RUNNING, vertex1.getCurrentExecutionAttempt().getState()); - assertEquals(ExecutionState.RUNNING, vertex2.getCurrentExecutionAttempt().getState()); - assertEquals(1, vertex1.getCurrentExecutionAttempt().getAttemptNumber()); - assertEquals(2, vertex2.getCurrentExecutionAttempt().getAttemptNumber()); - assertEquals(1, vertex1.getCopyOfPriorExecutionsList().size()); - assertEquals(2, vertex2.getCopyOfPriorExecutionsList().size()); - - // make sure all slots are in use - assertEquals(0, slotProvider.getNumberOfAvailableSlots()); - } - - /** - * Tests that a local failure fails all pending checkpoints which have not been acknowledged by the failing - * task. - */ - @Test - public void testLocalFailureFailsPendingCheckpoints() throws Exception { - final JobID jid = new JobID(); - final int parallelism = 2; - final long verifyTimeout = 5000L; - - final TaskManagerGateway taskManagerGateway = mock(TaskManagerGateway.class); - when(taskManagerGateway.submitTask(any(TaskDeploymentDescriptor.class), any(Time.class))).thenReturn(CompletableFuture.completedFuture(Acknowledge.get())); - when(taskManagerGateway.cancelTask(any(ExecutionAttemptID.class), any(Time.class))).thenReturn(CompletableFuture.completedFuture(Acknowledge.get())); - - final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism, taskManagerGateway); - - final CheckpointCoordinatorConfiguration checkpointCoordinatorConfiguration = new CheckpointCoordinatorConfiguration( - 10L, - 100000L, - 1L, - 3, - CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, - true, - false, - 0); - - final ExecutionGraph graph = createSampleGraph( - jid, - (eg) -> new RestartIndividualStrategy(eg) { - @Override - protected void performExecutionVertexRestart( - ExecutionVertex vertexToRecover, - long globalModVersion) { - } - } - , - new TestRestartStrategy(2, false), // twice restart, no delay - slotProvider, - parallelism); - - graph.start(mainThreadExecutor); - - final List allVertices = new ArrayList<>(graph.getAllVertices().values()); - - final StandaloneCheckpointIDCounter standaloneCheckpointIDCounter = new StandaloneCheckpointIDCounter(); - - graph.enableCheckpointing( - checkpointCoordinatorConfiguration, - allVertices, - allVertices, - allVertices, - Collections.emptyList(), - standaloneCheckpointIDCounter, - new StandaloneCompletedCheckpointStore(1), - new MemoryStateBackend(), - new CheckpointStatsTracker( - 1, - allVertices, - checkpointCoordinatorConfiguration, - UnregisteredMetricGroups.createUnregisteredTaskMetricGroup())); - - final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator(); - - final ExecutionJobVertex ejv = graph.getVerticesTopologically().iterator().next(); - final ExecutionVertex vertex1 = ejv.getTaskVertices()[0]; - final ExecutionVertex vertex2 = ejv.getTaskVertices()[1]; - - graph.scheduleForExecution(); - assertEquals(JobStatus.RUNNING, graph.getState()); - - verify(taskManagerGateway, timeout(verifyTimeout).times(parallelism)).submitTask(any(TaskDeploymentDescriptor.class), any(Time.class)); - - // switch all executions to running - for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) { - executionVertex.getCurrentExecutionAttempt().switchToRunning(); - } - - // wait for a first checkpoint to be triggered - verify(taskManagerGateway, timeout(verifyTimeout).times(3)).triggerCheckpoint( - eq(vertex1.getCurrentExecutionAttempt().getAttemptId()), - any(JobID.class), - anyLong(), - anyLong(), - any(CheckpointOptions.class), - any(Boolean.class)); - - verify(taskManagerGateway, timeout(verifyTimeout).times(3)).triggerCheckpoint( - eq(vertex2.getCurrentExecutionAttempt().getAttemptId()), - any(JobID.class), - anyLong(), - anyLong(), - any(CheckpointOptions.class), - any(Boolean.class)); - - assertEquals(3, checkpointCoordinator.getNumberOfPendingCheckpoints()); - - long checkpointToAcknowledge = standaloneCheckpointIDCounter.getLast(); - - checkpointCoordinator.receiveAcknowledgeMessage( - new AcknowledgeCheckpoint( - graph.getJobID(), - vertex1.getCurrentExecutionAttempt().getAttemptId(), - checkpointToAcknowledge), - "Unknown location"); - - Map oldPendingCheckpoints = new HashMap<>(3); - - for (PendingCheckpoint pendingCheckpoint : checkpointCoordinator.getPendingCheckpoints().values()) { - assertFalse(pendingCheckpoint.isDiscarded()); - oldPendingCheckpoints.put(pendingCheckpoint.getCheckpointId(), pendingCheckpoint); - } - - // let one of the vertices fail - this should trigger the failing of not acknowledged pending checkpoints - vertex1.getCurrentExecutionAttempt().fail(new Exception("test failure")); - - for (PendingCheckpoint pendingCheckpoint : oldPendingCheckpoints.values()) { - if (pendingCheckpoint.getCheckpointId() == checkpointToAcknowledge) { - assertFalse(pendingCheckpoint.isDiscarded()); - } else { - assertTrue(pendingCheckpoint.isDiscarded()); - } - } - } - - // ------------------------------------------------------------------------ - // utilities - // ------------------------------------------------------------------------ - - private ExecutionGraph createSampleGraph( - JobID jid, - Factory failoverStrategy, - RestartStrategy restartStrategy, - SlotProvider slotProvider, - int parallelism) throws Exception { - - final JobInformation jobInformation = new DummyJobInformation( - jid, - "test job"); - - // build a simple execution graph with on job vertex, parallelism 2 - final Time timeout = Time.seconds(10L); - final ExecutionGraph graph = new ExecutionGraph( - jobInformation, - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - timeout, - restartStrategy, - failoverStrategy, - slotProvider, - getClass().getClassLoader(), - VoidBlobWriter.getInstance(), - timeout); - - JobVertex jv = new JobVertex("test vertex"); - jv.setInvokableClass(NoOpInvokable.class); - jv.setParallelism(parallelism); - - JobGraph jg = new JobGraph(jid, "testjob", jv); - graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources()); - - return graph; - } - - /** - * Test implementation of the {@link RestartPipelinedRegionStrategy} that makes it possible to control when the - * failover action is performed via {@link CompletableFuture}. - */ - static class TestRestartPipelinedRegionStrategy extends RestartPipelinedRegionStrategy { - - @Nonnull - CompletableFuture blockerFuture; - - public TestRestartPipelinedRegionStrategy(ExecutionGraph executionGraph) { - super(executionGraph); - this.blockerFuture = CompletableFuture.completedFuture(null); - } - - public void setBlockerFuture(@Nonnull CompletableFuture blockerFuture) { - this.blockerFuture = blockerFuture; - } - - @Override - protected FailoverRegion createFailoverRegion(ExecutionGraph eg, List connectedExecutions) { - Map tasks = initTasks(connectedExecutions); - return new FailoverRegion(eg, connectedExecutions, tasks) { - @Override - protected CompletableFuture createTerminationFutureOverAllConnectedVertexes() { - ArrayList> terminationAndBlocker = new ArrayList<>(2); - terminationAndBlocker.add(super.createTerminationFutureOverAllConnectedVertexes()); - terminationAndBlocker.add(blockerFuture); - return FutureUtils.waitForAll(terminationAndBlocker); - } - }; - } - } -} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java index 25d169c8fd2d5589be46152fe03eec635f489385..dbe2a0f29c0937e60451014130d9cc7033e1dbfe 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionGraphTestUtils.java @@ -33,7 +33,6 @@ import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory; import org.apache.flink.runtime.client.JobExecutionException; import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter; import org.apache.flink.runtime.execution.ExecutionState; -import org.apache.flink.runtime.executiongraph.failover.FailoverRegion; import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy; import org.apache.flink.runtime.executiongraph.failover.RestartAllStrategy; import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy; @@ -233,26 +232,6 @@ public class ExecutionGraphTestUtils { return (AccessExecution execution) -> execution.getState() == executionState; } - public static void waitUntilFailoverRegionState(FailoverRegion region, JobStatus status, long maxWaitMillis) - throws TimeoutException { - checkNotNull(region); - checkNotNull(status); - checkArgument(maxWaitMillis >= 0); - - // this is a poor implementation - we may want to improve it eventually - final long deadline = maxWaitMillis == 0 ? Long.MAX_VALUE : System.nanoTime() + (maxWaitMillis * 1_000_000); - - while (region.getState() != status && System.nanoTime() < deadline) { - try { - Thread.sleep(2); - } catch (InterruptedException ignored) {} - } - - if (System.nanoTime() >= deadline) { - throw new TimeoutException(); - } - } - /** * Takes all vertices in the given ExecutionGraph and switches their current * execution to RUNNING. diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexDeploymentTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexDeploymentTest.java index 6093000cbe2e0ab79e37ddf519b9483721fa89c1..f0a4524e4a0c53496e730fbbc923e8cb4d5c472a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexDeploymentTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ExecutionVertexDeploymentTest.java @@ -314,7 +314,7 @@ public class ExecutionVertexDeploymentTest extends TestLogger { new AllocationID(), 0, null, - Execution.registerProducedPartitions(vertex, location, attemptID).get().values()); + Execution.registerProducedPartitions(vertex, location, attemptID, scheduleMode.allowLazyDeployment()).get().values()); Collection producedPartitions = tdd.getProducedPartitions(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FailoverRegionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FailoverRegionTest.java deleted file mode 100644 index ba4226b562c8d006b5df4e36e21b4ece75e4b519..0000000000000000000000000000000000000000 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/FailoverRegionTest.java +++ /dev/null @@ -1,694 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.executiongraph; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.api.common.time.Time; -import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; -import org.apache.flink.mock.Whitebox; -import org.apache.flink.runtime.JobException; -import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.checkpoint.CheckpointCoordinator; -import org.apache.flink.runtime.checkpoint.CheckpointCoordinatorTest; -import org.apache.flink.runtime.checkpoint.CheckpointException; -import org.apache.flink.runtime.checkpoint.CheckpointMetrics; -import org.apache.flink.runtime.checkpoint.CheckpointProperties; -import org.apache.flink.runtime.checkpoint.CheckpointRetentionPolicy; -import org.apache.flink.runtime.checkpoint.CheckpointStatsTracker; -import org.apache.flink.runtime.checkpoint.JobManagerTaskRestore; -import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; -import org.apache.flink.runtime.checkpoint.PendingCheckpoint; -import org.apache.flink.runtime.checkpoint.StandaloneCheckpointIDCounter; -import org.apache.flink.runtime.checkpoint.StandaloneCompletedCheckpointStore; -import org.apache.flink.runtime.checkpoint.TaskStateSnapshot; -import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter; -import org.apache.flink.runtime.deployment.TaskDeploymentDescriptor; -import org.apache.flink.runtime.execution.ExecutionState; -import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy; -import org.apache.flink.runtime.executiongraph.failover.FailoverStrategy.Factory; -import org.apache.flink.runtime.executiongraph.failover.RestartPipelinedRegionStrategy; -import org.apache.flink.runtime.executiongraph.restart.InfiniteDelayRestartStrategy; -import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy; -import org.apache.flink.runtime.executiongraph.restart.RestartStrategy; -import org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway; -import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider; -import org.apache.flink.runtime.io.network.partition.ResultPartitionType; -import org.apache.flink.runtime.jobgraph.DistributionPattern; -import org.apache.flink.runtime.jobgraph.JobStatus; -import org.apache.flink.runtime.jobgraph.JobVertex; -import org.apache.flink.runtime.jobgraph.JobVertexID; -import org.apache.flink.runtime.jobgraph.OperatorID; -import org.apache.flink.runtime.jobgraph.ScheduleMode; -import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; -import org.apache.flink.runtime.jobgraph.tasks.CheckpointCoordinatorConfiguration; -import org.apache.flink.runtime.jobmanager.scheduler.LocationPreferenceConstraint; -import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; -import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.messages.checkpoint.AcknowledgeCheckpoint; -import org.apache.flink.runtime.state.CheckpointStorageCoordinatorView; -import org.apache.flink.runtime.state.OperatorStateHandle; -import org.apache.flink.runtime.state.memory.MemoryStateBackend; -import org.apache.flink.runtime.testingUtils.TestingUtils; -import org.apache.flink.util.FlinkException; -import org.apache.flink.util.TestLogger; - -import org.junit.Ignore; -import org.junit.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; - -import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilExecutionState; -import static org.apache.flink.runtime.executiongraph.ExecutionGraphTestUtils.waitUntilFailoverRegionState; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; - -public class FailoverRegionTest extends TestLogger { - - private static final long checkpointId = 42L; - - /** - * Tests that a job only has one failover region and can recover from task failure successfully with state. - * @throws Exception if fail to create the single region execution graph or fail to acknowledge all checkpoints. - */ - @Test - public void testSingleRegionFailover() throws Exception { - RestartStrategy restartStrategy = new InfiniteDelayRestartStrategy(10); - ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy); - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - - ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next(); - - assertNotNull(eg.getCheckpointCoordinator()); - assertFalse(eg.getCheckpointCoordinator().getPendingCheckpoints().isEmpty()); - - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev).getState()); - - acknowledgeAllCheckpoints(eg.getCheckpointCoordinator(), eg.getAllExecutionVertices().iterator()); - - // verify checkpoint has been completed successfully. - assertEquals(1, eg.getCheckpointCoordinator().getCheckpointStore().getNumberOfRetainedCheckpoints()); - assertEquals(checkpointId, eg.getCheckpointCoordinator().getCheckpointStore().getLatestCheckpoint(false).getCheckpointID()); - - ev.getCurrentExecutionAttempt().fail(new Exception("Test Exception")); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev).getState()); - - for (ExecutionVertex evs : eg.getAllExecutionVertices()) { - evs.getCurrentExecutionAttempt().completeCancelling(); - } - - verifyCheckpointRestoredAsExpected(eg); - - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev).getState()); - } - - /** - * Tests that a job has server failover regions and one region failover does not influence others - * - * 

    -	 *     (a1) ---> (b1) -+-> (c1) ---+-> (d1) 
-	 *                     X          /
-	 *     (a2) ---> (b2) -+-> (c2) -+
-	 *
-	 *           ^         ^         ^
-	 *           |         |         |
-	 *     (pipelined) (blocking) (pipelined)
-	 *
-	 *
    - */ - @Test - public void testMultiRegionsFailover() throws Exception { - final JobID jobId = new JobID(); - final String jobName = "Test Job Sample Name"; - - final Map attemptIDInitStateMap = new HashMap<>(); - final SlotProvider slotProvider = new SimpleSlotProvider(jobId, 20, new CollectTddTaskManagerGateway(attemptIDInitStateMap)); - - JobVertex v1 = new JobVertex("vertex1"); - JobVertex v2 = new JobVertex("vertex2"); - JobVertex v3 = new JobVertex("vertex3"); - JobVertex v4 = new JobVertex("vertex4"); - - v1.setParallelism(2); - v2.setParallelism(2); - v3.setParallelism(2); - v4.setParallelism(1); - - v1.setInvokableClass(AbstractInvokable.class); - v2.setInvokableClass(AbstractInvokable.class); - v3.setInvokableClass(AbstractInvokable.class); - v4.setInvokableClass(AbstractInvokable.class); - - v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); - v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - List ordered = Arrays.asList(v1, v2, v3, v4); - - final ExecutionGraph eg = new ExecutionGraphTestUtils.TestingExecutionGraphBuilder(jobId, jobName, v1, v2, v3, v4) - .setRestartStrategy(new InfiniteDelayRestartStrategy(10)) - .setFailoverStrategyFactory(new FailoverPipelinedRegionWithDirectExecutor()) - .setSlotProvider(slotProvider) - .allowQueuedScheduling() - .build(); - - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - - // the following two vertices are in the same failover region - ExecutionVertex ev11 = eg.getJobVertex(v1.getID()).getTaskVertices()[0]; - ExecutionVertex ev21 = eg.getJobVertex(v2.getID()).getTaskVertices()[0]; - - // the following two vertices are in the same failover region - ExecutionVertex ev12 = eg.getJobVertex(v1.getID()).getTaskVertices()[1]; - ExecutionVertex ev22 = eg.getJobVertex(v2.getID()).getTaskVertices()[1]; - - // the following vertices are in one failover region - ExecutionVertex ev31 = eg.getJobVertex(v3.getID()).getTaskVertices()[0]; - ExecutionVertex ev32 = eg.getJobVertex(v3.getID()).getTaskVertices()[1]; - ExecutionVertex ev4 = eg.getJobVertex(v4.getID()).getTaskVertices()[0]; - - enableCheckpointing(eg); - - eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread()); - eg.scheduleForExecution(); - assertEquals(JobStatus.RUNNING, eg.getState()); - - attachPendingCheckpoints(eg); - - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev21).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState()); - - acknowledgeAllCheckpoints(eg.getCheckpointCoordinator(), Arrays.asList(ev11, ev21, ev12, ev22, ev31, ev32, ev4).iterator()); - - ev21.scheduleForExecution(eg.getSlotProviderStrategy(), LocationPreferenceConstraint.ALL, Collections.emptySet()); - ev21.getCurrentExecutionAttempt().fail(new Exception("New fail")); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev11).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState()); - - ev11.getCurrentExecutionAttempt().completeCancelling(); - verifyCheckpointRestoredAsExpected(eg); - - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState()); - - ev11.getCurrentExecutionAttempt().markFinished(); - ev21.getCurrentExecutionAttempt().markFinished(); - ev22.scheduleForExecution(eg.getSlotProviderStrategy(), LocationPreferenceConstraint.ALL, Collections.emptySet()); - ev22.getCurrentExecutionAttempt().markFinished(); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState()); - - waitUntilExecutionState(ev31.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 2000); - waitUntilExecutionState(ev32.getCurrentExecutionAttempt(), ExecutionState.DEPLOYING, 2000); - - ev31.getCurrentExecutionAttempt().fail(new Exception("New fail")); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState()); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev31).getState()); - - ev32.getCurrentExecutionAttempt().completeCancelling(); - verifyCheckpointRestoredAsExpected(eg); - - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev22).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState()); - - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev4).getState()); - } - - /** - * Tests that when a task fail, and restart strategy doesn't support restarting, the job will go to failed. - * @throws Exception if fail to create the single region execution graph. - */ - @Test - public void testNoManualRestart() throws Exception { - NoRestartStrategy restartStrategy = new NoRestartStrategy(); - ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy); - - ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next(); - - ev.fail(new Exception("Test Exception")); - - for (ExecutionVertex evs : eg.getAllExecutionVertices()) { - evs.getCurrentExecutionAttempt().completeCancelling(); - } - assertEquals(JobStatus.FAILED, eg.getState()); - } - - /** - * Tests that two regions failover at the same time, they will not influence each other. - * @throws Exception if fail to create dummy job information, fail to schedule for execution - * or timeout before region switches to expected status. - */ - @Test - public void testMultiRegionFailoverAtSameTime() throws Exception { - final JobID jobId = new JobID(); - final String jobName = "Test Job Sample Name"; - - final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jobId, 16); - - JobVertex v1 = new JobVertex("vertex1"); - JobVertex v2 = new JobVertex("vertex2"); - JobVertex v3 = new JobVertex("vertex3"); - JobVertex v4 = new JobVertex("vertex4"); - - v1.setParallelism(2); - v2.setParallelism(2); - v3.setParallelism(2); - v4.setParallelism(2); - - v1.setInvokableClass(AbstractInvokable.class); - v2.setInvokableClass(AbstractInvokable.class); - v3.setInvokableClass(AbstractInvokable.class); - v4.setInvokableClass(AbstractInvokable.class); - - v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - List ordered = Arrays.asList(v1, v2, v3, v4); - - ExecutionGraph eg = new ExecutionGraph( - new DummyJobInformation( - jobId, - jobName), - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - AkkaUtils.getDefaultTimeout(), - new InfiniteDelayRestartStrategy(10), - new RestartPipelinedRegionStrategy.Factory(), - slotProvider); - try { - eg.attachJobGraph(ordered); - } - catch (JobException e) { - e.printStackTrace(); - fail("Job failed with exception: " + e.getMessage()); - } - eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread()); - eg.scheduleForExecution(); - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy(); - - ExecutionVertex ev11 = eg.getJobVertex(v1.getID()).getTaskVertices()[0]; - ExecutionVertex ev12 = eg.getJobVertex(v1.getID()).getTaskVertices()[1]; - ExecutionVertex ev31 = eg.getJobVertex(v3.getID()).getTaskVertices()[0]; - ExecutionVertex ev32 = eg.getJobVertex(v3.getID()).getTaskVertices()[1]; - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState()); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState()); - - ev11.getCurrentExecutionAttempt().fail(new Exception("new fail")); - ev31.getCurrentExecutionAttempt().fail(new Exception("new fail")); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev11).getState()); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev31).getState()); - - ev32.getCurrentExecutionAttempt().completeCancelling(); - waitUntilFailoverRegionState(strategy.getFailoverRegion(ev31), JobStatus.RUNNING, 1000); - - ev12.getCurrentExecutionAttempt().completeCancelling(); - waitUntilFailoverRegionState(strategy.getFailoverRegion(ev11), JobStatus.RUNNING, 1000); - } - - /** - * Tests that if a task reports the result of its preceding task is failed, - * its preceding task will be considered as failed, and start to failover - * TODO: as the report part is not finished yet, this case is ignored temporarily - * @throws Exception if fail to create dummy job information or fail to schedule for execution. - */ - @Ignore - @Test - public void testSucceedingNoticePreceding() throws Exception { - final JobID jobId = new JobID(); - final String jobName = "Test Job Sample Name"; - - final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jobId, 14); - - JobVertex v1 = new JobVertex("vertex1"); - JobVertex v2 = new JobVertex("vertex2"); - - v1.setParallelism(1); - v2.setParallelism(1); - - v1.setInvokableClass(AbstractInvokable.class); - v2.setInvokableClass(AbstractInvokable.class); - - v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - - ExecutionGraph eg = new ExecutionGraphTestUtils.TestingExecutionGraphBuilder(jobId, jobName, v1, v2) - .setRestartStrategy(new InfiniteDelayRestartStrategy(10)) - .setFailoverStrategyFactory(new FailoverPipelinedRegionWithDirectExecutor()) - .setSlotProvider(slotProvider) - .setScheduleMode(ScheduleMode.EAGER) - .build(); - - eg.scheduleForExecution(); - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy(); - - ExecutionVertex ev11 = eg.getJobVertex(v2.getID()).getTaskVertices()[0]; - ExecutionVertex ev21 = eg.getJobVertex(v2.getID()).getTaskVertices()[0]; - ev21.getCurrentExecutionAttempt().fail(new Exception("Fail with v1")); - - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev21).getState()); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev11).getState()); - } - - /** - * Tests that a new failure comes while the failover region is in CANCELLING. - * @throws Exception if fail to create the single region execution graph. - */ - @Test - public void testFailWhileCancelling() throws Exception { - RestartStrategy restartStrategy = new InfiniteDelayRestartStrategy(); - ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy); - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy(); - - Iterator iter = eg.getAllExecutionVertices().iterator(); - ExecutionVertex ev1 = iter.next(); - ev1.getCurrentExecutionAttempt().switchToRunning(); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev1).getState()); - - ev1.getCurrentExecutionAttempt().fail(new Exception("new fail")); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev1).getState()); - - ExecutionVertex ev2 = iter.next(); - ev2.getCurrentExecutionAttempt().fail(new Exception("new fail")); - assertEquals(JobStatus.RUNNING, eg.getState()); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev1).getState()); - } - - /** - * Tests that a new failure comes while the failover region is restarting. - * @throws Exception if fail to create the single region execution graph. - */ - @Test - public void testFailWhileRestarting() throws Exception { - RestartStrategy restartStrategy = new InfiniteDelayRestartStrategy(); - ExecutionGraph eg = createSingleRegionExecutionGraph(restartStrategy); - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy(); - - Iterator iter = eg.getAllExecutionVertices().iterator(); - ExecutionVertex ev1 = iter.next(); - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev1).getState()); - - ev1.getCurrentExecutionAttempt().fail(new Exception("new fail")); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev1).getState()); - - for (ExecutionVertex evs : eg.getAllExecutionVertices()) { - evs.getCurrentExecutionAttempt().completeCancelling(); - } - assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev1).getState()); - - ev1.getCurrentExecutionAttempt().fail(new Exception("new fail")); - assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev1).getState()); - } - - @Test - public void testStatusResettingOnRegionFailover() throws Exception { - final JobID jobId = new JobID(); - final String jobName = "Test Job Sample Name"; - - final SlotProvider slotProvider = new SimpleSlotProvider(jobId, 20); - - JobVertex v1 = new JobVertex("vertex1"); - JobVertex v2 = new JobVertex("vertex2"); - - v1.setParallelism(2); - v2.setParallelism(2); - - v1.setInvokableClass(AbstractInvokable.class); - v2.setInvokableClass(AbstractInvokable.class); - - v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - - List ordered = Arrays.asList(v1, v2); - - ExecutionGraph eg = new ExecutionGraph( - new DummyJobInformation( - jobId, - jobName), - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - AkkaUtils.getDefaultTimeout(), - new InfiniteDelayRestartStrategy(10), - new FailoverPipelinedRegionWithDirectExecutor(), - slotProvider); - - eg.attachJobGraph(ordered); - eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread()); - - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy(); - - ExecutionVertex ev11 = eg.getJobVertex(v1.getID()).getTaskVertices()[0]; - ExecutionVertex ev12 = eg.getJobVertex(v1.getID()).getTaskVertices()[1]; - ExecutionVertex ev21 = eg.getJobVertex(v2.getID()).getTaskVertices()[0]; - ExecutionVertex ev22 = eg.getJobVertex(v2.getID()).getTaskVertices()[1]; - - eg.scheduleForExecution(); - - // initial state - assertEquals(ExecutionState.DEPLOYING, ev11.getExecutionState()); - assertEquals(ExecutionState.DEPLOYING, ev12.getExecutionState()); - assertEquals(ExecutionState.CREATED, ev21.getExecutionState()); - assertEquals(ExecutionState.CREATED, ev22.getExecutionState()); - assertFalse(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].areAllPartitionsFinished()); - assertFalse(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].getPartitions()[0].isConsumable()); - assertFalse(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].getPartitions()[1].isConsumable()); - - // partitions all finished - ev11.getCurrentExecutionAttempt().markFinished(); - ev12.getCurrentExecutionAttempt().markFinished(); - assertEquals(ExecutionState.FINISHED, ev11.getExecutionState()); - assertEquals(ExecutionState.FINISHED, ev12.getExecutionState()); - assertEquals(ExecutionState.DEPLOYING, ev21.getExecutionState()); - assertEquals(ExecutionState.DEPLOYING, ev22.getExecutionState()); - assertTrue(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].areAllPartitionsFinished()); - assertTrue(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].getPartitions()[0].isConsumable()); - assertTrue(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].getPartitions()[1].isConsumable()); - - // force the partition producer to restart - strategy.onTaskFailure(ev11.getCurrentExecutionAttempt(), new FlinkException("Fail for testing")); - assertFalse(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].areAllPartitionsFinished()); - assertFalse(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].getPartitions()[0].isConsumable()); - assertFalse(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].getPartitions()[1].isConsumable()); - - // failed partition finishes again - ev11.getCurrentExecutionAttempt().markFinished(); - assertTrue(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].areAllPartitionsFinished()); - assertTrue(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].getPartitions()[0].isConsumable()); - assertTrue(eg.getJobVertex(v1.getID()).getProducedDataSets()[0].getPartitions()[1].isConsumable()); - } - - // -------------------------------------------------------------------------------------------- - - private void verifyCheckpointRestoredAsExpected(ExecutionGraph eg) throws Exception { - // pending checkpoints have already been cancelled. - assertNotNull(eg.getCheckpointCoordinator()); - assertTrue(eg.getCheckpointCoordinator().getPendingCheckpoints().isEmpty()); - - // verify checkpoint has been restored successfully. - assertEquals(1, eg.getCheckpointCoordinator().getCheckpointStore().getNumberOfRetainedCheckpoints()); - assertEquals(checkpointId, eg.getCheckpointCoordinator().getCheckpointStore().getLatestCheckpoint(false).getCheckpointID()); - } - - private ExecutionGraph createSingleRegionExecutionGraph(RestartStrategy restartStrategy) throws Exception { - final JobID jobId = new JobID(); - final String jobName = "Test Job Sample Name"; - - final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jobId, 14); - - JobVertex v1 = new JobVertex("vertex1"); - JobVertex v2 = new JobVertex("vertex2"); - JobVertex v3 = new JobVertex("vertex3"); - - v1.setParallelism(3); - v2.setParallelism(2); - v3.setParallelism(2); - - v1.setInvokableClass(AbstractInvokable.class); - v2.setInvokableClass(AbstractInvokable.class); - v3.setInvokableClass(AbstractInvokable.class); - - v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v3.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v3.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - List ordered = new ArrayList<>(Arrays.asList(v1, v2, v3)); - - ExecutionGraph eg = new ExecutionGraph( - new DummyJobInformation( - jobId, - jobName), - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - AkkaUtils.getDefaultTimeout(), - restartStrategy, - new FailoverPipelinedRegionWithDirectExecutor(), - slotProvider); - try { - eg.attachJobGraph(ordered); - } - catch (JobException e) { - e.printStackTrace(); - fail("Job failed with exception: " + e.getMessage()); - } - - enableCheckpointing(eg); - - eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread()); - eg.scheduleForExecution(); - - attachPendingCheckpoints(eg); - return eg; - } - - // ------------------------------------------------------------------------ - - /** - * A factory to create a RestartPipelinedRegionStrategy that uses a - * direct (synchronous) executor for easier testing. - */ - private static class FailoverPipelinedRegionWithDirectExecutor implements Factory { - - @Override - public FailoverStrategy create(ExecutionGraph executionGraph) { - return new RestartPipelinedRegionStrategy(executionGraph); - } - } - - private static void enableCheckpointing(ExecutionGraph eg) { - ArrayList jobVertices = new ArrayList<>(eg.getAllVertices().values()); - CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration( - 1000, - 100, - 0, - 1, - CheckpointRetentionPolicy.RETAIN_ON_CANCELLATION, - true, - false, - 0); - eg.enableCheckpointing( - chkConfig, - jobVertices, - jobVertices, - jobVertices, - Collections.emptyList(), - new StandaloneCheckpointIDCounter(), - new StandaloneCompletedCheckpointStore(1), - new MemoryStateBackend(), - new CheckpointStatsTracker( - 0, - jobVertices, - mock(CheckpointCoordinatorConfiguration.class), - new UnregisteredMetricsGroup())); - } - - /** - * Attach pending checkpoints of chk-42 and chk-43 to the execution graph. - * If {@link #acknowledgeAllCheckpoints(CheckpointCoordinator, Iterator)} called then, - * chk-42 would become the completed checkpoint. - */ - private void attachPendingCheckpoints(ExecutionGraph eg) throws IOException { - final Map pendingCheckpoints = new HashMap<>(); - final Map verticesToConfirm = new HashMap<>(); - eg.getAllExecutionVertices().forEach(e -> { - Execution ee = e.getCurrentExecutionAttempt(); - if (ee != null) { - verticesToConfirm.put(ee.getAttemptId(), e); - } - }); - - CheckpointCoordinator checkpointCoordinator = eg.getCheckpointCoordinator(); - assertNotNull(checkpointCoordinator); - CheckpointStorageCoordinatorView checkpointStorage = checkpointCoordinator.getCheckpointStorage(); - pendingCheckpoints.put(checkpointId, new PendingCheckpoint( - eg.getJobID(), - checkpointId, - 0L, - verticesToConfirm, - CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), - checkpointStorage.initializeLocationForCheckpoint(checkpointId), - eg.getFutureExecutor())); - - long newCheckpointId = checkpointId + 1; - pendingCheckpoints.put(newCheckpointId, new PendingCheckpoint( - eg.getJobID(), - newCheckpointId, - 0L, - verticesToConfirm, - CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.RETAIN_ON_FAILURE), - checkpointStorage.initializeLocationForCheckpoint(newCheckpointId), - eg.getFutureExecutor())); - Whitebox.setInternalState(checkpointCoordinator, "pendingCheckpoints", pendingCheckpoints); - } - - /** - * Let the checkpoint coordinator to receive all acknowledges from given executionVertexes so that to complete the expected checkpoint. - */ - private void acknowledgeAllCheckpoints(CheckpointCoordinator checkpointCoordinator, Iterator executionVertexes) throws IOException, CheckpointException { - while (executionVertexes.hasNext()) { - ExecutionVertex executionVertex = executionVertexes.next(); - for (int index = 0; index < executionVertex.getJobVertex().getParallelism(); index++) { - JobVertexID jobVertexID = executionVertex.getJobvertexId(); - OperatorStateHandle opStateBackend = CheckpointCoordinatorTest.generatePartitionableStateHandle(jobVertexID, index, 2, 8, false); - OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState(opStateBackend, null, null, null); - TaskStateSnapshot taskOperatorSubtaskStates = new TaskStateSnapshot(); - taskOperatorSubtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), operatorSubtaskState); - - AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint( - executionVertex.getJobId(), - executionVertex.getJobVertex().getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(), - checkpointId, - new CheckpointMetrics(), - taskOperatorSubtaskStates); - - checkpointCoordinator.receiveAcknowledgeMessage(acknowledgeCheckpoint, "Unknown location"); - } - } - } - - private static class CollectTddTaskManagerGateway extends SimpleAckingTaskManagerGateway { - - private final Map attemptIDInitStateMap; - - CollectTddTaskManagerGateway(Map attemptIDInitStateMap) { - this.attemptIDInitStateMap = attemptIDInitStateMap; - } - - @Override - public CompletableFuture submitTask(TaskDeploymentDescriptor tdd, Time timeout) { - attemptIDInitStateMap.put(tdd.getExecutionAttemptId(), tdd.getTaskRestore()); - return super.submitTask(tdd, timeout); - } - } - -} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java deleted file mode 100644 index 7b00e4f09f1558ec72208a2088380c91ed2fcf4a..0000000000000000000000000000000000000000 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/RestartPipelinedRegionStrategyTest.java +++ /dev/null @@ -1,388 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.executiongraph; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.runtime.JobException; -import org.apache.flink.runtime.akka.AkkaUtils; -import org.apache.flink.runtime.blob.VoidBlobWriter; -import org.apache.flink.runtime.executiongraph.failover.FailoverRegion; -import org.apache.flink.runtime.executiongraph.failover.RestartPipelinedRegionStrategy; -import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy; -import org.apache.flink.runtime.io.network.partition.ResultPartitionType; -import org.apache.flink.runtime.jobgraph.DistributionPattern; -import org.apache.flink.runtime.jobgraph.JobVertex; -import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; -import org.apache.flink.runtime.testingUtils.TestingUtils; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CompletableFuture; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; - -public class RestartPipelinedRegionStrategyTest { - - /** - * Creates a JobGraph of the following form: - * - * 
    -	 *  v1--->v2-->\
-	 *              \
-	 *               v4 --->\
-	 *        ----->/        \
-	 *  v3-->/                v5
-	 *       \               /
-	 *        ------------->/
-	 *
    - */ - @Test - public void testSimpleFailoverRegion() throws Exception { - - final JobID jobId = new JobID(); - final String jobName = "Test Job Sample Name"; - - JobVertex v1 = new JobVertex("vertex1"); - JobVertex v2 = new JobVertex("vertex2"); - JobVertex v3 = new JobVertex("vertex3"); - JobVertex v4 = new JobVertex("vertex4"); - JobVertex v5 = new JobVertex("vertex5"); - - v1.setParallelism(5); - v2.setParallelism(7); - v3.setParallelism(2); - v4.setParallelism(11); - v5.setParallelism(4); - - v1.setInvokableClass(AbstractInvokable.class); - v2.setInvokableClass(AbstractInvokable.class); - v3.setInvokableClass(AbstractInvokable.class); - v4.setInvokableClass(AbstractInvokable.class); - v5.setInvokableClass(AbstractInvokable.class); - - v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - List ordered = new ArrayList(Arrays.asList(v1, v2, v3, v4, v5)); - - final JobInformation jobInformation = new DummyJobInformation( - jobId, - jobName); - - ExecutionGraph eg = new ExecutionGraph( - jobInformation, - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - AkkaUtils.getDefaultTimeout(), - new NoRestartStrategy(), - new RestartPipelinedRegionStrategy.Factory(), - new TestingSlotProvider(ignored -> new CompletableFuture<>()), - ExecutionGraph.class.getClassLoader(), - VoidBlobWriter.getInstance(), - AkkaUtils.getDefaultTimeout()); - try { - eg.attachJobGraph(ordered); - } - catch (JobException e) { - e.printStackTrace(); - fail("Job failed with exception: " + e.getMessage()); - } - - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy(); - ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID()); - ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID()); - ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID()); - ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID()); - ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID()); - FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[2]); - FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[3]); - FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]); - FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[4]); - FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]); - - assertEquals(region1, region2); - assertEquals(region3, region2); - assertEquals(region4, region2); - assertEquals(region5, region2); - } - - /** - * Creates a JobGraph of the following form: - * - * 
    -     *  v2 ------->\
-     *              \
-     *  v1---------> v4 --->|\
-     *                        \
-     *                        v5
-     *                       /
-     *  v3--------------->|/
-     *
    - */ - @Test - public void testMultipleFailoverRegions() throws Exception { - final JobID jobId = new JobID(); - final String jobName = "Test Job Sample Name"; - - JobVertex v1 = new JobVertex("vertex1"); - JobVertex v2 = new JobVertex("vertex2"); - JobVertex v3 = new JobVertex("vertex3"); - JobVertex v4 = new JobVertex("vertex4"); - JobVertex v5 = new JobVertex("vertex5"); - - v1.setParallelism(3); - v2.setParallelism(2); - v3.setParallelism(2); - v4.setParallelism(5); - v5.setParallelism(2); - - v1.setInvokableClass(AbstractInvokable.class); - v2.setInvokableClass(AbstractInvokable.class); - v3.setInvokableClass(AbstractInvokable.class); - v4.setInvokableClass(AbstractInvokable.class); - v5.setInvokableClass(AbstractInvokable.class); - - v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v4.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - - List ordered = new ArrayList(Arrays.asList(v1, v2, v3, v4, v5)); - - final JobInformation jobInformation = new DummyJobInformation( - jobId, - jobName); - - ExecutionGraph eg = new ExecutionGraph( - jobInformation, - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - AkkaUtils.getDefaultTimeout(), - new NoRestartStrategy(), - new RestartPipelinedRegionStrategy.Factory(), - new TestingSlotProvider(ignored -> new CompletableFuture<>()), - ExecutionGraph.class.getClassLoader(), - VoidBlobWriter.getInstance(), - AkkaUtils.getDefaultTimeout()); - try { - eg.attachJobGraph(ordered); - } - catch (JobException e) { - e.printStackTrace(); - fail("Job failed with exception: " + e.getMessage()); - } - - // All in one failover region - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy(); - ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID()); - ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID()); - ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID()); - ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID()); - ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID()); - FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]); - FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]); - FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]); - FailoverRegion region31 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]); - FailoverRegion region32 = strategy.getFailoverRegion(ejv3.getTaskVertices()[1]); - FailoverRegion region51 = strategy.getFailoverRegion(ejv5.getTaskVertices()[0]); - FailoverRegion region52 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]); - - //There should be 5 failover regions. v1 v2 v4 in one, v3 has two, v5 has two - assertEquals(region1, region2); - assertEquals(region2, region4); - assertFalse(region31.equals(region32)); - assertFalse(region51.equals(region52)); - } - - /** - * Creates a JobGraph of the following form: - * - * 
    -     *  v1--->v2-->\
-     *              \
-     *               v4 --->|\
-     *        ----->/        \
-     *  v3-->/                v5
-     *       \               /
-     *        ------------->/
-     *
    - */ - @Test - public void testSingleRegionWithMixedInput() throws Exception { - final JobID jobId = new JobID(); - final String jobName = "Test Job Sample Name"; - - JobVertex v1 = new JobVertex("vertex1"); - JobVertex v2 = new JobVertex("vertex2"); - JobVertex v3 = new JobVertex("vertex3"); - JobVertex v4 = new JobVertex("vertex4"); - JobVertex v5 = new JobVertex("vertex5"); - - v1.setParallelism(3); - v2.setParallelism(2); - v3.setParallelism(2); - v4.setParallelism(5); - v5.setParallelism(2); - - v1.setInvokableClass(AbstractInvokable.class); - v2.setInvokableClass(AbstractInvokable.class); - v3.setInvokableClass(AbstractInvokable.class); - v4.setInvokableClass(AbstractInvokable.class); - v5.setInvokableClass(AbstractInvokable.class); - - v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v5.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - v5.connectNewDataSetAsInput(v4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - - List ordered = new ArrayList(Arrays.asList(v1, v2, v3, v4, v5)); - - final JobInformation jobInformation = new DummyJobInformation( - jobId, - jobName); - - ExecutionGraph eg = new ExecutionGraph( - jobInformation, - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - AkkaUtils.getDefaultTimeout(), - new NoRestartStrategy(), - new RestartPipelinedRegionStrategy.Factory(), - new TestingSlotProvider(ignored -> new CompletableFuture<>()), - ExecutionGraph.class.getClassLoader(), - VoidBlobWriter.getInstance(), - AkkaUtils.getDefaultTimeout()); - try { - eg.attachJobGraph(ordered); - } - catch (JobException e) { - e.printStackTrace(); - fail("Job failed with exception: " + e.getMessage()); - } - - // All in one failover region - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy(); - ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID()); - ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID()); - ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID()); - ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID()); - ExecutionJobVertex ejv5 = eg.getJobVertex(v5.getID()); - FailoverRegion region1 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]); - FailoverRegion region2 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]); - FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]); - FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]); - FailoverRegion region5 = strategy.getFailoverRegion(ejv5.getTaskVertices()[1]); - - assertEquals(region1, region2); - assertEquals(region2, region4); - assertEquals(region3, region2); - assertEquals(region1, region5); - } - - /** - * Creates a JobGraph of the following form: - * - * 
    -     *  v1-->v2-->|\
-     *              \
-     *               v4
-     *             /
-     *  v3------>/
-     *
    - */ - @Test - public void testMultiRegionNotAllToAll() throws Exception { - final JobID jobId = new JobID(); - final String jobName = "Test Job Sample Name"; - - JobVertex v1 = new JobVertex("vertex1"); - JobVertex v2 = new JobVertex("vertex2"); - JobVertex v3 = new JobVertex("vertex3"); - JobVertex v4 = new JobVertex("vertex4"); - JobVertex v5 = new JobVertex("vertex5"); - - v1.setParallelism(2); - v2.setParallelism(2); - v3.setParallelism(5); - v4.setParallelism(5); - - v1.setInvokableClass(AbstractInvokable.class); - v2.setInvokableClass(AbstractInvokable.class); - v3.setInvokableClass(AbstractInvokable.class); - v4.setInvokableClass(AbstractInvokable.class); - - v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); - v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - - List ordered = new ArrayList(Arrays.asList(v1, v2, v3, v4)); - - final JobInformation jobInformation = new DummyJobInformation( - jobId, - jobName); - - ExecutionGraph eg = new ExecutionGraph( - jobInformation, - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - AkkaUtils.getDefaultTimeout(), - new NoRestartStrategy(), - new RestartPipelinedRegionStrategy.Factory(), - new TestingSlotProvider(ignored -> new CompletableFuture<>()), - ExecutionGraph.class.getClassLoader(), - VoidBlobWriter.getInstance(), - AkkaUtils.getDefaultTimeout()); - try { - eg.attachJobGraph(ordered); - } - catch (JobException e) { - e.printStackTrace(); - fail("Job failed with exception: " + e.getMessage()); - } - - // All in one failover region - RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy(); - ExecutionJobVertex ejv1 = eg.getJobVertex(v1.getID()); - ExecutionJobVertex ejv2 = eg.getJobVertex(v2.getID()); - ExecutionJobVertex ejv3 = eg.getJobVertex(v3.getID()); - ExecutionJobVertex ejv4 = eg.getJobVertex(v4.getID()); - FailoverRegion region11 = strategy.getFailoverRegion(ejv1.getTaskVertices()[0]); - FailoverRegion region12 = strategy.getFailoverRegion(ejv1.getTaskVertices()[1]); - FailoverRegion region21 = strategy.getFailoverRegion(ejv2.getTaskVertices()[0]); - FailoverRegion region22 = strategy.getFailoverRegion(ejv2.getTaskVertices()[1]); - FailoverRegion region3 = strategy.getFailoverRegion(ejv3.getTaskVertices()[0]); - FailoverRegion region4 = strategy.getFailoverRegion(ejv4.getTaskVertices()[3]); - - //There should be 3 failover regions. v1 v2 in two, v3 and v4 in one - assertEquals(region11, region21); - assertEquals(region12, region22); - assertFalse(region11.equals(region12)); - assertFalse(region3.equals(region4)); - } - -} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java deleted file mode 100644 index f17dddcfb0614c85fed4f57c758a9179745a66de..0000000000000000000000000000000000000000 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/PipelinedFailoverRegionBuildingTest.java +++ /dev/null @@ -1,651 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.executiongraph.failover; - -import org.apache.flink.api.common.time.Time; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.JobManagerOptions; -import org.apache.flink.metrics.groups.UnregisteredMetricsGroup; -import org.apache.flink.runtime.JobException; -import org.apache.flink.runtime.blob.VoidBlobWriter; -import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory; -import org.apache.flink.runtime.client.JobExecutionException; -import org.apache.flink.runtime.executiongraph.ExecutionGraph; -import org.apache.flink.runtime.executiongraph.ExecutionGraphBuilder; -import org.apache.flink.runtime.executiongraph.ExecutionVertex; -import org.apache.flink.runtime.executiongraph.restart.NoRestartStrategy; -import org.apache.flink.runtime.io.network.partition.NoOpPartitionTracker; -import org.apache.flink.runtime.jobmaster.slotpool.SlotProvider; -import org.apache.flink.runtime.io.network.partition.ResultPartitionType; -import org.apache.flink.runtime.jobgraph.DistributionPattern; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.jobgraph.JobVertex; -import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup; -import org.apache.flink.runtime.shuffle.NettyShuffleMaster; -import org.apache.flink.runtime.testingUtils.TestingUtils; -import org.apache.flink.runtime.testtasks.NoOpInvokable; -import org.apache.flink.util.TestLogger; - -import org.junit.Test; - -import java.util.Iterator; - -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -/** - * Tests that make sure that the building of pipelined connected failover regions works - * correctly. - */ -public class PipelinedFailoverRegionBuildingTest extends TestLogger { - - /** - * Tests that validates that a graph with single unconnected vertices works correctly. - * - * 
    -	 *     (v1)
-	 *     
-	 *     (v2)
-	 *     
-	 *     (v3)
-	 *     
-	 *     ...
-	 *
    - */ - @Test - public void testIndividualVertices() throws Exception { - final JobVertex source1 = new JobVertex("source1"); - source1.setInvokableClass(NoOpInvokable.class); - source1.setParallelism(2); - - final JobVertex source2 = new JobVertex("source2"); - source2.setInvokableClass(NoOpInvokable.class); - source2.setParallelism(2); - - final JobGraph jobGraph = new JobGraph("test job", source1, source2); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - FailoverRegion sourceRegion11 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source1.getID()).getTaskVertices()[0]); - FailoverRegion sourceRegion12 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source1.getID()).getTaskVertices()[1]); - FailoverRegion targetRegion21 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source2.getID()).getTaskVertices()[0]); - FailoverRegion targetRegion22 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source2.getID()).getTaskVertices()[1]); - - assertTrue(sourceRegion11 != sourceRegion12); - assertTrue(sourceRegion12 != targetRegion21); - assertTrue(targetRegion21 != targetRegion22); - } - - /** - * Tests that validates that embarrassingly parallel chains of vertices work correctly. - * - * 
    -	 *     (a1) --> (b1)
-	 *
-	 *     (a2) --> (b2)
-	 *
-	 *     (a3) --> (b3)
-	 *
-	 *     ...
-	 *
    - */ - @Test - public void testEmbarrassinglyParallelCase() throws Exception { - int parallelism = 10000; - final JobVertex vertex1 = new JobVertex("vertex1"); - vertex1.setInvokableClass(NoOpInvokable.class); - vertex1.setParallelism(parallelism); - - final JobVertex vertex2 = new JobVertex("vertex2"); - vertex2.setInvokableClass(NoOpInvokable.class); - vertex2.setParallelism(parallelism); - - final JobVertex vertex3 = new JobVertex("vertex3"); - vertex3.setInvokableClass(NoOpInvokable.class); - vertex3.setParallelism(parallelism); - - vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); - vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); - - final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - FailoverRegion preRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[0]); - FailoverRegion preRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]); - FailoverRegion preRegion3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); - - assertTrue(preRegion1 == preRegion2); - assertTrue(preRegion2 == preRegion3); - - for (int i = 1; i < parallelism; ++i) { - FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[i]); - FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[i]); - FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[i]); - - assertTrue(region1 == region2); - assertTrue(region2 == region3); - - assertTrue(preRegion1 != region1); - } - } - - /** - * Tests that validates that a single pipelined component via a sequence of all-to-all - * connections works correctly. - * - * 
    -	 *     (a1) -+-> (b1) -+-> (c1) 
-	 *           X         X
-	 *     (a2) -+-> (b2) -+-> (c2)
-	 *           X         X
-	 *     (a3) -+-> (b3) -+-> (c3)
-	 *
-	 *     ...
-	 *
    - */ - @Test - public void testOneComponentViaTwoExchanges() throws Exception { - final JobVertex vertex1 = new JobVertex("vertex1"); - vertex1.setInvokableClass(NoOpInvokable.class); - vertex1.setParallelism(3); - - final JobVertex vertex2 = new JobVertex("vertex2"); - vertex2.setInvokableClass(NoOpInvokable.class); - vertex2.setParallelism(5); - - final JobVertex vertex3 = new JobVertex("vertex3"); - vertex3.setInvokableClass(NoOpInvokable.class); - vertex3.setParallelism(2); - - vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]); - FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[4]); - FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); - - assertTrue(region1 == region2); - assertTrue(region2 == region3); - } - - /** - * Tests that validates that a single pipelined component via a cascade of joins - * works correctly. - * - *

    Non-parallelized view: - * 

    -	 *     (1)--+
-	 *          +--(5)-+
-	 *     (2)--+      |
-	 *                 +--(7)
-	 *     (3)--+      |
-	 *          +--(6)-+
-	 *     (4)--+
-	 *     ...
-	 *
    - */ - @Test - public void testOneComponentViaCascadeOfJoins() throws Exception { - final JobVertex vertex1 = new JobVertex("vertex1"); - vertex1.setInvokableClass(NoOpInvokable.class); - vertex1.setParallelism(8); - - final JobVertex vertex2 = new JobVertex("vertex2"); - vertex2.setInvokableClass(NoOpInvokable.class); - vertex2.setParallelism(8); - - final JobVertex vertex3 = new JobVertex("vertex3"); - vertex3.setInvokableClass(NoOpInvokable.class); - vertex3.setParallelism(8); - - final JobVertex vertex4 = new JobVertex("vertex4"); - vertex4.setInvokableClass(NoOpInvokable.class); - vertex4.setParallelism(8); - - final JobVertex vertex5 = new JobVertex("vertex5"); - vertex5.setInvokableClass(NoOpInvokable.class); - vertex5.setParallelism(4); - - final JobVertex vertex6 = new JobVertex("vertex6"); - vertex6.setInvokableClass(NoOpInvokable.class); - vertex6.setParallelism(4); - - final JobVertex vertex7 = new JobVertex("vertex7"); - vertex7.setInvokableClass(NoOpInvokable.class); - vertex7.setParallelism(2); - - vertex5.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex5.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - vertex6.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex6.connectNewDataSetAsInput(vertex4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - vertex7.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex7.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4, vertex5, vertex6, vertex7); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - - Iterator evs = eg.getAllExecutionVertices().iterator(); - - FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next()); - - while (evs.hasNext()) { - FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next()); - assertTrue(preRegion == region); - } - } - - /** - * Tests that validates that a single pipelined component instance from one source - * works correctly. - * - *

    Non-parallelized view: - * 

    -	 *                 +--(1)
-	 *          +--(5)-+
-	 *          |      +--(2)
-	 *     (7)--+
-	 *          |      +--(3)
-	 *          +--(6)-+
-	 *                 +--(4)
-	 *     ...
-	 *
    - */ - @Test - public void testOneComponentInstanceFromOneSource() throws Exception { - final JobVertex vertex1 = new JobVertex("vertex1"); - vertex1.setInvokableClass(NoOpInvokable.class); - vertex1.setParallelism(8); - - final JobVertex vertex2 = new JobVertex("vertex2"); - vertex2.setInvokableClass(NoOpInvokable.class); - vertex2.setParallelism(8); - - final JobVertex vertex3 = new JobVertex("vertex3"); - vertex3.setInvokableClass(NoOpInvokable.class); - vertex3.setParallelism(8); - - final JobVertex vertex4 = new JobVertex("vertex4"); - vertex4.setInvokableClass(NoOpInvokable.class); - vertex4.setParallelism(8); - - final JobVertex vertex5 = new JobVertex("vertex5"); - vertex5.setInvokableClass(NoOpInvokable.class); - vertex5.setParallelism(4); - - final JobVertex vertex6 = new JobVertex("vertex6"); - vertex6.setInvokableClass(NoOpInvokable.class); - vertex6.setParallelism(4); - - final JobVertex vertex7 = new JobVertex("vertex7"); - vertex7.setInvokableClass(NoOpInvokable.class); - vertex7.setParallelism(2); - - vertex1.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex2.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - vertex3.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex4.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - vertex5.connectNewDataSetAsInput(vertex7, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex6.connectNewDataSetAsInput(vertex7, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - final JobGraph jobGraph = new JobGraph("test job", vertex7, vertex5, vertex6, vertex1, vertex2, vertex3, vertex4); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - - Iterator evs = eg.getAllExecutionVertices().iterator(); - - FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next()); - - while (evs.hasNext()) { - FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next()); - assertTrue(preRegion == region); - } - } - - /** - * 
    -	 *     (a1) -+-> (b1) -+-> (c1) 
-	 *           X
-	 *     (a2) -+-> (b2) -+-> (c2)
-	 *           X
-	 *     (a3) -+-> (b3) -+-> (c3)
-	 *
-	 *           ^         ^
-	 *           |         |
-	 *     (pipelined) (blocking)
-	 *
-	 *
    - */ - @Test - public void testTwoComponentsViaBlockingExchange() throws Exception { - final JobVertex vertex1 = new JobVertex("vertex1"); - vertex1.setInvokableClass(NoOpInvokable.class); - vertex1.setParallelism(3); - - final JobVertex vertex2 = new JobVertex("vertex2"); - vertex2.setInvokableClass(NoOpInvokable.class); - vertex2.setParallelism(2); - - final JobVertex vertex3 = new JobVertex("vertex3"); - vertex3.setInvokableClass(NoOpInvokable.class); - vertex3.setParallelism(2); - - vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING); - - final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]); - FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]); - FailoverRegion region31 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); - FailoverRegion region32 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[1]); - - assertTrue(region1 == region2); - assertTrue(region2 != region31); - assertTrue(region32 != region31); - } - - /** - * 
    -	 *     (a1) -+-> (b1) -+-> (c1) 
-	 *           X         X
-	 *     (a2) -+-> (b2) -+-> (c2)
-	 *           X         X
-	 *     (a3) -+-> (b3) -+-> (c3)
-	 *
-	 *           ^         ^
-	 *           |         |
-	 *     (pipelined) (blocking)
-	 *
    - */ - @Test - public void testTwoComponentsViaBlockingExchange2() throws Exception { - final JobVertex vertex1 = new JobVertex("vertex1"); - vertex1.setInvokableClass(NoOpInvokable.class); - vertex1.setParallelism(3); - - final JobVertex vertex2 = new JobVertex("vertex2"); - vertex2.setInvokableClass(NoOpInvokable.class); - vertex2.setParallelism(2); - - final JobVertex vertex3 = new JobVertex("vertex3"); - vertex3.setInvokableClass(NoOpInvokable.class); - vertex3.setParallelism(2); - - vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - - final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[1]); - FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[0]); - FailoverRegion region31 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); - FailoverRegion region32 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[1]); - - assertTrue(region1 == region2); - assertTrue(region2 != region31); - assertTrue(region32 != region31); - } - - /** - * Cascades of joins with partially blocking, partially pipelined exchanges: - * 
    -	 *     (1)--+
-	 *          +--(5)-+
-	 *     (2)--+      |
-	 *              (block)
-	 *                 |
-	 *                 +--(7)
-	 *                 |
-	 *              (block)
-	 *     (3)--+      |
-	 *          +--(6)-+
-	 *     (4)--+
-	 *     ...
-	 *
    - * - * Component 1: 1, 2, 5; component 2: 3,4,6; component 3: 7 - */ - @Test - public void testMultipleComponentsViaCascadeOfJoins() throws Exception { - final JobVertex vertex1 = new JobVertex("vertex1"); - vertex1.setInvokableClass(NoOpInvokable.class); - vertex1.setParallelism(8); - - final JobVertex vertex2 = new JobVertex("vertex2"); - vertex2.setInvokableClass(NoOpInvokable.class); - vertex2.setParallelism(8); - - final JobVertex vertex3 = new JobVertex("vertex3"); - vertex3.setInvokableClass(NoOpInvokable.class); - vertex3.setParallelism(8); - - final JobVertex vertex4 = new JobVertex("vertex4"); - vertex4.setInvokableClass(NoOpInvokable.class); - vertex4.setParallelism(8); - - final JobVertex vertex5 = new JobVertex("vertex5"); - vertex5.setInvokableClass(NoOpInvokable.class); - vertex5.setParallelism(4); - - final JobVertex vertex6 = new JobVertex("vertex6"); - vertex6.setInvokableClass(NoOpInvokable.class); - vertex6.setParallelism(4); - - final JobVertex vertex7 = new JobVertex("vertex7"); - vertex7.setInvokableClass(NoOpInvokable.class); - vertex7.setParallelism(2); - - vertex5.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex5.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - vertex6.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex6.connectNewDataSetAsInput(vertex4, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - vertex7.connectNewDataSetAsInput(vertex5, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - vertex7.connectNewDataSetAsInput(vertex6, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - - final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4, vertex5, vertex6, vertex7); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - - FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex1.getID()).getTaskVertices()[0]); - FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex2.getID()).getTaskVertices()[5]); - FailoverRegion region5 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex5.getID()).getTaskVertices()[2]); - - assertTrue(region1 == region2); - assertTrue(region1 == region5); - - FailoverRegion region3 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex3.getID()).getTaskVertices()[0]); - FailoverRegion region4 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex4.getID()).getTaskVertices()[5]); - FailoverRegion region6 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex6.getID()).getTaskVertices()[2]); - - assertTrue(region3 == region4); - assertTrue(region3 == region6); - - FailoverRegion region71 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex7.getID()).getTaskVertices()[0]); - FailoverRegion region72 = failoverStrategy.getFailoverRegion(eg.getJobVertex(vertex7.getID()).getTaskVertices()[1]); - - assertTrue(region71 != region72); - assertTrue(region1 != region71); - assertTrue(region1 != region72); - assertTrue(region3 != region71); - assertTrue(region3 != region72); - } - - @Test - public void testDiamondWithMixedPipelinedAndBlockingExchanges() throws Exception { - final JobVertex vertex1 = new JobVertex("vertex1"); - vertex1.setInvokableClass(NoOpInvokable.class); - vertex1.setParallelism(8); - - final JobVertex vertex2 = new JobVertex("vertex2"); - vertex2.setInvokableClass(NoOpInvokable.class); - vertex2.setParallelism(8); - - final JobVertex vertex3 = new JobVertex("vertex3"); - vertex3.setInvokableClass(NoOpInvokable.class); - vertex3.setParallelism(8); - - final JobVertex vertex4 = new JobVertex("vertex4"); - vertex4.setInvokableClass(NoOpInvokable.class); - vertex4.setParallelism(8); - - vertex2.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - vertex3.connectNewDataSetAsInput(vertex1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - vertex4.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - vertex4.connectNewDataSetAsInput(vertex3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED); - - final JobGraph jobGraph = new JobGraph("test job", vertex1, vertex2, vertex3, vertex4); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - - Iterator evs = eg.getAllExecutionVertices().iterator(); - - FailoverRegion preRegion = failoverStrategy.getFailoverRegion(evs.next()); - - while (evs.hasNext()) { - FailoverRegion region = failoverStrategy.getFailoverRegion(evs.next()); - assertTrue(preRegion == region); - } - } - - /** - * This test checks that are strictly co-located vertices are in the same failover region, - * even through they are connected via a blocking pattern. - * This is currently an assumption / limitation of the scheduler. - */ - @Test - public void testBlockingAllToAllTopologyWithCoLocation() throws Exception { - final JobVertex source = new JobVertex("source"); - source.setInvokableClass(NoOpInvokable.class); - source.setParallelism(10); - - final JobVertex target = new JobVertex("target"); - target.setInvokableClass(NoOpInvokable.class); - target.setParallelism(13); - - target.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING); - - final SlotSharingGroup sharingGroup = new SlotSharingGroup(); - source.setSlotSharingGroup(sharingGroup); - target.setSlotSharingGroup(sharingGroup); - - source.setStrictlyCoLocatedWith(target); - - final JobGraph jobGraph = new JobGraph("test job", source, target); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]); - FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]); - - // we use 'assertTrue' here rather than 'assertEquals' because we want to test - // for referential equality, to be on the safe side - assertTrue(region1 == region2); - } - - /** - * This test checks that are strictly co-located vertices are in the same failover region, - * even through they are connected via a blocking pattern. - * This is currently an assumption / limitation of the scheduler. - */ - @Test - public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception { - final JobVertex source = new JobVertex("source"); - source.setInvokableClass(NoOpInvokable.class); - source.setParallelism(10); - - final JobVertex target = new JobVertex("target"); - target.setInvokableClass(NoOpInvokable.class); - target.setParallelism(10); - - target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); - - final SlotSharingGroup sharingGroup = new SlotSharingGroup(); - source.setSlotSharingGroup(sharingGroup); - target.setSlotSharingGroup(sharingGroup); - - source.setStrictlyCoLocatedWith(target); - - final JobGraph jobGraph = new JobGraph("test job", source, target); - final ExecutionGraph eg = createExecutionGraph(jobGraph); - - RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy(); - FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]); - FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]); - FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]); - FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]); - - // we use 'assertTrue' here rather than 'assertEquals' because we want to test - // for referential equality, to be on the safe side - assertTrue(sourceRegion1 == sourceRegion2); - assertTrue(sourceRegion2 == targetRegion1); - assertTrue(targetRegion1 == targetRegion2); - } - - // ------------------------------------------------------------------------ - // utilities - // ------------------------------------------------------------------------ - - private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException { - // configure the pipelined failover strategy - final Configuration jobManagerConfig = new Configuration(); - jobManagerConfig.setString( - JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, - FailoverStrategyLoader.LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME); - - final Time timeout = Time.seconds(10L); - return ExecutionGraphBuilder.buildGraph( - null, - jobGraph, - jobManagerConfig, - TestingUtils.defaultExecutor(), - TestingUtils.defaultExecutor(), - mock(SlotProvider.class), - PipelinedFailoverRegionBuildingTest.class.getClassLoader(), - new StandaloneCheckpointRecoveryFactory(), - timeout, - new NoRestartStrategy(), - new UnregisteredMetricsGroup(), - VoidBlobWriter.getInstance(), - timeout, - log, - NettyShuffleMaster.INSTANCE, - NoOpPartitionTracker.INSTANCE); - } -} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartBackoffTimeStrategyFactoryLoaderTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartBackoffTimeStrategyFactoryLoaderTest.java new file mode 100644 index 0000000000000000000000000000000000000000..3114e76ba27b9ef65c779c32d8bca6d00c991145 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/RestartBackoffTimeStrategyFactoryLoaderTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.executiongraph.failover.flip1; + +import org.apache.flink.api.common.restartstrategy.RestartStrategies; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.RestartStrategyOptions; +import org.apache.flink.runtime.executiongraph.restart.NoOrFixedIfCheckpointingEnabledRestartStrategyFactory; +import org.apache.flink.util.TestLogger; + +import org.junit.Test; + +import static junit.framework.TestCase.assertEquals; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsInstanceOf.instanceOf; + +/** + * Unit tests for {@link RestartBackoffTimeStrategyFactoryLoader}. + */ +public class RestartBackoffTimeStrategyFactoryLoaderTest extends TestLogger { + + private static final RestartStrategies.RestartStrategyConfiguration DEFAULT_JOB_LEVEL_RESTART_CONFIGURATION = + RestartStrategies.fallBackRestart(); + + @Test + public void testNoRestartStrategySpecifiedInJobConfig() { + final Configuration conf = new Configuration(); + conf.setString(RestartStrategyOptions.RESTART_STRATEGY, "failure-rate"); + + final RestartBackoffTimeStrategy.Factory factory = + RestartBackoffTimeStrategyFactoryLoader.createRestartStrategyFactory( + RestartStrategies.noRestart(), + conf, + false); + + assertEquals(NoRestartBackoffTimeStrategy.NoRestartBackoffTimeStrategyFactory.INSTANCE, factory); + } + + @Test + public void testFixedDelayRestartStrategySpecifiedInJobConfig() { + final Configuration conf = new Configuration(); + conf.setString(RestartStrategyOptions.RESTART_STRATEGY, "failure-rate"); + + final RestartBackoffTimeStrategy.Factory factory = + RestartBackoffTimeStrategyFactoryLoader.createRestartStrategyFactory( + RestartStrategies.fixedDelayRestart(1, Time.milliseconds(1000)), + conf, + false); + + assertThat( + factory, + instanceOf(FixedDelayRestartBackoffTimeStrategy.FixedDelayRestartBackoffTimeStrategyFactory.class)); + } + + @Test + public void testFailureRateRestartStrategySpecifiedInJobConfig() { + final Configuration conf = new Configuration(); + conf.setString(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay"); + + final RestartBackoffTimeStrategy.Factory factory = + RestartBackoffTimeStrategyFactoryLoader.createRestartStrategyFactory( + RestartStrategies.failureRateRestart(1, Time.milliseconds(1000), Time.milliseconds(1000)), + conf, + false); + + assertThat( + factory, + instanceOf(FailureRateRestartBackoffTimeStrategy.FailureRateRestartBackoffTimeStrategyFactory.class)); + } + + @Test + public void testNoRestartStrategySpecifiedInClusterConfig() { + final Configuration conf = new Configuration(); + conf.setString(RestartStrategyOptions.RESTART_STRATEGY, "none"); + + final RestartBackoffTimeStrategy.Factory factory = + RestartBackoffTimeStrategyFactoryLoader.createRestartStrategyFactory( + DEFAULT_JOB_LEVEL_RESTART_CONFIGURATION, + conf, + false); + + assertEquals(NoRestartBackoffTimeStrategy.NoRestartBackoffTimeStrategyFactory.INSTANCE, factory); + } + + @Test + public void testFixedDelayStrategySpecifiedInClusterConfig() { + final Configuration conf = new Configuration(); + conf.setString(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay"); + + final RestartBackoffTimeStrategy.Factory factory = + RestartBackoffTimeStrategyFactoryLoader.createRestartStrategyFactory( + DEFAULT_JOB_LEVEL_RESTART_CONFIGURATION, + conf, + false); + + assertThat( + factory, + instanceOf(FixedDelayRestartBackoffTimeStrategy.FixedDelayRestartBackoffTimeStrategyFactory.class)); + } + + @Test + public void testFailureRateStrategySpecifiedInClusterConfig() { + final Configuration conf = new Configuration(); + conf.setString(RestartStrategyOptions.RESTART_STRATEGY, "failure-rate"); + + final RestartBackoffTimeStrategy.Factory factory = + RestartBackoffTimeStrategyFactoryLoader.createRestartStrategyFactory( + DEFAULT_JOB_LEVEL_RESTART_CONFIGURATION, + conf, + false); + + assertThat( + factory, + instanceOf(FailureRateRestartBackoffTimeStrategy.FailureRateRestartBackoffTimeStrategyFactory.class)); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidStrategySpecifiedInClusterConfig() { + final Configuration conf = new Configuration(); + conf.setString(RestartStrategyOptions.RESTART_STRATEGY, "invalid-strategy"); + + RestartBackoffTimeStrategyFactoryLoader.createRestartStrategyFactory( + DEFAULT_JOB_LEVEL_RESTART_CONFIGURATION, + conf, + false); + } + + @Test + public void testNoStrategySpecifiedWhenCheckpointingEnabled() { + final RestartBackoffTimeStrategy.Factory factory = + RestartBackoffTimeStrategyFactoryLoader.createRestartStrategyFactory( + DEFAULT_JOB_LEVEL_RESTART_CONFIGURATION, + new Configuration(), + true); + + RestartBackoffTimeStrategy strategy = factory.create(); + assertThat( + strategy, + instanceOf(FixedDelayRestartBackoffTimeStrategy.class)); + + FixedDelayRestartBackoffTimeStrategy fixedDelayStrategy = (FixedDelayRestartBackoffTimeStrategy) strategy; + assertEquals( + NoOrFixedIfCheckpointingEnabledRestartStrategyFactory.DEFAULT_RESTART_DELAY, + fixedDelayStrategy.getBackoffTime()); + assertEquals( + NoOrFixedIfCheckpointingEnabledRestartStrategyFactory.DEFAULT_RESTART_ATTEMPTS, + fixedDelayStrategy.getMaxNumberRestartAttempts()); + } + + @Test + public void testNoStrategySpecifiedWhenCheckpointingDisabled() { + final RestartBackoffTimeStrategy.Factory factory = + RestartBackoffTimeStrategyFactoryLoader.createRestartStrategyFactory( + DEFAULT_JOB_LEVEL_RESTART_CONFIGURATION, + new Configuration(), + false); + + assertThat( + factory, + instanceOf(NoRestartBackoffTimeStrategy.NoRestartBackoffTimeStrategyFactory.class)); + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/TestRestartBackoffTimeStrategy.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/TestRestartBackoffTimeStrategy.java index 878c89f4c5642bb119d54454bdc71864bc7736d2..0606cce3ee827c3b1333c4fc8c0a09196cb63f34 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/TestRestartBackoffTimeStrategy.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/failover/flip1/TestRestartBackoffTimeStrategy.java @@ -24,9 +24,9 @@ package org.apache.flink.runtime.executiongraph.failover.flip1; */ public class TestRestartBackoffTimeStrategy implements RestartBackoffTimeStrategy { - private final boolean canRestart; + private boolean canRestart; - private final long backoffTime; + private long backoffTime; public TestRestartBackoffTimeStrategy(boolean canRestart, long backoffTime) { this.canRestart = canRestart; @@ -47,4 +47,12 @@ public class TestRestartBackoffTimeStrategy implements RestartBackoffTimeStrateg public void notifyFailure(Throwable cause) { // ignore } + + public void setCanRestart(final boolean canRestart) { + this.canRestart = canRestart; + } + + public void setBackoffTime(final long backoffTime) { + this.backoffTime = backoffTime; + } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java index 15ec34379a6963646588aafd62cc9373da23d80e..fe55bf351177626da9783eee20052105fe7f0351 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/utils/SimpleSlotProvider.java @@ -46,6 +46,7 @@ import java.net.InetAddress; import java.util.ArrayDeque; import java.util.HashMap; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; import static org.apache.flink.util.Preconditions.checkArgument; import static org.apache.flink.util.Preconditions.checkNotNull; @@ -106,12 +107,14 @@ public class SimpleSlotProvider implements SlotProvider, SlotOwner { .setSlotNumber(slot.getPhysicalSlotNumber()) .setAllocationId(slot.getAllocationId()) .setSlotRequestId(slotRequestId) + .setSlotSharingGroupId(task.getSlotSharingGroupId()) .setSlotOwner(this) .createTestingLogicalSlot(); allocatedSlots.put(slotRequestId, slot); return CompletableFuture.completedFuture(result); - } - else { + } else if (allowQueued) { + return FutureUtils.completedExceptionally(new TimeoutException()); + } else { return FutureUtils.completedExceptionally(new NoResourceAvailableException()); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/FsNegativeRunningJobsRegistryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/FsNegativeRunningJobsRegistryTest.java deleted file mode 100644 index b0c7778a2bc2580690a34ca2decc7f6bea259c7e..0000000000000000000000000000000000000000 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/FsNegativeRunningJobsRegistryTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.highavailability; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.core.fs.Path; -import org.apache.flink.runtime.highavailability.RunningJobsRegistry.JobSchedulingStatus; -import org.apache.flink.util.TestLogger; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; - -import static org.junit.Assert.assertEquals; - -/** - * Tests for the {@link FsNegativeRunningJobsRegistry} on a local file system. - */ -public class FsNegativeRunningJobsRegistryTest extends TestLogger { - - @Rule - public final TemporaryFolder tempFolder = new TemporaryFolder(); - - @Test - public void testCreateAndSetFinished() throws Exception { - final File folder = tempFolder.newFolder(); - final String uri = folder.toURI().toString(); - - final JobID jid = new JobID(); - - FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri)); - - // another registry should pick this up - FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri)); - - // initially, without any call, the job is pending - assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid)); - assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid)); - - // after set running, the job is running - registry.setJobRunning(jid); - assertEquals(JobSchedulingStatus.RUNNING, registry.getJobSchedulingStatus(jid)); - assertEquals(JobSchedulingStatus.RUNNING, otherRegistry.getJobSchedulingStatus(jid)); - - // set the job to finished and validate - registry.setJobFinished(jid); - assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid)); - assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid)); - } - - @Test - public void testSetFinishedAndRunning() throws Exception { - final File folder = tempFolder.newFolder(); - final String uri = folder.toURI().toString(); - - final JobID jid = new JobID(); - - FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri)); - - // set the job to finished and validate - registry.setJobFinished(jid); - assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid)); - - // set the job to running does not overwrite the finished status - registry.setJobRunning(jid); - assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid)); - - // another registry should pick this up - FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri)); - assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid)); - - // clear the running and finished marker, it will be pending - otherRegistry.clearJob(jid); - assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid)); - assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid)); - } -} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java index 7f06d60c949b78ef15472f7224ace7ed785cf728..840d96cf51bb316e39f120ad9651331bd3756219 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/EmbeddedHaServicesTest.java @@ -22,8 +22,9 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.runtime.concurrent.Executors; import org.apache.flink.runtime.leaderelection.LeaderContender; import org.apache.flink.runtime.leaderelection.LeaderElectionService; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; +import org.apache.flink.runtime.util.LeaderConnectionInfo; +import org.apache.flink.runtime.util.LeaderRetrievalUtils; import org.apache.flink.util.TestLogger; import org.junit.After; @@ -37,17 +38,17 @@ import static junit.framework.TestCase.assertTrue; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThat; import static org.mockito.Matchers.any; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; /** * Tests for the {@link EmbeddedHaServices}. */ public class EmbeddedHaServicesTest extends TestLogger { + private static final String ADDRESS = "foobar"; + private EmbeddedHaServices embeddedHaServices; @Before @@ -120,26 +121,29 @@ public class EmbeddedHaServicesTest extends TestLogger { */ @Test public void testJobManagerLeaderRetrieval() throws Exception { - final String address = "foobar"; JobID jobId = new JobID(); - LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class); - LeaderContender leaderContender = mock(LeaderContender.class); - when(leaderContender.getAddress()).thenReturn(address); LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId); LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId); + runLeaderRetrievalTest(leaderElectionService, leaderRetrievalService); + } + + private void runLeaderRetrievalTest(LeaderElectionService leaderElectionService, LeaderRetrievalService leaderRetrievalService) throws Exception { + LeaderRetrievalUtils.LeaderConnectionInfoListener leaderRetrievalListener = new LeaderRetrievalUtils.LeaderConnectionInfoListener(); + TestingLeaderContender leaderContender = new TestingLeaderContender(); + leaderRetrievalService.start(leaderRetrievalListener); leaderElectionService.start(leaderContender); - ArgumentCaptor leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class); - verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture()); + final UUID leaderId = leaderContender.getLeaderSessionFuture().get(); - final UUID leaderId = leaderIdArgumentCaptor.getValue(); + leaderElectionService.confirmLeadership(leaderId, ADDRESS); - leaderElectionService.confirmLeaderSessionID(leaderId); + final LeaderConnectionInfo leaderConnectionInfo = leaderRetrievalListener.getLeaderConnectionInfoFuture().get(); - verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId)); + assertThat(leaderConnectionInfo.getAddress(), is(ADDRESS)); + assertThat(leaderConnectionInfo.getLeaderSessionId(), is(leaderId)); } /** @@ -147,25 +151,10 @@ public class EmbeddedHaServicesTest extends TestLogger { */ @Test public void testResourceManagerLeaderRetrieval() throws Exception { - final String address = "foobar"; - LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class); - LeaderContender leaderContender = mock(LeaderContender.class); - when(leaderContender.getAddress()).thenReturn(address); - LeaderElectionService leaderElectionService = embeddedHaServices.getResourceManagerLeaderElectionService(); LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getResourceManagerLeaderRetriever(); - leaderRetrievalService.start(leaderRetrievalListener); - leaderElectionService.start(leaderContender); - - ArgumentCaptor leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class); - verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture()); - - final UUID leaderId = leaderIdArgumentCaptor.getValue(); - - leaderElectionService.confirmLeaderSessionID(leaderId); - - verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId)); + runLeaderRetrievalTest(leaderElectionService, leaderRetrievalService); } /** @@ -191,8 +180,8 @@ public class EmbeddedHaServicesTest extends TestLogger { assertThat(dispatcherLeaderElectionService.hasLeadership(newLeaderSessionId), is(true)); - dispatcherLeaderElectionService.confirmLeaderSessionID(oldLeaderSessionId); - dispatcherLeaderElectionService.confirmLeaderSessionID(newLeaderSessionId); + dispatcherLeaderElectionService.confirmLeadership(oldLeaderSessionId, ADDRESS); + dispatcherLeaderElectionService.confirmLeadership(newLeaderSessionId, ADDRESS); assertThat(dispatcherLeaderElectionService.hasLeadership(newLeaderSessionId), is(true)); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/TestingLeaderContender.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/TestingLeaderContender.java index 0b286716769ce1cd0256051478b57a42971fad1e..9bad8128758c545065b688711732b46ba25a19c6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/TestingLeaderContender.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/embedded/TestingLeaderContender.java @@ -58,7 +58,7 @@ final class TestingLeaderContender implements LeaderContender { } @Override - public String getAddress() { + public String getDescription() { return "foobar"; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionServiceTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionServiceTest.java deleted file mode 100644 index 3875c4f171c50fef739ba997675283f0fc850d4b..0000000000000000000000000000000000000000 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/highavailability/nonha/leaderelection/SingleLeaderElectionServiceTest.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.runtime.highavailability.nonha.leaderelection; - -import org.apache.flink.runtime.concurrent.Executors; -import org.apache.flink.runtime.leaderelection.LeaderContender; -import org.apache.flink.runtime.leaderelection.LeaderElectionService; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; -import org.apache.flink.util.StringUtils; - -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import java.util.Random; -import java.util.UUID; -import java.util.concurrent.Executor; - -import static org.junit.Assert.fail; -import static org.mockito.Mockito.*; - -/** - * Tests for the {@link SingleLeaderElectionService}. - */ -public class SingleLeaderElectionServiceTest { - - private static final Random RND = new Random(); - - private final Executor executor = Executors.directExecutor(); - - // ------------------------------------------------------------------------ - - @Test - public void testStartStopAssignLeadership() throws Exception { - final UUID uuid = UUID.randomUUID(); - final SingleLeaderElectionService service = new SingleLeaderElectionService(executor, uuid); - - final LeaderContender contender = mockContender(service); - final LeaderContender otherContender = mockContender(service); - - service.start(contender); - verify(contender, times(1)).grantLeadership(uuid); - - service.stop(); - verify(contender, times(1)).revokeLeadership(); - - // start with a new contender - the old contender must not gain another leadership - service.start(otherContender); - verify(otherContender, times(1)).grantLeadership(uuid); - - verify(contender, times(1)).grantLeadership(uuid); - verify(contender, times(1)).revokeLeadership(); - } - - @Test - public void testStopBeforeConfirmingLeadership() throws Exception { - final UUID uuid = UUID.randomUUID(); - final SingleLeaderElectionService service = new SingleLeaderElectionService(executor, uuid); - - final LeaderContender contender = mock(LeaderContender.class); - - service.start(contender); - verify(contender, times(1)).grantLeadership(uuid); - - service.stop(); - - // because the leadership was never confirmed, there is no "revoke" call - verifyNoMoreInteractions(contender); - } - - @Test - public void testStartOnlyOnce() throws Exception { - final UUID uuid = UUID.randomUUID(); - final SingleLeaderElectionService service = new SingleLeaderElectionService(executor, uuid); - - final LeaderContender contender = mock(LeaderContender.class); - final LeaderContender otherContender = mock(LeaderContender.class); - - service.start(contender); - verify(contender, times(1)).grantLeadership(uuid); - - // should not be possible to start again this with another contender - try { - service.start(otherContender); - fail("should fail with an exception"); - } catch (IllegalStateException e) { - // expected - } - - // should not be possible to start this again with the same contender - try { - service.start(contender); - fail("should fail with an exception"); - } catch (IllegalStateException e) { - // expected - } - } - - @Test - public void testShutdown() throws Exception { - final UUID uuid = UUID.randomUUID(); - final SingleLeaderElectionService service = new SingleLeaderElectionService(executor, uuid); - - // create a leader contender and let it grab leadership - final LeaderContender contender = mockContender(service); - service.start(contender); - verify(contender, times(1)).grantLeadership(uuid); - - // some leader listeners - final LeaderRetrievalListener listener1 = mock(LeaderRetrievalListener.class); - final LeaderRetrievalListener listener2 = mock(LeaderRetrievalListener.class); - - LeaderRetrievalService listenerService1 = service.createLeaderRetrievalService(); - LeaderRetrievalService listenerService2 = service.createLeaderRetrievalService(); - - listenerService1.start(listener1); - listenerService2.start(listener2); - - // one listener stops - listenerService1.stop(); - - // shut down the service - service.shutdown(); - - // the leader contender and running listener should get error notifications - verify(contender, times(1)).handleError(any(Exception.class)); - verify(listener2, times(1)).handleError(any(Exception.class)); - - // the stopped listener gets no notification - verify(listener1, times(0)).handleError(any(Exception.class)); - - // should not be possible to start again after shutdown - try { - service.start(contender); - fail("should fail with an exception"); - } catch (IllegalStateException e) { - // expected - } - - // no additional leadership grant - verify(contender, times(1)).grantLeadership(any(UUID.class)); - } - - @Test - public void testImmediateShutdown() throws Exception { - final UUID uuid = UUID.randomUUID(); - final SingleLeaderElectionService service = new SingleLeaderElectionService(executor, uuid); - service.shutdown(); - - final LeaderContender contender = mock(LeaderContender.class); - - // should not be possible to start - try { - service.start(contender); - fail("should fail with an exception"); - } catch (IllegalStateException e) { - // expected - } - - // no additional leadership grant - verify(contender, times(0)).grantLeadership(any(UUID.class)); - } - -// @Test -// public void testNotifyListenersWhenLeaderElected() throws Exception { -// final UUID uuid = UUID.randomUUID(); -// final SingleLeaderElectionService service = new SingleLeaderElectionService(executor, uuid); -// -// final LeaderRetrievalListener listener1 = mock(LeaderRetrievalListener.class); -// final LeaderRetrievalListener listener2 = mock(LeaderRetrievalListener.class); -// -// LeaderRetrievalService listenerService1 = service.createLeaderRetrievalService(); -// LeaderRetrievalService listenerService2 = service.createLeaderRetrievalService(); -// -// listenerService1.start(listener1); -// listenerService1.start(listener2); -// -// final LeaderContender contender = mockContender(service); -// service.start(contender); -// -// veri -// } - - // ------------------------------------------------------------------------ - // utilities - // ------------------------------------------------------------------------ - - private static LeaderContender mockContender(final LeaderElectionService service) { - String address = StringUtils.getRandomString(RND, 5, 10, 'a', 'z'); - return mockContender(service, address); - } - - private static LeaderContender mockContender(final LeaderElectionService service, final String address) { - LeaderContender mockContender = mock(LeaderContender.class); - - when(mockContender.getAddress()).thenReturn(address); - - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - final UUID uuid = (UUID) invocation.getArguments()[0]; - service.confirmLeaderSessionID(uuid); - return null; - } - }).when(mockContender).grantLeadership(any(UUID.class)); - - return mockContender; - } -} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/BroadcastRecordWriterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/BroadcastRecordWriterTest.java new file mode 100644 index 0000000000000000000000000000000000000000..d95b87c836cf2b119b2d5caf55cfe9de0a0fc0bf --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/BroadcastRecordWriterTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.io.network.api.writer; + +import org.apache.flink.core.io.IOReadableWritable; +import org.apache.flink.runtime.io.network.api.serialization.RecordDeserializer; +import org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer; +import org.apache.flink.runtime.io.network.buffer.BufferConsumer; +import org.apache.flink.runtime.io.network.util.TestPooledBufferProvider; +import org.apache.flink.testutils.serialization.types.SerializationTestType; +import org.apache.flink.testutils.serialization.types.SerializationTestTypeFactory; +import org.apache.flink.testutils.serialization.types.Util; + +import org.junit.Test; + +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.Map; +import java.util.Queue; + +import static org.junit.Assert.assertEquals; + +/** + * Tests for the {@link BroadcastRecordWriter}. + */ +public class BroadcastRecordWriterTest extends RecordWriterTest { + + public BroadcastRecordWriterTest() { + super(true); + } + + /** + * Tests the number of requested buffers and results are correct in the case of switching + * modes between {@link BroadcastRecordWriter#broadcastEmit(IOReadableWritable)} and + * {@link BroadcastRecordWriter#randomEmit(IOReadableWritable)}. + */ + @Test + public void testBroadcastMixedRandomEmitRecord() throws Exception { + final int numberOfChannels = 4; + final int numberOfRecords = 8; + final int bufferSize = 32; + + @SuppressWarnings("unchecked") + final Queue[] queues = new Queue[numberOfChannels]; + for (int i = 0; i < numberOfChannels; i++) { + queues[i] = new ArrayDeque<>(); + } + + final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize); + final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider); + final BroadcastRecordWriter writer = new BroadcastRecordWriter<>(partitionWriter, 0, "test"); + final RecordDeserializer deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>( + new String[]{ tempFolder.getRoot().getAbsolutePath() }); + + // generate the configured number of int values as global record set + final Iterable records = Util.randomRecords(numberOfRecords, SerializationTestTypeFactory.INT); + // restore the corresponding record set for every input channel + final Map> serializedRecords = new HashMap<>(); + for (int i = 0; i < numberOfChannels; i++) { + serializedRecords.put(i, new ArrayDeque<>()); + } + + // every record in global set would both emit into one random channel and broadcast to all the channels + int index = 0; + for (SerializationTestType record : records) { + int randomChannel = index++ % numberOfChannels; + writer.randomEmit(record, randomChannel); + serializedRecords.get(randomChannel).add(record); + + writer.broadcastEmit(record); + for (int i = 0; i < numberOfChannels; i++) { + serializedRecords.get(i).add(record); + } + } + + final int numberOfCreatedBuffers = bufferProvider.getNumberOfCreatedBuffers(); + // verify the expected number of requested buffers, and it would always request a new buffer while random emitting + assertEquals(numberOfRecords, numberOfCreatedBuffers); + + for (int i = 0; i < numberOfChannels; i++) { + // every channel would queue the number of above crated buffers + assertEquals(numberOfRecords, queues[i].size()); + + final int excessRandomRecords = i < numberOfRecords % numberOfChannels ? 1 : 0; + final int numberOfRandomRecords = numberOfRecords / numberOfChannels + excessRandomRecords; + final int numberOfTotalRecords = numberOfRecords + numberOfRandomRecords; + // verify the data correctness in every channel queue + verifyDeserializationResults( + queues[i], + deserializer, + serializedRecords.get(i), + numberOfCreatedBuffers, + numberOfTotalRecords); + } + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java index 882f83c5a9bf40841e987af70fd504f733908f5b..239c47d3a5cf6ec6d99a0e84d92775039d41db79 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/api/writer/RecordWriterTest.java @@ -83,6 +83,16 @@ import static org.mockito.Mockito.when; */ public class RecordWriterTest { + private final boolean isBroadcastWriter; + + public RecordWriterTest() { + this(false); + } + + RecordWriterTest(boolean isBroadcastWriter) { + this.isBroadcastWriter = isBroadcastWriter; + } + @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); @@ -132,7 +142,7 @@ public class RecordWriterTest { ResultPartitionWriter partitionWriter = new RecyclingPartitionWriter(bufferProvider); - final RecordWriter recordWriter = new RecordWriterBuilder().build(partitionWriter); + final RecordWriter recordWriter = createRecordWriter(partitionWriter); Future result = executor.submit(new Callable() { @Override @@ -184,7 +194,7 @@ public class RecordWriterTest { ResultPartitionWriter partitionWriter = spy(new RecyclingPartitionWriter(new TestPooledBufferProvider(1, 16))); - RecordWriter recordWriter = new RecordWriterBuilder().build(partitionWriter); + RecordWriter recordWriter = createRecordWriter(partitionWriter); // Fill a buffer, but don't write it out. recordWriter.emit(new IntValue(0)); @@ -214,7 +224,7 @@ public class RecordWriterTest { TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize); ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider); - RecordWriter writer = new RecordWriterBuilder().build(partitionWriter); + RecordWriter writer = createRecordWriter(partitionWriter); CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 919192L, Integer.MAX_VALUE + 18828228L, CheckpointOptions.forCheckpointWithDefaultLocation()); // No records emitted yet, broadcast should not request a buffer @@ -251,53 +261,66 @@ public class RecordWriterTest { TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize); ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider); - RecordWriter writer = new RecordWriterBuilder().build(partitionWriter); + RecordWriter writer = createRecordWriter(partitionWriter); CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 1292L, Integer.MAX_VALUE + 199L, CheckpointOptions.forCheckpointWithDefaultLocation()); // Emit records on some channels first (requesting buffers), then // broadcast the event. The record buffers should be emitted first, then // the event. After the event, no new buffer should be requested. - // (i) Smaller than the buffer size (single buffer request => 1) + // (i) Smaller than the buffer size byte[] bytes = new byte[bufferSize / 2]; rand.nextBytes(bytes); writer.emit(new ByteArrayIO(bytes)); - // (ii) Larger than the buffer size (two buffer requests => 1 + 2) + // (ii) Larger than the buffer size bytes = new byte[bufferSize + 1]; rand.nextBytes(bytes); writer.emit(new ByteArrayIO(bytes)); - // (iii) Exactly the buffer size (single buffer request => 1 + 2 + 1) + // (iii) Exactly the buffer size bytes = new byte[bufferSize - lenBytes]; rand.nextBytes(bytes); writer.emit(new ByteArrayIO(bytes)); - // (iv) Nothing on the 4th channel (no buffer request => 1 + 2 + 1 + 0 = 4) - - // (v) Broadcast the event + // (iv) Broadcast the event writer.broadcastEvent(barrier); - assertEquals(4, bufferProvider.getNumberOfCreatedBuffers()); + if (isBroadcastWriter) { + assertEquals(3, bufferProvider.getNumberOfCreatedBuffers()); - BufferOrEvent boe; - assertEquals(2, queues[0].size()); // 1 buffer + 1 event - assertTrue(parseBuffer(queues[0].remove(), 0).isBuffer()); - assertEquals(3, queues[1].size()); // 2 buffers + 1 event - assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer()); - assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer()); - assertEquals(2, queues[2].size()); // 1 buffer + 1 event - assertTrue(parseBuffer(queues[2].remove(), 2).isBuffer()); - assertEquals(1, queues[3].size()); // 0 buffers + 1 event + for (int i = 0; i < numberOfChannels; i++) { + assertEquals(4, queues[i].size()); // 3 buffer + 1 event - // every queue's last element should be the event - for (int i = 0; i < numberOfChannels; i++) { - boe = parseBuffer(queues[i].remove(), i); - assertTrue(boe.isEvent()); - assertEquals(barrier, boe.getEvent()); + for (int j = 0; j < 3; j++) { + assertTrue(parseBuffer(queues[i].remove(), 0).isBuffer()); + } + + BufferOrEvent boe = parseBuffer(queues[i].remove(), i); + assertTrue(boe.isEvent()); + assertEquals(barrier, boe.getEvent()); + } + } else { + assertEquals(4, bufferProvider.getNumberOfCreatedBuffers()); + + assertEquals(2, queues[0].size()); // 1 buffer + 1 event + assertTrue(parseBuffer(queues[0].remove(), 0).isBuffer()); + assertEquals(3, queues[1].size()); // 2 buffers + 1 event + assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer()); + assertTrue(parseBuffer(queues[1].remove(), 1).isBuffer()); + assertEquals(2, queues[2].size()); // 1 buffer + 1 event + assertTrue(parseBuffer(queues[2].remove(), 2).isBuffer()); + assertEquals(1, queues[3].size()); // 0 buffers + 1 event + + // every queue's last element should be the event + for (int i = 0; i < numberOfChannels; i++) { + BufferOrEvent boe = parseBuffer(queues[i].remove(), i); + assertTrue(boe.isEvent()); + assertEquals(barrier, boe.getEvent()); + } } } @@ -313,7 +336,7 @@ public class RecordWriterTest { ResultPartitionWriter partition = new CollectingPartitionWriter(queues, new TestPooledBufferProvider(Integer.MAX_VALUE)); - RecordWriter writer = new RecordWriterBuilder().build(partition); + RecordWriter writer = createRecordWriter(partition); writer.broadcastEvent(EndOfPartitionEvent.INSTANCE); @@ -352,21 +375,46 @@ public class RecordWriterTest { verifyBroadcastBufferOrEventIndependence(false); } - /** - * Tests that records are broadcast via {@link ChannelSelector} and - * {@link RecordWriter#emit(IOReadableWritable)}. - */ - @Test - public void testEmitRecordWithBroadcastPartitioner() throws Exception { - emitRecordWithBroadcastPartitionerOrBroadcastEmitRecord(false); - } - /** * Tests that records are broadcast via {@link RecordWriter#broadcastEmit(IOReadableWritable)}. */ @Test public void testBroadcastEmitRecord() throws Exception { - emitRecordWithBroadcastPartitionerOrBroadcastEmitRecord(true); + final int numberOfChannels = 4; + final int bufferSize = 32; + final int numValues = 8; + final int serializationLength = 4; + + @SuppressWarnings("unchecked") + final Queue[] queues = new Queue[numberOfChannels]; + for (int i = 0; i < numberOfChannels; i++) { + queues[i] = new ArrayDeque<>(); + } + + final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize); + final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider); + final RecordWriter writer = createRecordWriter(partitionWriter); + final RecordDeserializer deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>( + new String[]{ tempFolder.getRoot().getAbsolutePath() }); + + final ArrayDeque serializedRecords = new ArrayDeque<>(); + final Iterable records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT); + for (SerializationTestType record : records) { + serializedRecords.add(record); + writer.broadcastEmit(record); + } + + final int numRequiredBuffers = numValues / (bufferSize / (4 + serializationLength)); + if (isBroadcastWriter) { + assertEquals(numRequiredBuffers, bufferProvider.getNumberOfCreatedBuffers()); + } else { + assertEquals(numRequiredBuffers * numberOfChannels, bufferProvider.getNumberOfCreatedBuffers()); + } + + for (int i = 0; i < numberOfChannels; i++) { + assertEquals(numRequiredBuffers, queues[i].size()); + verifyDeserializationResults(queues[i], deserializer, serializedRecords.clone(), numRequiredBuffers, numValues); + } } private void verifyBroadcastBufferOrEventIndependence(boolean broadcastEvent) throws Exception { @@ -375,7 +423,7 @@ public class RecordWriterTest { ResultPartitionWriter partition = new CollectingPartitionWriter(queues, new TestPooledBufferProvider(Integer.MAX_VALUE)); - RecordWriter writer = new RecordWriterBuilder().build(partition); + RecordWriter writer = createRecordWriter(partition); if (broadcastEvent) { writer.broadcastEvent(EndOfPartitionEvent.INSTANCE); @@ -396,59 +444,32 @@ public class RecordWriterTest { assertEquals("Buffer 2 shares the same reader index as buffer 1", 0, buffer2.getReaderIndex()); } - /** - * The results of emitting records via BroadcastPartitioner or broadcasting records directly are the same, - * that is all the target channels can receive the whole outputs. - * - * @param isBroadcastEmit whether using {@link RecordWriter#broadcastEmit(IOReadableWritable)} or not - */ - private void emitRecordWithBroadcastPartitionerOrBroadcastEmitRecord(boolean isBroadcastEmit) throws Exception { - final int numberOfChannels = 4; - final int bufferSize = 32; - final int numValues = 8; - final int serializationLength = 4; + protected void verifyDeserializationResults( + Queue queue, + RecordDeserializer deserializer, + ArrayDeque expectedRecords, + int numRequiredBuffers, + int numValues) throws Exception { + int assertRecords = 0; + for (int j = 0; j < numRequiredBuffers; j++) { + Buffer buffer = buildSingleBuffer(queue.remove()); + deserializer.setNextBuffer(buffer); - @SuppressWarnings("unchecked") - final Queue[] queues = new Queue[numberOfChannels]; - for (int i = 0; i < numberOfChannels; i++) { - queues[i] = new ArrayDeque<>(); + assertRecords += DeserializationUtils.deserializeRecords(expectedRecords, deserializer); } + Assert.assertEquals(numValues, assertRecords); + } - final TestPooledBufferProvider bufferProvider = new TestPooledBufferProvider(Integer.MAX_VALUE, bufferSize); - final ResultPartitionWriter partitionWriter = new CollectingPartitionWriter(queues, bufferProvider); - final ChannelSelector selector = new OutputEmitter(ShipStrategyType.BROADCAST, 0); - final RecordWriter writer = new RecordWriterBuilder() - .setChannelSelector(selector) - .setTimeout(0) - .build(partitionWriter); - final RecordDeserializer deserializer = new SpillingAdaptiveSpanningRecordDeserializer<>( - new String[]{ tempFolder.getRoot().getAbsolutePath() }); - - final ArrayDeque serializedRecords = new ArrayDeque<>(); - final Iterable records = Util.randomRecords(numValues, SerializationTestTypeFactory.INT); - for (SerializationTestType record : records) { - serializedRecords.add(record); - - if (isBroadcastEmit) { - writer.broadcastEmit(record); - } else { - writer.emit(record); - } - } - - final int requiredBuffers = numValues / (bufferSize / (4 + serializationLength)); - for (int i = 0; i < numberOfChannels; i++) { - assertEquals(requiredBuffers, queues[i].size()); - - final ArrayDeque expectedRecords = serializedRecords.clone(); - int assertRecords = 0; - for (int j = 0; j < requiredBuffers; j++) { - Buffer buffer = buildSingleBuffer(queues[i].remove()); - deserializer.setNextBuffer(buffer); - - assertRecords += DeserializationUtils.deserializeRecords(expectedRecords, deserializer); - } - Assert.assertEquals(numValues, assertRecords); + /** + * Creates the {@link RecordWriter} instance based on whether it is a broadcast writer. + */ + private RecordWriter createRecordWriter(ResultPartitionWriter writer) { + if (isBroadcastWriter) { + return new RecordWriterBuilder() + .setChannelSelector(new OutputEmitter(ShipStrategyType.BROADCAST, 0)) + .build(writer); + } else { + return new RecordWriterBuilder().build(writer); } } @@ -459,7 +480,7 @@ public class RecordWriterTest { /** * Partition writer that collects the added buffers/events in multiple queue. */ - private static class CollectingPartitionWriter implements ResultPartitionWriter { + static class CollectingPartitionWriter implements ResultPartitionWriter { private final Queue[] queues; private final BufferProvider bufferProvider; private final ResultPartitionID partitionId = new ResultPartitionID(); @@ -470,7 +491,7 @@ public class RecordWriterTest { * @param queues one queue per outgoing channel * @param bufferProvider buffer provider */ - private CollectingPartitionWriter(Queue[] queues, BufferProvider bufferProvider) { + CollectingPartitionWriter(Queue[] queues, BufferProvider bufferProvider) { this.queues = queues; this.bufferProvider = bufferProvider; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderAndConsumerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderAndConsumerTest.java index b5d9da0f1aa3e28fe347bca3852fb2861522ede6..3975a71f72056977e904927612ba7e4719823a08 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderAndConsumerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderAndConsumerTest.java @@ -57,6 +57,7 @@ public class BufferBuilderAndConsumerTest { @Test public void append() { BufferBuilder bufferBuilder = createBufferBuilder(); + BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer(); int[] intsToWrite = new int[] {0, 1, 2, 3, 42}; ByteBuffer bytesToWrite = toByteBuffer(intsToWrite); @@ -66,7 +67,7 @@ public class BufferBuilderAndConsumerTest { assertEquals(bytesToWrite.limit(), bytesToWrite.position()); assertFalse(bufferBuilder.isFull()); - assertContent(bufferBuilder.createBufferConsumer(), intsToWrite); + assertContent(bufferConsumer, intsToWrite); } @Test @@ -116,11 +117,21 @@ public class BufferBuilderAndConsumerTest { assertContent(bufferConsumer, 42); } - @Test(expected = IllegalStateException.class) + @Test public void creatingBufferConsumerTwice() { BufferBuilder bufferBuilder = createBufferBuilder(); - bufferBuilder.createBufferConsumer(); - bufferBuilder.createBufferConsumer(); + BufferConsumer bufferConsumer1 = bufferBuilder.createBufferConsumer(); + + assertEquals(0, bufferConsumer1.getCurrentReaderPosition()); + assertContent(bufferConsumer1); + + ByteBuffer bytesToWrite = toByteBuffer(0, 1); + bufferBuilder.appendAndCommit(bytesToWrite); + BufferConsumer bufferConsumer2 = bufferBuilder.createBufferConsumer(); + bufferBuilder.appendAndCommit(toByteBuffer(2)); + + assertEquals(bytesToWrite.position(), bufferConsumer2.getCurrentReaderPosition()); + assertContent(bufferConsumer2, 2); } @Test diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderTestUtils.java index 7696e08ad87dfe95de7a4a093d1c85ea1c821715..331854c4b8e9d1708c38d6010a9b4a206a6ea301 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderTestUtils.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/buffer/BufferBuilderTestUtils.java @@ -41,10 +41,6 @@ public class BufferBuilderTestUtils { return createFilledBufferBuilder(size, 0); } - public static BufferBuilder createFilledBufferBuilder(int dataSize) { - return createFilledBufferBuilder(BUFFER_SIZE, dataSize); - } - public static BufferBuilder createFilledBufferBuilder(int size, int dataSize) { checkArgument(size >= dataSize); BufferBuilder bufferBuilder = new BufferBuilder( @@ -70,14 +66,26 @@ public class BufferBuilderTestUtils { return buffer; } - public static BufferConsumer createFilledBufferConsumer(int size, int dataSize) { - BufferBuilder bufferBuilder = createFilledBufferBuilder(size, dataSize); - bufferBuilder.finish(); - return bufferBuilder.createBufferConsumer(); + public static BufferConsumer createFilledFinishedBufferConsumer(int dataSize) { + return createFilledBufferConsumer(dataSize, dataSize, true); } - public static BufferConsumer createFilledBufferConsumer(int dataSize) { - return createFilledBufferConsumer(BUFFER_SIZE, dataSize); + public static BufferConsumer createFilledUnfinishedBufferConsumer(int dataSize) { + return createFilledBufferConsumer(dataSize, dataSize, false); + } + + public static BufferConsumer createFilledBufferConsumer(int size, int dataSize, boolean isFinished) { + checkArgument(size >= dataSize); + + BufferBuilder bufferBuilder = createBufferBuilder(size); + BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer(); + fillBufferBuilder(bufferBuilder, dataSize); + + if (isFinished) { + bufferBuilder.finish(); + } + + return bufferConsumer; } public static BufferConsumer createEventBufferConsumer(int size) { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/BoundedBlockingSubpartitionAvailabilityTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/BoundedBlockingSubpartitionAvailabilityTest.java index 915cf43faed9246dda04ab336bbeef4ba76ba4a0..1110ccad534810be1962eccc1e6b99460c312af5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/BoundedBlockingSubpartitionAvailabilityTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/BoundedBlockingSubpartitionAvailabilityTest.java @@ -134,7 +134,7 @@ public class BoundedBlockingSubpartitionAvailabilityTest { private static void writeBuffers(ResultSubpartition partition, int numberOfBuffers) throws IOException { for (int i = 0; i < numberOfBuffers; i++) { - partition.add(BufferBuilderTestUtils.createFilledBufferConsumer(BUFFER_SIZE, BUFFER_SIZE)); + partition.add(BufferBuilderTestUtils.createFilledFinishedBufferConsumer(BUFFER_SIZE)); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/FileChannelBoundedDataTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/FileChannelBoundedDataTest.java index 1ca2bc8eeb05679fa9ec5a6412035391f046989a..c088d2542be2b6ec65e5ba22fd45836fe2c8e3a7 100755 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/FileChannelBoundedDataTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/FileChannelBoundedDataTest.java @@ -32,7 +32,7 @@ import java.io.IOException; import java.nio.file.Path; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.buildSomeBuffer; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -173,7 +173,7 @@ public class FileChannelBoundedDataTest extends BoundedDataTestBase { private static void writeBuffers(ResultSubpartition subpartition, int numberOfBuffers) throws IOException { for (int i = 0; i < numberOfBuffers; i++) { - subpartition.add(createFilledBufferConsumer(BUFFER_SIZE, BUFFER_SIZE)); + subpartition.add(createFilledFinishedBufferConsumer(BUFFER_SIZE)); } subpartition.finish(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputGateFairnessTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputGateFairnessTest.java index da05f8395438ac09aa0aa079ed8152fc61d6ebf8..45e33481570e696fdebf011c6e6449d86d0fa883 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputGateFairnessTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/InputGateFairnessTest.java @@ -47,7 +47,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.Optional; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer; import static org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createDummyConnectionManager; import static org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createLocalInputChannel; import static org.apache.flink.runtime.io.network.partition.InputChannelTestUtils.createResultPartitionManager; @@ -68,7 +68,7 @@ public class InputGateFairnessTest { final int buffersPerChannel = 27; final ResultPartition resultPartition = mock(ResultPartition.class); - final BufferConsumer bufferConsumer = createFilledBufferConsumer(42); + final BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42); // ----- create some source channels and fill them with buffers ----- @@ -122,7 +122,7 @@ public class InputGateFairnessTest { final int buffersPerChannel = 27; final ResultPartition resultPartition = mock(ResultPartition.class); - try (BufferConsumer bufferConsumer = createFilledBufferConsumer(42)) { + try (BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42)) { // ----- create some source channels and fill them with one buffer each ----- diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java index 12ab07754f09a81d31ff251ffa2400f108f5a7b8..fa8373544a466530f5aab804877b3c779667db53 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartialConsumePipelinedResultTest.java @@ -34,14 +34,17 @@ import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.runtime.testutils.MiniClusterResource; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.TestLogger; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; /** * Test for consuming a pipelined result only partially. */ +@Category(AlsoRunWithSchedulerNG.class) public class PartialConsumePipelinedResultTest extends TestLogger { // Test configuration diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartitionTestUtils.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartitionTestUtils.java index b1a58a00536d3a5f21fb5f0b3d08d622ab3e2b03..a29f50f197b64e7f6fc6ee8275a26e52c9bf17a4 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartitionTestUtils.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PartitionTestUtils.java @@ -31,7 +31,7 @@ import org.hamcrest.Matchers; import java.io.IOException; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -112,16 +112,12 @@ public enum PartitionTestUtils { true); } - public static ResultPartitionDeploymentDescriptor createPartitionDeploymentDescriptor() { - return createPartitionDeploymentDescriptor(ResultPartitionType.BLOCKING); - } - public static void writeBuffers( ResultPartitionWriter partition, int numberOfBuffers, int bufferSize) throws IOException { for (int i = 0; i < numberOfBuffers; i++) { - partition.addBufferConsumer(createFilledBufferConsumer(bufferSize, bufferSize), 0); + partition.addBufferConsumer(createFilledFinishedBufferConsumer(bufferSize), 0); } partition.finish(); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java index ff15b420c9367d7199cb0a0714aea2de45cca453..21e085d707bbede899c661979ca11cfca1cf7469 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionTest.java @@ -46,7 +46,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer; import static org.apache.flink.util.Preconditions.checkState; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -160,6 +160,7 @@ public class PipelinedSubpartitionTest extends SubpartitionTestBase { } final BufferBuilder bufferBuilder = bufferProvider.requestBufferBuilderBlocking(); + final BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer(); int segmentSize = bufferBuilder.getMaxCapacity(); MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(segmentSize); @@ -176,7 +177,7 @@ public class PipelinedSubpartitionTest extends SubpartitionTestBase { numberOfBuffers++; - return new BufferConsumerAndChannel(bufferBuilder.createBufferConsumer(), 0); + return new BufferConsumerAndChannel(bufferConsumer, 0); } }; @@ -250,8 +251,8 @@ public class PipelinedSubpartitionTest extends SubpartitionTestBase { private void testCleanupReleasedPartition(boolean createView) throws Exception { PipelinedSubpartition partition = createSubpartition(); - BufferConsumer buffer1 = createFilledBufferConsumer(4096); - BufferConsumer buffer2 = createFilledBufferConsumer(4096); + BufferConsumer buffer1 = createFilledFinishedBufferConsumer(4096); + BufferConsumer buffer2 = createFilledFinishedBufferConsumer(4096); boolean buffer1Recycled; boolean buffer2Recycled; try { @@ -306,7 +307,7 @@ public class PipelinedSubpartitionTest extends SubpartitionTestBase { private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition) throws Exception { // Add a bufferConsumer - BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE); + BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE); partition.add(bufferConsumer); partition.finish(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionWithReadViewTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionWithReadViewTest.java index e945342f99e68e5e5ed3d79d97e2699ed2b6c718..26d3f76052e3618ed6176bba39036a38ce096d90 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionWithReadViewTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/PipelinedSubpartitionWithReadViewTest.java @@ -31,12 +31,11 @@ import org.junit.Test; import javax.annotation.Nullable; import java.io.IOException; -import java.nio.ByteBuffer; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createBufferBuilder; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createEventBufferConsumer; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferBuilder; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledUnfinishedBufferConsumer; import static org.apache.flink.runtime.io.network.util.TestBufferFactory.BUFFER_SIZE; import static org.apache.flink.util.Preconditions.checkArgument; import static org.hamcrest.MatcherAssert.assertThat; @@ -106,9 +105,7 @@ public class PipelinedSubpartitionWithReadViewTest { public void testAddNonEmptyNotFinishedBuffer() throws Exception { assertEquals(0, availablityListener.getNumNotifications()); - BufferBuilder bufferBuilder = createBufferBuilder(); - bufferBuilder.appendAndCommit(ByteBuffer.allocate(1024)); - subpartition.add(bufferBuilder.createBufferConsumer()); + subpartition.add(createFilledUnfinishedBufferConsumer(1024)); // note that since the buffer builder is not finished, there is still a retained instance! assertEquals(0, subpartition.getBuffersInBacklog()); @@ -121,8 +118,8 @@ public class PipelinedSubpartitionWithReadViewTest { */ @Test public void testUnfinishedBufferBehindFinished() throws Exception { - subpartition.add(createFilledBufferConsumer(1025)); // finished - subpartition.add(createFilledBufferBuilder(1024).createBufferConsumer()); // not finished + subpartition.add(createFilledFinishedBufferConsumer(1025)); // finished + subpartition.add(createFilledUnfinishedBufferConsumer(1024)); // not finished assertEquals(1, subpartition.getBuffersInBacklog()); assertThat(availablityListener.getNumNotifications(), greaterThan(0L)); @@ -138,8 +135,8 @@ public class PipelinedSubpartitionWithReadViewTest { */ @Test public void testFlushWithUnfinishedBufferBehindFinished() throws Exception { - subpartition.add(createFilledBufferConsumer(1025)); // finished - subpartition.add(createFilledBufferBuilder(1024).createBufferConsumer()); // not finished + subpartition.add(createFilledFinishedBufferConsumer(1025)); // finished + subpartition.add(createFilledUnfinishedBufferConsumer(1024)); // not finished long oldNumNotifications = availablityListener.getNumNotifications(); assertEquals(1, subpartition.getBuffersInBacklog()); @@ -164,8 +161,8 @@ public class PipelinedSubpartitionWithReadViewTest { subpartition.flush(); assertEquals(0, availablityListener.getNumNotifications()); - subpartition.add(createFilledBufferConsumer(1025)); // finished - subpartition.add(createFilledBufferBuilder(1024).createBufferConsumer()); // not finished + subpartition.add(createFilledFinishedBufferConsumer(1025)); // finished + subpartition.add(createFilledUnfinishedBufferConsumer(1024)); // not finished assertEquals(1, subpartition.getBuffersInBacklog()); assertNextBuffer(readView, 1025, false, 0, false, true); @@ -187,18 +184,18 @@ public class PipelinedSubpartitionWithReadViewTest { public void testMultipleEmptyBuffers() throws Exception { assertEquals(0, availablityListener.getNumNotifications()); - subpartition.add(createFilledBufferConsumer(0)); + subpartition.add(createFilledFinishedBufferConsumer(0)); + assertEquals(0, availablityListener.getNumNotifications()); + subpartition.add(createFilledFinishedBufferConsumer(0)); assertEquals(1, availablityListener.getNumNotifications()); - subpartition.add(createFilledBufferConsumer(0)); - assertEquals(2, availablityListener.getNumNotifications()); - subpartition.add(createFilledBufferConsumer(0)); - assertEquals(2, availablityListener.getNumNotifications()); + subpartition.add(createFilledFinishedBufferConsumer(0)); + assertEquals(1, availablityListener.getNumNotifications()); assertEquals(2, subpartition.getBuffersInBacklog()); - subpartition.add(createFilledBufferConsumer(1024)); - assertEquals(2, availablityListener.getNumNotifications()); + subpartition.add(createFilledFinishedBufferConsumer(1024)); + assertEquals(1, availablityListener.getNumNotifications()); assertNextBuffer(readView, 1024, false, 0, false, true); } @@ -218,15 +215,14 @@ public class PipelinedSubpartitionWithReadViewTest { assertEquals(0, availablityListener.getNumNotifications()); // Add data to the queue... - subpartition.add(createFilledBufferConsumer(BUFFER_SIZE)); + subpartition.add(createFilledFinishedBufferConsumer(BUFFER_SIZE)); assertFalse(readView.nextBufferIsEvent()); assertEquals(1, subpartition.getTotalNumberOfBuffers()); assertEquals(0, subpartition.getBuffersInBacklog()); assertEquals(0, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer - // ...should have resulted in a notification - assertEquals(1, availablityListener.getNumNotifications()); + assertEquals(0, availablityListener.getNumNotifications()); // ...and one available result assertNextBuffer(readView, BUFFER_SIZE, false, 0, false, true); @@ -236,13 +232,13 @@ public class PipelinedSubpartitionWithReadViewTest { assertEquals(0, subpartition.getBuffersInBacklog()); // Add data to the queue... - subpartition.add(createFilledBufferConsumer(BUFFER_SIZE)); + subpartition.add(createFilledFinishedBufferConsumer(BUFFER_SIZE)); assertFalse(readView.nextBufferIsEvent()); assertEquals(2, subpartition.getTotalNumberOfBuffers()); assertEquals(0, subpartition.getBuffersInBacklog()); assertEquals(BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer - assertEquals(2, availablityListener.getNumNotifications()); + assertEquals(0, availablityListener.getNumNotifications()); assertNextBuffer(readView, BUFFER_SIZE, false, 0, false, true); assertEquals(2 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer @@ -253,17 +249,17 @@ public class PipelinedSubpartitionWithReadViewTest { // some tests with events // fill with: buffer, event, and buffer - subpartition.add(createFilledBufferConsumer(BUFFER_SIZE)); + subpartition.add(createFilledFinishedBufferConsumer(BUFFER_SIZE)); assertFalse(readView.nextBufferIsEvent()); subpartition.add(createEventBufferConsumer(BUFFER_SIZE)); assertFalse(readView.nextBufferIsEvent()); - subpartition.add(createFilledBufferConsumer(BUFFER_SIZE)); + subpartition.add(createFilledFinishedBufferConsumer(BUFFER_SIZE)); assertFalse(readView.nextBufferIsEvent()); assertEquals(5, subpartition.getTotalNumberOfBuffers()); assertEquals(1, subpartition.getBuffersInBacklog()); // two buffers (events don't count) assertEquals(2 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer - assertEquals(4, availablityListener.getNumNotifications()); + assertEquals(1, availablityListener.getNumNotifications()); // the first buffer assertNextBuffer(readView, BUFFER_SIZE, true, 0, true, true); @@ -271,7 +267,7 @@ public class PipelinedSubpartitionWithReadViewTest { assertEquals(0, subpartition.getBuffersInBacklog()); // the event - assertNextEvent(readView, BUFFER_SIZE, null, true, 0, false, true); + assertNextEvent(readView, BUFFER_SIZE, null, false, 0, false, true); assertEquals(4 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); // only updated when getting the buffer assertEquals(0, subpartition.getBuffersInBacklog()); @@ -286,7 +282,7 @@ public class PipelinedSubpartitionWithReadViewTest { assertEquals(5, subpartition.getTotalNumberOfBuffers()); assertEquals(5 * BUFFER_SIZE, subpartition.getTotalNumberOfBytes()); - assertEquals(4, availablityListener.getNumNotifications()); + assertEquals(1, availablityListener.getNumNotifications()); } @Test @@ -308,11 +304,10 @@ public class PipelinedSubpartitionWithReadViewTest { final int numberOfAddedBuffers = 5; for (int i = 1; i <= numberOfAddedBuffers; i++) { - final BufferBuilder bufferBuilder = createFilledBufferBuilder(1024, 10); - subpartition.add(bufferBuilder.createBufferConsumer()); - if (i < numberOfAddedBuffers || isFinished) { - bufferBuilder.finish(); + subpartition.add(createFilledFinishedBufferConsumer(1024)); + } else { + subpartition.add(createFilledUnfinishedBufferConsumer(1024)); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionTest.java index dc693e53a95e617f5c62e1d9a52bc17d10aa8db6..0b1e9a0c98c1bc5ecb01043b7fe1cc0f65ee4d92 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/ResultPartitionTest.java @@ -40,7 +40,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Collections; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer; import static org.apache.flink.runtime.io.network.partition.PartitionTestUtils.createPartition; import static org.apache.flink.runtime.io.network.partition.PartitionTestUtils.verifyCreateSubpartitionViewThrowsException; import static org.hamcrest.MatcherAssert.assertThat; @@ -90,7 +90,7 @@ public class ResultPartitionTest { taskActions, jobId, notifier); - consumableNotifyingPartitionWriter.addBufferConsumer(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE), 0); + consumableNotifyingPartitionWriter.addBufferConsumer(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE), 0); verify(notifier, times(1)) .notifyPartitionConsumable(eq(jobId), eq(consumableNotifyingPartitionWriter.getPartitionId()), eq(taskActions)); } @@ -103,7 +103,7 @@ public class ResultPartitionTest { taskActions, jobId, notifier); - partition.addBufferConsumer(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE), 0); + partition.addBufferConsumer(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE), 0); verify(notifier, never()).notifyPartitionConsumable(eq(jobId), eq(partition.getPartitionId()), eq(taskActions)); } } @@ -151,7 +151,7 @@ public class ResultPartitionTest { * @param partitionType the result partition type to set up */ private void testAddOnFinishedPartition(final ResultPartitionType partitionType) throws Exception { - BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE); + BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE); ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class); JobID jobId = new JobID(); TaskActions taskActions = new NoOpTaskActions(); @@ -197,7 +197,7 @@ public class ResultPartitionTest { * @param partitionType the result partition type to set up */ private void testAddOnReleasedPartition(final ResultPartitionType partitionType) throws Exception { - BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE); + BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE); ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class); JobID jobId = new JobID(); TaskActions taskActions = new NoOpTaskActions(); @@ -267,7 +267,7 @@ public class ResultPartitionTest { taskActions, jobId, notifier); - BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE); + BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE); try { // partition.add() adds the bufferConsumer without recycling it (if not spilling) consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java index 13cab65bb9c7197a2cf07c91e36ab2dbcdecaa5c..2fef1828e078cf0ab3cb4a3e797068ce9cf0ce16 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/SubpartitionTestBase.java @@ -24,7 +24,7 @@ import org.apache.flink.util.TestLogger; import org.junit.Test; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledBufferConsumer; +import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createFilledFinishedBufferConsumer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -70,7 +70,7 @@ public abstract class SubpartitionTestBase extends TestLogger { assertEquals(1, subpartition.getTotalNumberOfBuffers()); assertEquals(0, subpartition.getBuffersInBacklog()); - BufferConsumer bufferConsumer = createFilledBufferConsumer(4096, 4096); + BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(4096); assertFalse(subpartition.add(bufferConsumer)); assertTrue(bufferConsumer.isRecycled()); @@ -91,7 +91,7 @@ public abstract class SubpartitionTestBase extends TestLogger { try { subpartition.release(); - BufferConsumer bufferConsumer = createFilledBufferConsumer(4096, 4096); + BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(4096); assertFalse(subpartition.add(bufferConsumer)); assertTrue(bufferConsumer.isRecycled()); @@ -106,7 +106,7 @@ public abstract class SubpartitionTestBase extends TestLogger { @Test public void testReleasingReaderDoesNotReleasePartition() throws Exception { final ResultSubpartition partition = createSubpartition(); - partition.add(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE)); + partition.add(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE)); partition.finish(); final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener()); @@ -125,7 +125,7 @@ public abstract class SubpartitionTestBase extends TestLogger { @Test public void testReleaseIsIdempotent() throws Exception { final ResultSubpartition partition = createSubpartition(); - partition.add(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE)); + partition.add(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE)); partition.finish(); partition.release(); @@ -136,7 +136,7 @@ public abstract class SubpartitionTestBase extends TestLogger { @Test public void testReadAfterDispose() throws Exception { final ResultSubpartition partition = createSubpartition(); - partition.add(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE)); + partition.add(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE)); partition.finish(); final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener()); @@ -154,7 +154,7 @@ public abstract class SubpartitionTestBase extends TestLogger { public void testRecycleBufferAndConsumerOnFailure() throws Exception { final ResultSubpartition subpartition = createFailingWritesSubpartition(); try { - final BufferConsumer consumer = BufferBuilderTestUtils.createFilledBufferConsumer(100); + final BufferConsumer consumer = BufferBuilderTestUtils.createFilledFinishedBufferConsumer(100); try { subpartition.add(consumer); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateTestBase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateTestBase.java index b32ba89fd20b3a4d807daa16838b57bd27d59e2c..4069ec5faf8fc911aa85deca3bd3f6ba9b4ad432 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateTestBase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/InputGateTestBase.java @@ -18,7 +18,7 @@ package org.apache.flink.runtime.io.network.partition.consumer; -import org.apache.flink.runtime.io.AsyncDataInput; +import org.apache.flink.runtime.io.PullingAsyncDataInput; import org.apache.flink.runtime.io.network.NettyShuffleEnvironment; import org.apache.flink.runtime.io.network.partition.ResultPartitionType; @@ -69,7 +69,7 @@ public abstract class InputGateTestBase { assertTrue(isAvailable.isDone()); assertTrue(inputGateToTest.isAvailable().isDone()); - assertEquals(AsyncDataInput.AVAILABLE, inputGateToTest.isAvailable()); + assertEquals(PullingAsyncDataInput.AVAILABLE, inputGateToTest.isAvailable()); } protected void testIsAvailableAfterFinished( @@ -86,7 +86,7 @@ public abstract class InputGateTestBase { assertTrue(available.isDone()); assertTrue(inputGateToTest.isAvailable().isDone()); - assertEquals(AsyncDataInput.AVAILABLE, inputGateToTest.isAvailable()); + assertEquals(PullingAsyncDataInput.AVAILABLE, inputGateToTest.isAvailable()); } protected SingleInputGate createInputGate() { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/IteratorWrappingTestSingleInputGate.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/IteratorWrappingTestSingleInputGate.java index d691f3e22d8bf31c639fdb5d64ca39ef65cd8aed..4cbc6c8b235780b89c445697de55898f6f64335c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/IteratorWrappingTestSingleInputGate.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/IteratorWrappingTestSingleInputGate.java @@ -24,6 +24,7 @@ import org.apache.flink.runtime.io.network.api.serialization.EventSerializer; import org.apache.flink.runtime.io.network.api.serialization.RecordSerializer; import org.apache.flink.runtime.io.network.api.serialization.SpanningRecordSerializer; import org.apache.flink.runtime.io.network.buffer.BufferBuilder; +import org.apache.flink.runtime.io.network.buffer.BufferConsumer; import org.apache.flink.runtime.io.network.partition.consumer.InputChannel.BufferAndAvailability; import org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel.BufferAndAvailabilityProvider; import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID; @@ -33,7 +34,6 @@ import org.apache.flink.util.MutableObjectIterator; import java.io.IOException; import java.util.Optional; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.buildSingleBuffer; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createBufferBuilder; /** @@ -77,12 +77,13 @@ public class IteratorWrappingTestSingleInputGate e if (hasData) { serializer.serializeRecord(reuse); BufferBuilder bufferBuilder = createBufferBuilder(bufferSize); + BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer(); serializer.copyToBufferBuilder(bufferBuilder); hasData = inputIterator.next(reuse) != null; // Call getCurrentBuffer to ensure size is set - return Optional.of(new BufferAndAvailability(buildSingleBuffer(bufferBuilder), true, 0)); + return Optional.of(new BufferAndAvailability(bufferConsumer.build(), true, 0)); } else { inputChannel.setReleased(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java index dc8b501d87833d2e15ab53fcc876257ac037890a..d049f9b0a745171eed807c21e0d62774cea664a7 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/LocalInputChannelTest.java @@ -22,6 +22,7 @@ import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.execution.CancelTaskException; import org.apache.flink.runtime.io.network.TaskEventDispatcher; import org.apache.flink.runtime.io.network.buffer.BufferBuilder; +import org.apache.flink.runtime.io.network.buffer.BufferConsumer; import org.apache.flink.runtime.io.network.buffer.BufferPool; import org.apache.flink.runtime.io.network.buffer.BufferProvider; import org.apache.flink.runtime.io.network.buffer.NetworkBufferPool; @@ -465,9 +466,10 @@ public class LocalInputChannelTest { if (channelIndexes.size() > 0) { final int channelIndex = channelIndexes.remove(0); BufferBuilder bufferBuilder = bufferProvider.requestBufferBuilderBlocking(); + BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer(); bufferBuilder.appendAndCommit(ByteBuffer.wrap(new byte[4])); bufferBuilder.finish(); - return new BufferConsumerAndChannel(bufferBuilder.createBufferConsumer(), channelIndex); + return new BufferConsumerAndChannel(bufferConsumer, channelIndex); } return null; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java index 057ac5ecc5c19df3699d0922122d9478b19db9e7..57f0a17b2392dfd7a9852ee50f1e3c8f3e4c77bb 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/SingleInputGateTest.java @@ -594,7 +594,7 @@ public class SingleInputGateTest extends InputGateTestBase { remoteInputChannel.onBuffer(TestBufferFactory.createBuffer(1), 0, 0); assertEquals(1, inputGate.getNumberOfQueuedBuffers()); - resultPartition.addBufferConsumer(BufferBuilderTestUtils.createFilledBufferConsumer(1), 0); + resultPartition.addBufferConsumer(BufferBuilderTestUtils.createFilledFinishedBufferConsumer(1), 0); assertEquals(2, inputGate.getNumberOfQueuedBuffers()); } finally { resultPartition.release(); @@ -605,7 +605,7 @@ public class SingleInputGateTest extends InputGateTestBase { /** * Tests that if the {@link PartitionNotFoundException} is set onto one {@link InputChannel}, - * then it would be thrown directly via {@link SingleInputGate#getNextBufferOrEvent()}. So we + * then it would be thrown directly via {@link SingleInputGate#getNext()}. So we * could confirm the {@link SingleInputGate} would not swallow or transform the original exception. */ @Test diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/BlobsCleanupITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/BlobsCleanupITCase.java index bb2aede627ff91eb62eeb6dfcebd2eaa99c6d045..bbc57333b8975c0a2700ef300a805e72d9203913 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/BlobsCleanupITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/BlobsCleanupITCase.java @@ -38,6 +38,7 @@ import org.apache.flink.runtime.testtasks.FailingBlockingInvokable; import org.apache.flink.runtime.testtasks.NoOpInvokable; import org.apache.flink.runtime.testutils.MiniClusterResource; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.TestLogger; @@ -45,6 +46,7 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.junit.rules.TemporaryFolder; import javax.annotation.Nonnull; @@ -69,6 +71,7 @@ import static org.junit.Assert.fail; * Small test to check that the {@link org.apache.flink.runtime.blob.BlobServer} cleanup is executed * after job termination. */ +@Category(AlsoRunWithSchedulerNG.class) public class BlobsCleanupITCase extends TestLogger { private static final long RETRY_INTERVAL = 100L; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/SlotCountExceedingParallelismTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/SlotCountExceedingParallelismTest.java index faf2dfe9242971dc9e7b4aefe84002f8ac1392d9..8e6a22f0f9b2307640e046edf64447394803ef1c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/SlotCountExceedingParallelismTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/SlotCountExceedingParallelismTest.java @@ -33,11 +33,13 @@ import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.runtime.testutils.MiniClusterResource; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.types.IntValue; import org.apache.flink.util.TestLogger; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.util.BitSet; @@ -46,6 +48,7 @@ import java.util.BitSet; * of slots. This effectively tests that Flink can execute jobs with blocking results * in a staged fashion. */ +@Category(AlsoRunWithSchedulerNG.class) public class SlotCountExceedingParallelismTest extends TestLogger { // Test configuration @@ -141,7 +144,7 @@ public class SlotCountExceedingParallelismTest extends TestLogger { @Override public void invoke() throws Exception { - RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); + RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); final int numberOfTimesToSend = getTaskConfiguration().getInteger(CONFIG_KEY, 0); final IntValue subtaskIndex = new IntValue( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java index 58af0e20d610518be2f6804bbf496d9221d98018..cbd2fae49faf77a0af138acf7897fa913d02d694 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmanager/scheduler/ScheduleOrUpdateConsumersTest.java @@ -32,6 +32,7 @@ import org.apache.flink.runtime.jobmanager.SlotCountExceedingParallelismTest; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.runtime.testutils.MiniClusterResource; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.types.IntValue; import org.apache.flink.util.TestLogger; @@ -39,6 +40,7 @@ import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.util.List; @@ -48,6 +50,7 @@ import static org.apache.flink.runtime.jobmanager.SlotCountExceedingParallelismT * Tests for the lazy scheduling/updating of consumers depending on the * producers result. */ +@Category(AlsoRunWithSchedulerNG.class) public class ScheduleOrUpdateConsumersTest extends TestLogger { private static final int NUMBER_OF_TMS = 2; @@ -148,8 +151,8 @@ public class ScheduleOrUpdateConsumersTest extends TestLogger { // The order of intermediate result creation in the job graph specifies which produced // result partition is pipelined/blocking. - final RecordWriter pipelinedWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); - final RecordWriter blockingWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(1)); + final RecordWriter pipelinedWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); + final RecordWriter blockingWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(1)); writers.add(pipelinedWriter); writers.add(blockingWriter); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobExecutionITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobExecutionITCase.java index e19fe2020ddaa9ff381d4407e74c4ea0e85ba0ee..03f36b80295ce2ed7124f962bb9deb5e8d184dce 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobExecutionITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobExecutionITCase.java @@ -25,9 +25,11 @@ import org.apache.flink.runtime.jobgraph.JobVertex; import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup; import org.apache.flink.runtime.minicluster.TestingMiniCluster; import org.apache.flink.runtime.minicluster.TestingMiniClusterConfiguration; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.TestLogger; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.util.concurrent.CompletableFuture; @@ -37,6 +39,7 @@ import static org.junit.Assert.assertThat; /** * Integration tests for job scheduling. */ +@Category(AlsoRunWithSchedulerNG.class) public class JobExecutionITCase extends TestLogger { /** diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerImplTest.java similarity index 93% rename from flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java rename to flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerImplTest.java index b560600e6a57ee6bfb88f3a184c4169592219931..23c4e31f17ac7c2778e5b889ec315b86127d6252 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobManagerRunnerImplTest.java @@ -64,9 +64,9 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; /** - * Tests for the {@link JobManagerRunner}. + * Tests for the {@link JobManagerRunnerImpl}. */ -public class JobManagerRunnerTest extends TestLogger { +public class JobManagerRunnerImplTest extends TestLogger { @ClassRule public static TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -129,7 +129,7 @@ public class JobManagerRunnerTest extends TestLogger { @Test public void testJobCompletion() throws Exception { - final JobManagerRunner jobManagerRunner = createJobManagerRunner(); + final JobManagerRunnerImpl jobManagerRunner = createJobManagerRunner(); try { jobManagerRunner.start(); @@ -148,7 +148,7 @@ public class JobManagerRunnerTest extends TestLogger { @Test public void testJobFinishedByOther() throws Exception { - final JobManagerRunner jobManagerRunner = createJobManagerRunner(); + final JobManagerRunnerImpl jobManagerRunner = createJobManagerRunner(); try { jobManagerRunner.start(); @@ -217,7 +217,7 @@ public class JobManagerRunnerTest extends TestLogger { } /** - * Tests that the {@link JobManagerRunner} always waits for the previous leadership operation + * Tests that the {@link JobManagerRunnerImpl} always waits for the previous leadership operation * (granting or revoking leadership) to finish before starting a new leadership operation. */ @Test @@ -254,7 +254,7 @@ public class JobManagerRunnerTest extends TestLogger { } /** - * Tests that the {@link JobManagerRunner} always waits for the previous leadership operation + * Tests that the {@link JobManagerRunnerImpl} always waits for the previous leadership operation * (granting or revoking leadership) to finish before starting a new leadership operation. */ @Test @@ -299,7 +299,7 @@ public class JobManagerRunnerTest extends TestLogger { } @Nonnull - private JobManagerRunner createJobManagerRunner() throws Exception { + private JobManagerRunnerImpl createJobManagerRunner() throws Exception { return createJobManagerRunner(defaultJobMasterServiceFactory, libraryCacheManager); } @@ -309,8 +309,8 @@ public class JobManagerRunnerTest extends TestLogger { } @Nonnull - private JobManagerRunner createJobManagerRunner(JobMasterServiceFactory jobMasterServiceFactory, LibraryCacheManager libraryCacheManager) throws Exception{ - return new JobManagerRunner( + private JobManagerRunnerImpl createJobManagerRunner(JobMasterServiceFactory jobMasterServiceFactory, LibraryCacheManager libraryCacheManager) throws Exception{ + return new JobManagerRunnerImpl( jobGraph, jobMasterServiceFactory, haServices, diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobRecoveryITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobRecoveryITCase.java index d003912eae353b73a0c381379c789b7d77148306..d9f38d370dadd0d100e226db9e35749aa2aa1438 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobRecoveryITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/JobRecoveryITCase.java @@ -29,11 +29,13 @@ import org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup; import org.apache.flink.runtime.minicluster.MiniCluster; import org.apache.flink.runtime.testutils.MiniClusterResource; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.FlinkRuntimeException; import org.apache.flink.util.TestLogger; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.io.IOException; import java.util.concurrent.CompletableFuture; @@ -44,6 +46,7 @@ import static org.junit.Assert.assertThat; /** * Tests for the recovery of task failures. */ +@Category(AlsoRunWithSchedulerNG.class) public class JobRecoveryITCase extends TestLogger { private static final int NUM_TMS = 1; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingAbstractInvokables.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingAbstractInvokables.java index e97dec778ad512306fde81dc7496582a43ffb3b6..23e5dcee8939b62115a2f28efebb08bf9e44da6d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingAbstractInvokables.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingAbstractInvokables.java @@ -47,7 +47,7 @@ public class TestingAbstractInvokables { @Override public void invoke() throws Exception { - final RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); + final RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); try { writer.emit(new IntValue(42)); @@ -97,7 +97,7 @@ public class TestingAbstractInvokables { @Override public void invoke() throws Exception { final Object o = new Object(); - RecordWriter recordWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); + RecordWriter recordWriter = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); for (int i = 0; i < 1024; i++) { recordWriter.emit(new IntValue(42)); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingJobManagerRunner.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingJobManagerRunner.java new file mode 100644 index 0000000000000000000000000000000000000000..b0a0b9c540fd5c586e19775f8ce9accebe38db7b --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingJobManagerRunner.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.jobmaster; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.runtime.executiongraph.ArchivedExecutionGraph; + +import java.util.concurrent.CompletableFuture; + +/** + * Testing implementation of the {@link JobManagerRunner}. + */ +public class TestingJobManagerRunner implements JobManagerRunner { + + private final JobID jobId; + + private final boolean blockingTermination; + + private final CompletableFuture resultFuture; + + private final CompletableFuture jobMasterGatewayFuture; + + private final CompletableFuture terminationFuture; + + public TestingJobManagerRunner(JobID jobId) { + this(jobId, false); + } + + public TestingJobManagerRunner(JobID jobId, boolean blockingTermination) { + this.jobId = jobId; + this.blockingTermination = blockingTermination; + this.resultFuture = new CompletableFuture<>(); + this.jobMasterGatewayFuture = new CompletableFuture<>(); + this.terminationFuture = new CompletableFuture<>(); + + terminationFuture.whenComplete((ignored, ignoredThrowable) -> resultFuture.completeExceptionally(new JobNotFinishedException(jobId))); + } + + @Override + public void start() throws Exception {} + + @Override + public CompletableFuture getJobMasterGateway() { + return jobMasterGatewayFuture; + } + + @Override + public CompletableFuture getResultFuture() { + return resultFuture; + } + + @Override + public JobID getJobID() { + return jobId; + } + + @Override + public CompletableFuture closeAsync() { + if (!blockingTermination) { + terminationFuture.complete(null); + } + + return terminationFuture; + } + + public void completeResultFuture(ArchivedExecutionGraph archivedExecutionGraph) { + resultFuture.complete(archivedExecutionGraph); + } + + public void completeResultFutureExceptionally(Exception e) { + resultFuture.completeExceptionally(e); + } + + public void completeTerminationFuture() { + terminationFuture.complete(null); + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingLogicalSlot.java b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingLogicalSlot.java index 3304c961891c64e6e741b1b57152d51b50031546..ff680e36c7cb30d572e9ce8e68b2b1477fa81dd0 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingLogicalSlot.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/jobmaster/TestingLogicalSlot.java @@ -53,6 +53,7 @@ public class TestingLogicalSlot implements LogicalSlot { private final SlotRequestId slotRequestId; + @Nullable private final SlotSharingGroupId slotSharingGroupId; TestingLogicalSlot( @@ -61,7 +62,7 @@ public class TestingLogicalSlot implements LogicalSlot { int slotNumber, AllocationID allocationId, SlotRequestId slotRequestId, - SlotSharingGroupId slotSharingGroupId, + @Nullable SlotSharingGroupId slotSharingGroupId, boolean automaticallyCompleteReleaseFuture, SlotOwner slotOwner) { @@ -71,10 +72,10 @@ public class TestingLogicalSlot implements LogicalSlot { this.slotNumber = slotNumber; this.allocationId = Preconditions.checkNotNull(allocationId); this.slotRequestId = Preconditions.checkNotNull(slotRequestId); - this.slotSharingGroupId = Preconditions.checkNotNull(slotSharingGroupId); + this.slotSharingGroupId = slotSharingGroupId; this.releaseFuture = new CompletableFuture<>(); this.automaticallyCompleteReleaseFuture = automaticallyCompleteReleaseFuture; - this.slotOwner = slotOwner; + this.slotOwner = Preconditions.checkNotNull(slotOwner); } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeClusterComponentsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeClusterComponentsTest.java index 1d62b19f5f380ca8a3b1f939a758069a6b46fbd2..fbb1bad9f3796f3a91bf268a2bc677fb7a5837af 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeClusterComponentsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderChangeClusterComponentsTest.java @@ -21,7 +21,6 @@ package org.apache.flink.runtime.leaderelection; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.JobSubmissionResult; import org.apache.flink.api.common.time.Deadline; -import org.apache.flink.api.common.time.Time; import org.apache.flink.runtime.execution.Environment; import org.apache.flink.runtime.highavailability.nonha.embedded.TestingEmbeddedHaServices; import org.apache.flink.runtime.jobgraph.JobGraph; @@ -34,6 +33,7 @@ import org.apache.flink.runtime.minicluster.TestingMiniClusterConfiguration; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.runtime.testutils.CommonTestUtils; import org.apache.flink.runtime.util.LeaderRetrievalUtils; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.TestLogger; @@ -41,6 +41,7 @@ import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.time.Duration; import java.util.concurrent.CompletableFuture; @@ -54,6 +55,7 @@ import static org.junit.Assert.fail; /** * Tests which verify the cluster behaviour in case of leader changes. */ +@Category(AlsoRunWithSchedulerNG.class) public class LeaderChangeClusterComponentsTest extends TestLogger { private static final Duration TESTING_TIMEOUT = Duration.ofMinutes(2L); @@ -157,7 +159,11 @@ public class LeaderChangeClusterComponentsTest extends TestLogger { highAvailabilityServices.grantResourceManagerLeadership(); // wait for the ResourceManager to confirm the leadership - assertThat(LeaderRetrievalUtils.retrieveLeaderConnectionInfo(highAvailabilityServices.getResourceManagerLeaderRetriever(), Time.minutes(TESTING_TIMEOUT.toMinutes())).getLeaderSessionID(), is(notNullValue())); + assertThat( + LeaderRetrievalUtils.retrieveLeaderConnectionInfo( + highAvailabilityServices.getResourceManagerLeaderRetriever(), + TESTING_TIMEOUT).getLeaderSessionId(), + is(notNullValue())); waitUntilTaskExecutorsHaveConnected(NUM_TMS, deadline); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderElectionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderElectionTest.java index 44016f82cc546d6800341bc9bad48b3f2edd8a58..a1d99cdb4f34b6d7b6cd5ee77e27ee8317a4115a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderElectionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/LeaderElectionTest.java @@ -101,7 +101,7 @@ public class LeaderElectionTest extends TestLogger { assertThat(leaderElectionService.hasLeadership(leaderSessionId), is(true)); assertThat(leaderElectionService.hasLeadership(UUID.randomUUID()), is(false)); - leaderElectionService.confirmLeaderSessionID(leaderSessionId); + leaderElectionService.confirmLeadership(leaderSessionId, "foobar"); assertThat(leaderElectionService.hasLeadership(leaderSessionId), is(true)); @@ -132,7 +132,7 @@ public class LeaderElectionTest extends TestLogger { } @Override - public String getAddress() { + public String getDescription() { return "foobar"; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingContender.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingContender.java index b03c1654c7cec7d3b213ff0d2edd8a31ed5e81cb..3775d996357a3783edf7e59b5539d05ef5626039 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingContender.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingContender.java @@ -118,7 +118,7 @@ public class TestingContender implements LeaderContender { this.leaderSessionID = leaderSessionID; - leaderElectionService.confirmLeaderSessionID(leaderSessionID); + leaderElectionService.confirmLeadership(leaderSessionID, address); leader = true; @@ -139,7 +139,7 @@ public class TestingContender implements LeaderContender { } @Override - public String getAddress() { + public String getDescription() { return address; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java index e60f2f776ab24f61cab8d77621038cef01608deb..8d25f609b05011c6e95f7fe2e7d7fee6a1a65899 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/TestingLeaderElectionService.java @@ -18,6 +18,8 @@ package org.apache.flink.runtime.leaderelection; +import org.apache.flink.runtime.util.LeaderConnectionInfo; + import javax.annotation.Nonnull; import java.util.UUID; @@ -31,7 +33,7 @@ public class TestingLeaderElectionService implements LeaderElectionService { private LeaderContender contender = null; private boolean hasLeadership = false; - private CompletableFuture confirmationFuture = null; + private CompletableFuture confirmationFuture = null; private CompletableFuture startFuture = new CompletableFuture<>(); private UUID issuedLeaderSessionId = null; @@ -40,7 +42,7 @@ public class TestingLeaderElectionService implements LeaderElectionService { * *

    Note: the future is created upon calling {@link #isLeader(UUID)}. */ - public synchronized CompletableFuture getConfirmationFuture() { + public synchronized CompletableFuture getConfirmationFuture() { return confirmationFuture; } @@ -67,9 +69,9 @@ public class TestingLeaderElectionService implements LeaderElectionService { } @Override - public synchronized void confirmLeaderSessionID(UUID leaderSessionID) { + public synchronized void confirmLeadership(UUID leaderSessionID, String leaderAddress) { if (confirmationFuture != null) { - confirmationFuture.complete(leaderSessionID); + confirmationFuture.complete(new LeaderConnectionInfo(leaderSessionID, leaderAddress)); } } @@ -90,7 +92,7 @@ public class TestingLeaderElectionService implements LeaderElectionService { contender.grantLeadership(leaderSessionID); } - return confirmationFuture; + return confirmationFuture.thenApply(LeaderConnectionInfo::getLeaderSessionId); } public synchronized void notLeader() { @@ -102,8 +104,8 @@ public class TestingLeaderElectionService implements LeaderElectionService { } public synchronized String getAddress() { - if (contender != null) { - return contender.getAddress(); + if (confirmationFuture.isDone()) { + return confirmationFuture.join().getAddress(); } else { throw new IllegalStateException("TestingLeaderElectionService has not been started."); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java index b476896b5d4064390cd5227a769db87d7235c713..bf5b2c8d3d6f59fe63e9c85a082dfd5491247ee5 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderElectionTest.java @@ -43,6 +43,8 @@ import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nonnull; + import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.ObjectOutputStream; @@ -175,7 +177,7 @@ public class ZooKeeperLeaderElectionTest extends TestLogger { for (int i = 0; i < num; i++) { leaderElectionService[i] = ZooKeeperUtils.createLeaderElectionService(client, configuration); - contenders[i] = new TestingContender(TEST_URL + "_" + i, leaderElectionService[i]); + contenders[i] = new TestingContender(createAddress(i), leaderElectionService[i]); LOG.debug("Start leader election service for contender #{}.", i); @@ -199,7 +201,7 @@ public class ZooKeeperLeaderElectionTest extends TestLogger { TestingContender contender = contenders[index]; // check that the retrieval service has retrieved the correct leader - if (address.equals(contender.getAddress()) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) { + if (address.equals(createAddress(index)) && listener.getLeaderSessionID().equals(contender.getLeaderSessionID())) { // kill the election service of the leader LOG.debug("Stop leader election service of contender #{}.", numberSeenLeaders); leaderElectionService[index].stop(); @@ -228,6 +230,11 @@ public class ZooKeeperLeaderElectionTest extends TestLogger { } } + @Nonnull + private String createAddress(int i) { + return TEST_URL + "_" + i; + } + /** * Tests the repeated reelection of {@link LeaderContender} once the current leader dies. * Furthermore, it tests that new LeaderElectionServices can be started later on and that they @@ -367,7 +374,7 @@ public class ZooKeeperLeaderElectionTest extends TestLogger { } assertEquals(listener2.getLeaderSessionID(), contender.getLeaderSessionID()); - assertEquals(listener2.getAddress(), contender.getAddress()); + assertEquals(listener2.getAddress(), TEST_URL); } finally { if (leaderElectionService != null) { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java index d4e5dd98e9b5ebde7101ffe84f7206698dbfcdad..d98292b90b221fbbdade965eb7bfa7b12543f1ff 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/leaderelection/ZooKeeperLeaderRetrievalTest.java @@ -45,9 +45,7 @@ import java.net.ServerSocket; import java.net.Socket; import java.net.SocketAddress; import java.net.UnknownHostException; -import java.util.concurrent.TimeUnit; - -import scala.concurrent.duration.FiniteDuration; +import java.time.Duration; import static org.junit.Assert.assertEquals; @@ -102,7 +100,7 @@ public class ZooKeeperLeaderRetrievalTest extends TestLogger{ */ @Test public void testConnectingAddressRetrievalWithDelayedLeaderElection() throws Exception { - FiniteDuration timeout = new FiniteDuration(1, TimeUnit.MINUTES); + Duration timeout = Duration.ofMinutes(1L); long sleepingTime = 1000; @@ -197,7 +195,7 @@ public class ZooKeeperLeaderRetrievalTest extends TestLogger{ */ @Test public void testTimeoutOfFindConnectingAddress() throws Exception { - FiniteDuration timeout = new FiniteDuration(1L, TimeUnit.SECONDS); + Duration timeout = Duration.ofSeconds(1L); LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID); InetAddress result = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, timeout); @@ -207,14 +205,14 @@ public class ZooKeeperLeaderRetrievalTest extends TestLogger{ static class FindConnectingAddress implements Runnable { - private final FiniteDuration timeout; + private final Duration timeout; private final LeaderRetrievalService leaderRetrievalService; private InetAddress result; private Exception exception; public FindConnectingAddress( - FiniteDuration timeout, + Duration timeout, LeaderRetrievalService leaderRetrievalService) { this.timeout = timeout; this.leaderRetrievalService = leaderRetrievalService; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/MetricRegistryImplTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/MetricRegistryImplTest.java index 6089c936bdb6346976bcc9246af9dd568368304a..0f69aca2d25fa3dac954cba3e697138c64c9358a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/MetricRegistryImplTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/MetricRegistryImplTest.java @@ -45,13 +45,12 @@ import org.apache.flink.util.TestLogger; import org.junit.Assert; import org.junit.Test; +import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; -import scala.concurrent.duration.FiniteDuration; - import static org.apache.flink.util.Preconditions.checkNotNull; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -366,7 +365,7 @@ public class MetricRegistryImplTest extends TestLogger { */ @Test public void testQueryActorShutdown() throws Exception { - final FiniteDuration timeout = new FiniteDuration(10L, TimeUnit.SECONDS); + final Duration timeout = Duration.ofSeconds(10L); MetricRegistryImpl registry = new MetricRegistryImpl(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java index e31c3792bee08f1e47b763861608d3968a803798..790e85f6e2693403a2ec53044338693531c51962 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/metrics/groups/AbstractMetricGroupTest.java @@ -34,6 +34,7 @@ import org.apache.flink.runtime.metrics.ReporterSetup; import org.apache.flink.runtime.metrics.dump.QueryScopeInfo; import org.apache.flink.runtime.metrics.scope.ScopeFormats; import org.apache.flink.runtime.metrics.util.TestReporter; +import org.apache.flink.util.TestLogger; import org.junit.Test; @@ -45,7 +46,7 @@ import static org.junit.Assert.assertTrue; /** * Tests for the {@link AbstractMetricGroup}. */ -public class AbstractMetricGroupTest { +public class AbstractMetricGroupTest extends TestLogger { /** * Verifies that no {@link NullPointerException} is thrown when {@link AbstractMetricGroup#getAllVariables()} is * called and the parent is null. diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java index 3744f11618415a93339c5cf34985f664e73285d4..62878154093850d30e8c38f75343f8300163de55 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/MiniClusterITCase.java @@ -46,9 +46,11 @@ import org.apache.flink.runtime.jobmaster.TestingAbstractInvokables.Sender; import org.apache.flink.runtime.testtasks.BlockingNoOpInvokable; import org.apache.flink.runtime.testtasks.NoOpInvokable; import org.apache.flink.runtime.testtasks.WaitingNoOpInvokable; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.TestLogger; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.io.IOException; import java.util.concurrent.CompletableFuture; @@ -62,6 +64,7 @@ import static org.junit.Assert.fail; /** * Integration test cases for the {@link MiniCluster}. */ +@Category(AlsoRunWithSchedulerNG.class) public class MiniClusterITCase extends TestLogger { @Test @@ -110,7 +113,12 @@ public class MiniClusterITCase extends TestLogger { } catch (JobExecutionException e) { assertTrue(findThrowableWithMessage(e, "Job execution failed.").isPresent()); assertTrue(findThrowable(e, NoResourceAvailableException.class).isPresent()); - assertTrue(findThrowableWithMessage(e, "Slots required: 2, slots allocated: 1").isPresent()); + + //TODO: remove the legacy scheduler message check once legacy scheduler is removed + final String legacySchedulerErrorMessage = "Slots required: 2, slots allocated: 1"; + final String ngSchedulerErrorMessage = "Could not allocate the required slot within slot request timeout"; + assertTrue(findThrowableWithMessage(e, legacySchedulerErrorMessage).isPresent() || + findThrowableWithMessage(e, ngSchedulerErrorMessage).isPresent()); } } @@ -122,7 +130,12 @@ public class MiniClusterITCase extends TestLogger { } catch (JobExecutionException e) { assertTrue(findThrowableWithMessage(e, "Job execution failed.").isPresent()); assertTrue(findThrowable(e, NoResourceAvailableException.class).isPresent()); - assertTrue(findThrowableWithMessage(e, "Could not allocate enough slots").isPresent()); + + //TODO: remove the legacy scheduler message check once legacy scheduler is removed + final String legacySchedulerErrorMessage = "Could not allocate enough slots"; + final String ngSchedulerErrorMessage = "Could not allocate the required slot within slot request timeout"; + assertTrue(findThrowableWithMessage(e, legacySchedulerErrorMessage).isPresent() || + findThrowableWithMessage(e, ngSchedulerErrorMessage).isPresent()); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/SessionDispatcherWithUUIDFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/SessionDispatcherWithUUIDFactory.java index e735dfe0792705a0453fa7d11e549673c11d4acd..307c3cfeb38dc2cb6d16465a9b79bd798dd7efca 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/SessionDispatcherWithUUIDFactory.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/SessionDispatcherWithUUIDFactory.java @@ -18,58 +18,30 @@ package org.apache.flink.runtime.minicluster; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.blob.BlobServer; -import org.apache.flink.runtime.dispatcher.ArchivedExecutionGraphStore; import org.apache.flink.runtime.dispatcher.DefaultJobManagerRunnerFactory; -import org.apache.flink.runtime.dispatcher.Dispatcher; import org.apache.flink.runtime.dispatcher.DispatcherFactory; -import org.apache.flink.runtime.dispatcher.HistoryServerArchivist; +import org.apache.flink.runtime.dispatcher.DispatcherServices; +import org.apache.flink.runtime.dispatcher.PartialDispatcherServices; import org.apache.flink.runtime.dispatcher.StandaloneDispatcher; -import org.apache.flink.runtime.heartbeat.HeartbeatServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; -import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; -import org.apache.flink.runtime.webmonitor.retriever.GatewayRetriever; -import javax.annotation.Nullable; +import javax.annotation.Nonnull; /** * {@link DispatcherFactory} which creates a {@link StandaloneDispatcher} which has an * endpoint id with a random UUID suffix. */ -public enum SessionDispatcherWithUUIDFactory implements DispatcherFactory { +public enum SessionDispatcherWithUUIDFactory implements DispatcherFactory { INSTANCE; @Override - public Dispatcher createDispatcher( - Configuration configuration, - RpcService rpcService, - HighAvailabilityServices highAvailabilityServices, - GatewayRetriever resourceManagerGatewayRetriever, - BlobServer blobServer, - HeartbeatServices heartbeatServices, - JobManagerMetricGroup jobManagerMetricGroup, - @Nullable String metricQueryServiceAddress, - ArchivedExecutionGraphStore archivedExecutionGraphStore, - FatalErrorHandler fatalErrorHandler, - HistoryServerArchivist historyServerArchivist) throws Exception { + public StandaloneDispatcher createDispatcher( + @Nonnull RpcService rpcService, + @Nonnull PartialDispatcherServices partialDispatcherServices) throws Exception { // create the default dispatcher return new StandaloneDispatcher( rpcService, generateEndpointIdWithUUID(), - configuration, - highAvailabilityServices, - resourceManagerGatewayRetriever, - blobServer, - heartbeatServices, - jobManagerMetricGroup, - metricQueryServiceAddress, - archivedExecutionGraphStore, - DefaultJobManagerRunnerFactory.INSTANCE, - fatalErrorHandler, - historyServerArchivist); + DispatcherServices.from(partialDispatcherServices, DefaultJobManagerRunnerFactory.INSTANCE)); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/StandaloneResourceManagerWithUUIDFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/StandaloneResourceManagerWithUUIDFactory.java index 4719c4bcd9ab259a9c44c4d33fca3abfb40b8f82..40a306f2a5f925f63e9721028a1fd20c312374f7 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/StandaloneResourceManagerWithUUIDFactory.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/StandaloneResourceManagerWithUUIDFactory.java @@ -25,8 +25,7 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.ResourceManager; import org.apache.flink.runtime.resourcemanager.ResourceManagerFactory; import org.apache.flink.runtime.resourcemanager.ResourceManagerRuntimeServices; @@ -51,11 +50,10 @@ public enum StandaloneResourceManagerWithUUIDFactory implements ResourceManagerF RpcService rpcService, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, - MetricRegistry metricRegistry, FatalErrorHandler fatalErrorHandler, ClusterInformation clusterInformation, @Nullable String webInterfaceUrl, - JobManagerMetricGroup jobManagerMetricGroup) throws Exception { + ResourceManagerMetricGroup resourceManagerMetricGroup) throws Exception { final ResourceManagerRuntimeServicesConfiguration resourceManagerRuntimeServicesConfiguration = ResourceManagerRuntimeServicesConfiguration.fromConfiguration(configuration); final ResourceManagerRuntimeServices resourceManagerRuntimeServices = ResourceManagerRuntimeServices.fromConfiguration( resourceManagerRuntimeServicesConfiguration, @@ -71,11 +69,10 @@ public enum StandaloneResourceManagerWithUUIDFactory implements ResourceManagerF highAvailabilityServices, heartbeatServices, resourceManagerRuntimeServices.getSlotManager(), - metricRegistry, resourceManagerRuntimeServices.getJobLeaderIdService(), clusterInformation, fatalErrorHandler, - jobManagerMetricGroup, + resourceManagerMetricGroup, standaloneClusterStartupPeriodTime); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/TestingMiniCluster.java b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/TestingMiniCluster.java index 3b863137b038e80f741f4e2c4fd609b13503fd1c..44c2d947794ad20baa55efdae8b758699bb7667c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/TestingMiniCluster.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/minicluster/TestingMiniCluster.java @@ -22,8 +22,10 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.blob.BlobServer; import org.apache.flink.runtime.dispatcher.DispatcherGateway; import org.apache.flink.runtime.dispatcher.MemoryArchivedExecutionGraphStore; +import org.apache.flink.runtime.dispatcher.runner.DispatcherRunnerFactoryImpl; import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponent; -import org.apache.flink.runtime.entrypoint.component.SessionDispatcherResourceManagerComponentFactory; +import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponentFactory; +import org.apache.flink.runtime.entrypoint.component.TestingDefaultDispatcherResourceManagerComponentFactory; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.metrics.MetricRegistry; @@ -67,7 +69,7 @@ public class TestingMiniCluster extends MiniCluster { @Nonnull @Override - public Collection> getDispatcherResourceManagerComponents() { + public Collection getDispatcherResourceManagerComponents() { return super.getDispatcherResourceManagerComponents(); } @@ -97,7 +99,7 @@ public class TestingMiniCluster extends MiniCluster { } @Override - protected Collection> createDispatcherResourceManagerComponents( + protected Collection createDispatcherResourceManagerComponents( Configuration configuration, RpcServiceFactory rpcServiceFactory, HighAvailabilityServices haServices, @@ -106,14 +108,15 @@ public class TestingMiniCluster extends MiniCluster { MetricRegistry metricRegistry, MetricQueryServiceRetriever metricQueryServiceRetriever, FatalErrorHandler fatalErrorHandler) throws Exception { - SessionDispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createTestingDispatcherResourceManagerComponentFactory(); + DispatcherResourceManagerComponentFactory dispatcherResourceManagerComponentFactory = createTestingDispatcherResourceManagerComponentFactory(); - final List> result = new ArrayList<>(numberDispatcherResourceManagerComponents); + final List result = new ArrayList<>(numberDispatcherResourceManagerComponents); for (int i = 0; i < numberDispatcherResourceManagerComponents; i++) { result.add( dispatcherResourceManagerComponentFactory.create( configuration, + getIOExecutor(), rpcServiceFactory.createRpcService(), haServices, blobServer, @@ -132,9 +135,9 @@ public class TestingMiniCluster extends MiniCluster { return super.getDispatcherGatewayFuture(); } - private SessionDispatcherResourceManagerComponentFactory createTestingDispatcherResourceManagerComponentFactory() { - return new SessionDispatcherResourceManagerComponentFactory( - SessionDispatcherWithUUIDFactory.INSTANCE, + private DispatcherResourceManagerComponentFactory createTestingDispatcherResourceManagerComponentFactory() { + return TestingDefaultDispatcherResourceManagerComponentFactory.createSessionComponentFactory( + new DispatcherRunnerFactoryImpl(SessionDispatcherWithUUIDFactory.INSTANCE), StandaloneResourceManagerWithUUIDFactory.INSTANCE); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ActiveResourceManagerFactoryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ActiveResourceManagerFactoryTest.java index b1874e0989abb6343f5322d85d96a60b5b27b78f..ff61e65afdae57ecd6f3615b2b206432ca8d8e26 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ActiveResourceManagerFactoryTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ActiveResourceManagerFactoryTest.java @@ -27,9 +27,7 @@ import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.heartbeat.TestingHeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.rpc.FatalErrorHandler; import org.apache.flink.runtime.rpc.RpcService; @@ -69,11 +67,10 @@ public class ActiveResourceManagerFactoryTest extends TestLogger { rpcService, new TestingHighAvailabilityServices(), new TestingHeartbeatServices(), - NoOpMetricRegistry.INSTANCE, new TestingFatalErrorHandler(), new ClusterInformation("foobar", 1234), null, - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup()); + UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup()); } finally { RpcUtils.terminateRpcService(rpcService, Time.seconds(10L)); } @@ -88,12 +85,11 @@ public class ActiveResourceManagerFactoryTest extends TestLogger { RpcService rpcService, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, - MetricRegistry metricRegistry, FatalErrorHandler fatalErrorHandler, ClusterInformation clusterInformation, @Nullable String webInterfaceUrl, - JobManagerMetricGroup jobManagerMetricGroup) { - assertThat(configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE), is(true)); + ResourceManagerMetricGroup resourceManagerMetricGroup) { + assertThat(configuration.contains(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE), is(true)); return null; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java index 08106a4e04ea9e544728e7e322423540fe6d9e40..ff2f1ebc30d3ccd4dc1e96233a2426f85319d2d6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerHATest.java @@ -24,7 +24,6 @@ import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.TestingHeartbeatServices; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerConfiguration; import org.apache.flink.runtime.rpc.RpcService; @@ -53,7 +52,7 @@ public class ResourceManagerHATest extends TestLogger { TestingLeaderElectionService leaderElectionService = new TestingLeaderElectionService() { @Override - public void confirmLeaderSessionID(UUID leaderId) { + public void confirmLeadership(UUID leaderId, String leaderAddress) { leaderSessionIdFuture.complete(leaderId); } }; @@ -87,11 +86,10 @@ public class ResourceManagerHATest extends TestLogger { highAvailabilityServices, heartbeatServices, resourceManagerRuntimeServices.getSlotManager(), - NoOpMetricRegistry.INSTANCE, resourceManagerRuntimeServices.getJobLeaderIdService(), new ClusterInformation("localhost", 1234), testingFatalErrorHandler, - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), + UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(), Time.minutes(5L)) { @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java index 30024529d6e333f5ff1c54adedf1ffda79313af4..f527887530cfda1c0ff9f041fe7a58ebf68e9fa3 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerJobMasterTest.java @@ -33,7 +33,6 @@ import org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGateway; import org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGatewayBuilder; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.registration.RegistrationResponse; import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException; @@ -148,11 +147,10 @@ public class ResourceManagerJobMasterTest extends TestLogger { haServices, heartbeatServices, slotManager, - NoOpMetricRegistry.INSTANCE, jobLeaderIdService, new ClusterInformation("localhost", 1234), testingFatalErrorHandler, - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), + UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(), Time.minutes(5L)); resourceManager.start(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java index 39938b99a83c619b85af60ffeeb945aed7b58b25..394f63d6dad53e2baebfb2c670866b1bb163187a 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTaskExecutorTest.java @@ -30,7 +30,6 @@ import org.apache.flink.runtime.instance.HardwareDescription; import org.apache.flink.runtime.instance.InstanceID; import org.apache.flink.runtime.leaderelection.LeaderElectionService; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.registration.RegistrationResponse; import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager; @@ -158,11 +157,10 @@ public class ResourceManagerTaskExecutorTest extends TestLogger { highAvailabilityServices, heartbeatServices, slotManager, - NoOpMetricRegistry.INSTANCE, jobLeaderIdService, new ClusterInformation("localhost", 1234), fatalErrorHandler, - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), + UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(), Time.minutes(5L)); resourceManager.start(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTest.java index a9ce427a89e837bd075918348f86700234dc5caf..979860a1d64e2d6bc39bb813320af2e3d2403c5e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/ResourceManagerTest.java @@ -33,7 +33,6 @@ import org.apache.flink.runtime.jobmaster.utils.TestingJobMasterGatewayBuilder; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.runtime.leaderretrieval.SettableLeaderRetrievalService; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.registration.RegistrationResponse; import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager; @@ -262,10 +261,9 @@ public class ResourceManagerTest extends TestLogger { highAvailabilityServices, heartbeatServices, slotManager, - NoOpMetricRegistry.INSTANCE, jobLeaderIdService, testingFatalErrorHandler, - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup()); + UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup()); resourceManager.start(); @@ -282,7 +280,7 @@ public class ResourceManagerTest extends TestLogger { @Test public void testCreateWorkerSlotProfiles() { final Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "100m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "100m"); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 5); final ResourceProfile rmCalculatedResourceProfile = @@ -291,7 +289,7 @@ public class ResourceManagerTest extends TestLogger { final ResourceProfile tmCalculatedResourceProfile = TaskManagerServices.computeSlotResourceProfile( config.getInteger(TaskManagerOptions.NUM_TASK_SLOTS), - MemorySize.parse(config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE)).getBytes()); + MemorySize.parse(config.getString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE)).getBytes()); assertEquals(rmCalculatedResourceProfile, tmCalculatedResourceProfile); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManagerFactoryTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManagerFactoryTest.java index 9e64d48e696d56940dfb841c78e93d06747b4cba..e7921c7fd0244c474557666bab36072726126ce6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManagerFactoryTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManagerFactoryTest.java @@ -27,7 +27,6 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.TestingHeartbeatServices; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.rpc.RpcUtils; import org.apache.flink.runtime.rpc.TestingRpcService; @@ -57,11 +56,10 @@ public class StandaloneResourceManagerFactoryTest extends TestLogger { rpcService, new TestingHighAvailabilityServices(), new TestingHeartbeatServices(), - NoOpMetricRegistry.INSTANCE, new TestingFatalErrorHandler(), new ClusterInformation("foobar", 1234), null, - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup()); + UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup()); } finally { RpcUtils.terminateRpcService(rpcService, Time.seconds(10L)); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManagerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManagerTest.java index c39c14d297cc933adf28bd06713a7d044b94ad73..b433fa4bda70f2199f5bdeaef3792199127d5e06 100755 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManagerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/StandaloneResourceManagerTest.java @@ -23,8 +23,7 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager; import org.apache.flink.runtime.resourcemanager.slotmanager.TestingSlotManagerBuilder; @@ -118,11 +117,10 @@ public class StandaloneResourceManagerTest extends TestLogger { rmServices.highAvailabilityServices, rmServices.heartbeatServices, rmServices.slotManager, - rmServices.metricRegistry, rmServices.jobLeaderIdService, new ClusterInformation("localhost", 1234), fatalErrorHandler, - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(), + UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup(), startupPeriod, rmServices); @@ -142,11 +140,10 @@ public class StandaloneResourceManagerTest extends TestLogger { HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, SlotManager slotManager, - MetricRegistry metricRegistry, JobLeaderIdService jobLeaderIdService, ClusterInformation clusterInformation, FatalErrorHandler fatalErrorHandler, - JobManagerMetricGroup jobManagerMetricGroup, + ResourceManagerMetricGroup resourceManagerMetricGroup, Time startupPeriodTime, MockResourceManagerRuntimeServices rmServices) { super( @@ -156,11 +153,10 @@ public class StandaloneResourceManagerTest extends TestLogger { highAvailabilityServices, heartbeatServices, slotManager, - metricRegistry, jobLeaderIdService, clusterInformation, fatalErrorHandler, - jobManagerMetricGroup, + resourceManagerMetricGroup, startupPeriodTime); this.rmServices = rmServices; } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java index 3fdc8ce2c20f1201517955604ac88d34791c4168..42831e4bf30fc457cefa5e355c4e3f6be5d38db6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/TestingResourceManager.java @@ -24,8 +24,7 @@ import org.apache.flink.runtime.clusterframework.types.ResourceProfile; import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException; import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager; import org.apache.flink.runtime.rpc.FatalErrorHandler; @@ -48,10 +47,9 @@ public class TestingResourceManager extends ResourceManager { HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, SlotManager slotManager, - MetricRegistry metricRegistry, JobLeaderIdService jobLeaderIdService, FatalErrorHandler fatalErrorHandler, - JobManagerMetricGroup jobManagerMetricGroup) { + ResourceManagerMetricGroup resourceManagerMetricGroup) { super( rpcService, resourceManagerEndpointId, @@ -59,11 +57,10 @@ public class TestingResourceManager extends ResourceManager { highAvailabilityServices, heartbeatServices, slotManager, - metricRegistry, jobLeaderIdService, new ClusterInformation("localhost", 1234), fatalErrorHandler, - jobManagerMetricGroup); + resourceManagerMetricGroup); } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/utils/MockResourceManagerRuntimeServices.java b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/utils/MockResourceManagerRuntimeServices.java index 737decd93932de4fc37748b57a1a428136fb3f45..833b56d9a6bcb729f1f17a5fb07dc555a3f04526 100755 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/utils/MockResourceManagerRuntimeServices.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/resourcemanager/utils/MockResourceManagerRuntimeServices.java @@ -24,8 +24,6 @@ import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.heartbeat.TestingHeartbeatServices; import org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices; import org.apache.flink.runtime.leaderelection.TestingLeaderElectionService; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.NoOpMetricRegistry; import org.apache.flink.runtime.resourcemanager.JobLeaderIdService; import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager; import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManagerBuilder; @@ -46,7 +44,6 @@ public class MockResourceManagerRuntimeServices { public final Time timeout; public final TestingHighAvailabilityServices highAvailabilityServices; public final HeartbeatServices heartbeatServices; - public final MetricRegistry metricRegistry; public final TestingLeaderElectionService rmLeaderElectionService; public final JobLeaderIdService jobLeaderIdService; public final SlotManager slotManager; @@ -68,7 +65,6 @@ public class MockResourceManagerRuntimeServices { rmLeaderElectionService = new TestingLeaderElectionService(); highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService); heartbeatServices = new TestingHeartbeatServices(); - metricRegistry = NoOpMetricRegistry.INSTANCE; jobLeaderIdService = new JobLeaderIdService( highAvailabilityServices, rpcService.getScheduledExecutor(), diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java index 771fd8a837c2dd92f1840f287bacd71d0812ca8d..80fa4b957d8b0e5aa00f4fb44a654ba2cbf8d81c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/FileUploadHandlerTest.java @@ -38,8 +38,12 @@ import org.junit.Test; import java.io.File; import java.io.IOException; import java.io.StringWriter; +import java.lang.reflect.Field; +import java.util.LinkedHashSet; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Tests for the {@link FileUploadHandler}. Ensures that multipart http messages containing files and/or json are properly @@ -136,6 +140,8 @@ public class FileUploadHandlerTest extends TestLogger { try (Response response = client.newCall(fileRequest).execute()) { assertEquals(fileHandler.getMessageHeaders().getResponseStatusCode().code(), response.code()); } + + verifyNoFileIsRegisteredToDeleteOnExitHook(); } @Test @@ -162,6 +168,8 @@ public class FileUploadHandlerTest extends TestLogger { assertEquals(mixedHandler.getMessageHeaders().getResponseStatusCode().code(), response.code()); assertEquals(json, mixedHandler.lastReceivedRequest); } + + verifyNoFileIsRegisteredToDeleteOnExitHook(); } @Test @@ -188,6 +196,8 @@ public class FileUploadHandlerTest extends TestLogger { // FileUploads are outright forbidden assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.code()); } + + verifyNoFileIsRegisteredToDeleteOnExitHook(); } @Test @@ -212,6 +222,8 @@ public class FileUploadHandlerTest extends TestLogger { // JSON payload did not match expected format assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.code()); } + + verifyNoFileIsRegisteredToDeleteOnExitHook(); } @Test @@ -223,6 +235,8 @@ public class FileUploadHandlerTest extends TestLogger { assertEquals(HttpResponseStatus.BAD_REQUEST.code(), response.code()); } MULTIPART_UPLOAD_RESOURCE.assertUploadDirectoryIsEmpty(); + + verifyNoFileIsRegisteredToDeleteOnExitHook(); } /** @@ -238,5 +252,25 @@ public class FileUploadHandlerTest extends TestLogger { assertEquals(HttpResponseStatus.INTERNAL_SERVER_ERROR.code(), response.code()); } MULTIPART_UPLOAD_RESOURCE.assertUploadDirectoryIsEmpty(); + + verifyNoFileIsRegisteredToDeleteOnExitHook(); + } + + /** + * DiskAttribute and DiskFileUpload class of netty store post chunks and file chunks as temp files on local disk. + * By default, netty will register these temp files to java.io.DeleteOnExitHook which may lead to memory leak. + * {@link FileUploadHandler} disables the shutdown hook registration so no file should be registered. Note that + * clean up of temp files is handed over to {@link org.apache.flink.runtime.entrypoint.ClusterEntrypoint}. + */ + private void verifyNoFileIsRegisteredToDeleteOnExitHook() { + try { + Class clazz = Class.forName("java.io.DeleteOnExitHook"); + Field field = clazz.getDeclaredField("files"); + field.setAccessible(true); + LinkedHashSet files = (LinkedHashSet) field.get(null); + assertTrue(files.isEmpty()); + } catch (ClassNotFoundException | IllegalAccessException | NoSuchFieldException e) { + fail("This should never happen."); + } } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerSSLAuthITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerSSLAuthITCase.java index eecf7b0d089fa3ced04725df582d38045e6bd4e8..8f51a41740c2ca450b8f9e007b87d967d754c217 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerSSLAuthITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/RestServerSSLAuthITCase.java @@ -89,7 +89,7 @@ public class RestServerSSLAuthITCase extends TestLogger { RestServerEndpointConfiguration restServerConfig = RestServerEndpointConfiguration.fromConfiguration(serverConfig); RestClientConfiguration restClientConfig = RestClientConfiguration.fromConfiguration(clientConfig); - RestfulGateway restfulGateway = TestingRestfulGateway.newBuilder().build(); + RestfulGateway restfulGateway = new TestingRestfulGateway.Builder().build(); RestServerEndpointITCase.TestVersionHandler testVersionHandler = new RestServerEndpointITCase.TestVersionHandler( () -> CompletableFuture.completedFuture(restfulGateway), RpcUtils.INF_TIMEOUT); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/AbstractHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/AbstractHandlerTest.java index becbba7659a18ad8028b4fd3c5bb74680fa7f926..f07ee375879e3441742074740973aafe0c15cedf 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/AbstractHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/AbstractHandlerTest.java @@ -71,7 +71,7 @@ public class AbstractHandlerTest extends TestLogger { final Path file = dir.resolve("file"); Files.createFile(file); - RestfulGateway mockRestfulGateway = TestingRestfulGateway.newBuilder() + RestfulGateway mockRestfulGateway = new TestingRestfulGateway.Builder() .build(); final GatewayRetriever mockGatewayRetriever = () -> diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobExecutionResultHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobExecutionResultHandlerTest.java index d9f15601d363dceeebb7855b20a5d313517d7c59..a626d340acc5cf10ce40c5c2db4aefc98294c710 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobExecutionResultHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobExecutionResultHandlerTest.java @@ -65,7 +65,7 @@ public class JobExecutionResultHandlerTest extends TestLogger { @Before public void setUp() throws Exception { - final TestingRestfulGateway testingRestfulGateway = TestingRestfulGateway.newBuilder().build(); + final TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder().build(); jobExecutionResultHandler = new JobExecutionResultHandler( () -> CompletableFuture.completedFuture(testingRestfulGateway), @@ -81,7 +81,7 @@ public class JobExecutionResultHandlerTest extends TestLogger { @Test public void testResultInProgress() throws Exception { - final TestingRestfulGateway testingRestfulGateway = TestingRestfulGateway.newBuilder() + final TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder() .setRequestJobStatusFunction( jobId -> CompletableFuture.completedFuture(JobStatus.RUNNING)) .build(); @@ -103,7 +103,7 @@ public class JobExecutionResultHandlerTest extends TestLogger { .setState(jobStatus) .build(); - final TestingRestfulGateway testingRestfulGateway = TestingRestfulGateway.newBuilder() + final TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder() .setRequestJobStatusFunction( jobId -> { assertThat(jobId, equalTo(TEST_JOB_ID)); @@ -129,7 +129,7 @@ public class JobExecutionResultHandlerTest extends TestLogger { @Test public void testPropagateFlinkJobNotFoundExceptionAsRestHandlerException() throws Exception { - final TestingRestfulGateway testingRestfulGateway = TestingRestfulGateway.newBuilder() + final TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder() .setRequestJobStatusFunction( jobId -> FutureUtils.completedExceptionally(new FlinkJobNotFoundException(jobId)) ) diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobVertexBackPressureHandlerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobVertexBackPressureHandlerTest.java index ec8be0fa5aa160443ce27c79c8fefa0bcec4ebfa..db0715a2b393a8901f0e0325aef9d0167626c68e 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobVertexBackPressureHandlerTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/job/JobVertexBackPressureHandlerTest.java @@ -70,7 +70,7 @@ public class JobVertexBackPressureHandlerTest { @Before public void setUp() { - restfulGateway = TestingRestfulGateway.newBuilder().setRequestOperatorBackPressureStatsFunction( + restfulGateway = new TestingRestfulGateway.Builder().setRequestOperatorBackPressureStatsFunction( (jobId, jobVertexId) -> { if (jobId.equals(TEST_JOB_ID_BACK_PRESSURE_STATS_AVAILABLE)) { return CompletableFuture.completedFuture(OperatorBackPressureStatsResponse.of(new OperatorBackPressureStats( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/ExecutionGraphCacheTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/ExecutionGraphCacheTest.java index 545b34a335b336c604c3ef51e3980a6f327ba3d6..0ccfaaa58efb9f84d73857bdd8a1b8405f9d9b3d 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/ExecutionGraphCacheTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/handler/legacy/ExecutionGraphCacheTest.java @@ -168,7 +168,7 @@ public class ExecutionGraphCacheTest extends TestLogger { final ArchivedExecutionGraph expectedExecutionGraph2 = new ArchivedExecutionGraphBuilder().build(); final AtomicInteger requestJobCalls = new AtomicInteger(0); - final TestingRestfulGateway restfulGateway = TestingRestfulGateway.newBuilder() + final TestingRestfulGateway restfulGateway = new TestingRestfulGateway.Builder() .setRequestJobFunction( jobId -> { requestJobCalls.incrementAndGet(); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobVertexTaskManagersInfoTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobVertexTaskManagersInfoTest.java index 1a7b521f8c0a3a281de7d8e799fd40382c386704..3fe60d0c1faf47053576aa0a87e230530cf767b8 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobVertexTaskManagersInfoTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/messages/JobVertexTaskManagersInfoTest.java @@ -58,7 +58,7 @@ public class JobVertexTaskManagersInfoTest extends RestResponseMarshallingTestBa for (ExecutionState executionState : ExecutionState.values()) { statusCounts.put(executionState, count++); } - taskManagersInfoList.add(new TaskManagersInfo("host1", ExecutionState.CANCELING, 1L, 2L, 3L, jobVertexMetrics, statusCounts)); + taskManagersInfoList.add(new TaskManagersInfo("host1", ExecutionState.CANCELING, 1L, 2L, 3L, jobVertexMetrics, statusCounts, "taskmanagerId")); return new JobVertexTaskManagersInfo(new JobVertexID(), "test", System.currentTimeMillis(), taskManagersInfoList); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/util/DocumentingDispatcherRestEndpoint.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/util/DocumentingDispatcherRestEndpoint.java index 96e17e95e7d90512e36672ae5ba1d0a5ab23787f..db45ee02cb25c6479d2ed9beb9b864bab156deca 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rest/util/DocumentingDispatcherRestEndpoint.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rest/util/DocumentingDispatcherRestEndpoint.java @@ -110,7 +110,7 @@ public class DocumentingDispatcherRestEndpoint extends DispatcherRestEndpoint im } @Override - public void confirmLeaderSessionID(final UUID leaderSessionID) { + public void confirmLeadership(final UUID leaderSessionID, final String leaderAddress) { } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/AsyncCallsTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/AsyncCallsTest.java index 5ebdce22a61f7faa797fa06a2255fe8f9a3a8bc2..5d9745de763c57e743347b29ea47aa22772c6b92 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/AsyncCallsTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/AsyncCallsTest.java @@ -414,22 +414,13 @@ public class AsyncCallsTest extends TestLogger { UUID initialFencingToken, OneShotLatch enteringSetNewFencingToken, OneShotLatch triggerSetNewFencingToken) { - super(rpcService); + super(rpcService, initialFencingToken); this.lock = lock; this.concurrentAccess = concurrentAccess; this.enteringSetNewFencingToken = enteringSetNewFencingToken; this.triggerSetNewFencingToken = triggerSetNewFencingToken; - - // make it look as if we are running in the main thread - currentMainThread.set(Thread.currentThread()); - - try { - setFencingToken(initialFencingToken); - } finally { - currentMainThread.set(null); - } } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/FencedRpcEndpointTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/FencedRpcEndpointTest.java index ac350de823b86896371edc9c8750f731c1eaa053..43e5798574d955691141e84554c3092479dcb5c3 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/FencedRpcEndpointTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/FencedRpcEndpointTest.java @@ -296,20 +296,11 @@ public class FencedRpcEndpointTest extends TestLogger { } protected FencedTestingEndpoint(RpcService rpcService, String value, UUID initialFencingToken) { - super(rpcService); + super(rpcService, initialFencingToken); computationLatch = new OneShotLatch(); this.value = value; - - // make sure that it looks as if we are running in the main thread - currentMainThread.set(Thread.currentThread()); - - try { - setFencingToken(initialFencingToken); - } finally { - currentMainThread.set(null); - } } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java index c010810ef5fe23a3586183be4b1313c8f85a7701..0597b01f4654c186fc33965b1bd772aa13dca970 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/rpc/akka/AkkaRpcActorTest.java @@ -51,8 +51,6 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; -import scala.concurrent.Await; - import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -272,7 +270,7 @@ public class AkkaRpcActorTest extends TestLogger { terminationFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS); } finally { rpcActorSystem.terminate(); - Await.ready(rpcActorSystem.whenTerminated(), FutureUtils.toFiniteDuration(timeout)); + FutureUtils.toJava(rpcActorSystem.whenTerminated()).get(timeout.getSize(), timeout.getUnit()); } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultExecutionSlotAllocatorTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultExecutionSlotAllocatorTest.java index 4088f337cfcd5a0dce7c51dddb9776b5d02ccc6e..ce3f83711adca0b43087af75f7d2014cc6cc96c1 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultExecutionSlotAllocatorTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultExecutionSlotAllocatorTest.java @@ -23,8 +23,10 @@ import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.runtime.clusterframework.types.AllocationID; import org.apache.flink.runtime.clusterframework.types.ResourceProfile; import org.apache.flink.runtime.clusterframework.types.SlotProfile; +import org.apache.flink.runtime.executiongraph.SlotProviderStrategy; import org.apache.flink.runtime.instance.SlotSharingGroupId; import org.apache.flink.runtime.jobgraph.JobVertexID; +import org.apache.flink.runtime.jobgraph.ScheduleMode; import org.apache.flink.runtime.jobmanager.scheduler.CoLocationConstraint; import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup; import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; @@ -254,12 +256,37 @@ public class DefaultExecutionSlotAllocatorTest extends TestLogger { assertThat(allPriorAllocationIds, containsInAnyOrder(expectAllocationIds.toArray())); } + @Test + public void testDuplicatedSlotAllocationIsNotAllowed() { + final ExecutionVertexID executionVertexId = new ExecutionVertexID(new JobVertexID(), 0); + + final DefaultExecutionSlotAllocator executionSlotAllocator = createExecutionSlotAllocator(); + slotProvider.disableSlotAllocation(); + + final List schedulingRequirements = + createSchedulingRequirements(executionVertexId); + executionSlotAllocator.allocateSlotsFor(schedulingRequirements); + + try { + executionSlotAllocator.allocateSlotsFor(schedulingRequirements); + fail("exception should happen"); + } catch (IllegalStateException e) { + // IllegalStateException is expected + } + } + private DefaultExecutionSlotAllocator createExecutionSlotAllocator() { return createExecutionSlotAllocator(new TestingInputsLocationsRetriever.Builder().build()); } private DefaultExecutionSlotAllocator createExecutionSlotAllocator(InputsLocationsRetriever inputsLocationsRetriever) { - return new DefaultExecutionSlotAllocator(slotProvider, inputsLocationsRetriever, Time.seconds(10)); + return new DefaultExecutionSlotAllocator( + SlotProviderStrategy.from( + ScheduleMode.EAGER, + slotProvider, + Time.seconds(10), + true), + inputsLocationsRetriever); } private List createSchedulingRequirements(ExecutionVertexID... executionVertexIds) { diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultSchedulerTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultSchedulerTest.java new file mode 100644 index 0000000000000000000000000000000000000000..5c0b60c92cc8f5fce60a8049e3ebe7a89c75d34b --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/DefaultSchedulerTest.java @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.JobManagerOptions; +import org.apache.flink.runtime.blob.VoidBlobWriter; +import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory; +import org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter; +import org.apache.flink.runtime.concurrent.ManuallyTriggeredScheduledExecutor; +import org.apache.flink.runtime.execution.ExecutionState; +import org.apache.flink.runtime.executiongraph.ArchivedExecutionVertex; +import org.apache.flink.runtime.executiongraph.ExecutionAttemptID; +import org.apache.flink.runtime.executiongraph.failover.FailoverStrategyLoader; +import org.apache.flink.runtime.executiongraph.failover.flip1.RestartPipelinedRegionStrategy; +import org.apache.flink.runtime.executiongraph.failover.flip1.TestRestartBackoffTimeStrategy; +import org.apache.flink.runtime.executiongraph.utils.SimpleSlotProvider; +import org.apache.flink.runtime.io.network.partition.NoOpPartitionTracker; +import org.apache.flink.runtime.io.network.partition.ResultPartitionType; +import org.apache.flink.runtime.jobgraph.DistributionPattern; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.JobStatus; +import org.apache.flink.runtime.jobgraph.JobVertex; +import org.apache.flink.runtime.jobgraph.ScheduleMode; +import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException; +import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; +import org.apache.flink.runtime.rest.handler.legacy.backpressure.VoidBackPressureStatsTracker; +import org.apache.flink.runtime.scheduler.strategy.EagerSchedulingStrategy; +import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID; +import org.apache.flink.runtime.scheduler.strategy.LazyFromSourcesSchedulingStrategy; +import org.apache.flink.runtime.scheduler.strategy.SchedulingExecutionVertex; +import org.apache.flink.runtime.scheduler.strategy.SchedulingStrategyFactory; +import org.apache.flink.runtime.scheduler.strategy.SchedulingTopology; +import org.apache.flink.runtime.scheduler.strategy.TestSchedulingStrategy; +import org.apache.flink.runtime.shuffle.NettyShuffleMaster; +import org.apache.flink.runtime.taskmanager.TaskExecutionState; +import org.apache.flink.runtime.testtasks.NoOpInvokable; +import org.apache.flink.runtime.testutils.DirectScheduledExecutorService; +import org.apache.flink.util.ExecutorUtils; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.TestLogger; + +import org.apache.flink.shaded.guava18.com.google.common.collect.Iterables; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import static org.apache.flink.util.ExceptionUtils.findThrowable; +import static org.apache.flink.util.ExceptionUtils.findThrowableWithMessage; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Tests for {@link DefaultScheduler}. + */ +public class DefaultSchedulerTest extends TestLogger { + + private static final int TIMEOUT_MS = 1000; + + private static final JobID TEST_JOB_ID = new JobID(); + + private ManuallyTriggeredScheduledExecutor taskRestartExecutor = new ManuallyTriggeredScheduledExecutor(); + + private ExecutorService executor; + + private ScheduledExecutorService scheduledExecutorService; + + private Configuration configuration; + + private TestRestartBackoffTimeStrategy testRestartBackoffTimeStrategy; + + private TestExecutionVertexOperationsDecorator testExecutionVertexOperations; + + private ExecutionVertexVersioner executionVertexVersioner; + + private TestExecutionSlotAllocatorFactory executionSlotAllocatorFactory; + + private TestExecutionSlotAllocator testExecutionSlotAllocator; + + @Before + public void setUp() throws Exception { + executor = Executors.newSingleThreadExecutor(); + scheduledExecutorService = new DirectScheduledExecutorService(); + + configuration = new Configuration(); + configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, FailoverStrategyLoader.NO_OP_FAILOVER_STRATEGY); + + testRestartBackoffTimeStrategy = new TestRestartBackoffTimeStrategy(true, 0); + + testExecutionVertexOperations = new TestExecutionVertexOperationsDecorator(new DefaultExecutionVertexOperations()); + + executionVertexVersioner = new ExecutionVertexVersioner(); + + executionSlotAllocatorFactory = new TestExecutionSlotAllocatorFactory(); + testExecutionSlotAllocator = executionSlotAllocatorFactory.getTestExecutionSlotAllocator(); + } + + @After + public void tearDown() throws Exception { + if (scheduledExecutorService != null) { + ExecutorUtils.gracefulShutdown(TIMEOUT_MS, TimeUnit.MILLISECONDS, scheduledExecutorService); + } + + if (executor != null) { + ExecutorUtils.gracefulShutdown(TIMEOUT_MS, TimeUnit.MILLISECONDS, executor); + } + } + + @Test + public void startScheduling() { + final JobGraph jobGraph = singleNonParallelJobVertexJobGraph(); + final JobVertex onlyJobVertex = getOnlyJobVertex(jobGraph); + + createSchedulerAndStartScheduling(jobGraph); + + final List deployedExecutionVertices = testExecutionVertexOperations.getDeployedVertices(); + + final ExecutionVertexID executionVertexId = new ExecutionVertexID(onlyJobVertex.getID(), 0); + assertThat(deployedExecutionVertices, contains(executionVertexId)); + } + + @Test + public void restartAfterDeploymentFails() { + final JobGraph jobGraph = singleNonParallelJobVertexJobGraph(); + final JobVertex onlyJobVertex = getOnlyJobVertex(jobGraph); + + testExecutionVertexOperations.enableFailDeploy(); + + createSchedulerAndStartScheduling(jobGraph); + + testExecutionVertexOperations.disableFailDeploy(); + taskRestartExecutor.triggerScheduledTasks(); + + final List deployedExecutionVertices = testExecutionVertexOperations.getDeployedVertices(); + + final ExecutionVertexID executionVertexId = new ExecutionVertexID(onlyJobVertex.getID(), 0); + assertThat(deployedExecutionVertices, contains(executionVertexId, executionVertexId)); + } + + @Test + public void scheduleWithLazyStrategy() { + final JobGraph jobGraph = singleNonParallelJobVertexJobGraph(); + jobGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES); + final JobVertex onlyJobVertex = getOnlyJobVertex(jobGraph); + + createSchedulerAndStartScheduling(jobGraph); + + final List deployedExecutionVertices = testExecutionVertexOperations.getDeployedVertices(); + + final ExecutionVertexID executionVertexId = new ExecutionVertexID(onlyJobVertex.getID(), 0); + assertThat(deployedExecutionVertices, contains(executionVertexId)); + } + + @Test + public void restartFailedTask() { + final JobGraph jobGraph = singleNonParallelJobVertexJobGraph(); + final JobVertex onlyJobVertex = getOnlyJobVertex(jobGraph); + + final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph); + + final ArchivedExecutionVertex archivedExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices()); + final ExecutionAttemptID attemptId = archivedExecutionVertex.getCurrentExecutionAttempt().getAttemptId(); + + scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED)); + + taskRestartExecutor.triggerScheduledTasks(); + + final List deployedExecutionVertices = testExecutionVertexOperations.getDeployedVertices(); + final ExecutionVertexID executionVertexId = new ExecutionVertexID(onlyJobVertex.getID(), 0); + assertThat(deployedExecutionVertices, contains(executionVertexId, executionVertexId)); + } + + @Test + public void updateTaskExecutionStateReturnsFalseIfExecutionDoesNotExist() { + final JobGraph jobGraph = singleNonParallelJobVertexJobGraph(); + final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph); + + final TaskExecutionState taskExecutionState = new TaskExecutionState( + jobGraph.getJobID(), + new ExecutionAttemptID(), + ExecutionState.FAILED); + + assertFalse(scheduler.updateTaskExecutionState(taskExecutionState)); + } + + @Test + public void failJobIfCannotRestart() throws Exception { + final JobGraph jobGraph = singleNonParallelJobVertexJobGraph(); + testRestartBackoffTimeStrategy.setCanRestart(false); + + final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph); + + final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices()); + final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId(); + + scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED)); + + taskRestartExecutor.triggerScheduledTasks(); + + waitForTermination(scheduler); + final JobStatus jobStatus = scheduler.requestJobStatus(); + assertThat(jobStatus, is(equalTo(JobStatus.FAILED))); + } + + @Test + public void failJobIfNotEnoughResources() throws Exception { + final JobGraph jobGraph = singleNonParallelJobVertexJobGraph(); + testRestartBackoffTimeStrategy.setCanRestart(false); + testExecutionSlotAllocator.disableAutoCompletePendingRequests(); + + final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph); + + testExecutionSlotAllocator.timeoutPendingRequests(); + + waitForTermination(scheduler); + final JobStatus jobStatus = scheduler.requestJobStatus(); + assertThat(jobStatus, is(equalTo(JobStatus.FAILED))); + + Throwable failureCause = scheduler.requestJob() + .getFailureInfo() + .getException() + .deserializeError(DefaultSchedulerTest.class.getClassLoader()); + assertTrue(findThrowable(failureCause, NoResourceAvailableException.class).isPresent()); + assertTrue( + findThrowableWithMessage( + failureCause, + "Could not allocate the required slot within slot request timeout.").isPresent()); + assertThat(jobStatus, is(equalTo(JobStatus.FAILED))); + } + + @Test + public void skipDeploymentIfVertexVersionOutdated() { + testExecutionSlotAllocator.disableAutoCompletePendingRequests(); + + final JobGraph jobGraph = nonParallelSourceSinkJobGraph(); + final List sortedJobVertices = jobGraph.getVerticesSortedTopologicallyFromSources(); + final ExecutionVertexID sourceExecutionVertexId = new ExecutionVertexID(sortedJobVertices.get(0).getID(), 0); + final ExecutionVertexID sinkExecutionVertexId = new ExecutionVertexID(sortedJobVertices.get(1).getID(), 0); + + final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph); + testExecutionSlotAllocator.completePendingRequest(sourceExecutionVertexId); + + final ArchivedExecutionVertex sourceExecutionVertex = scheduler.requestJob().getAllExecutionVertices().iterator().next(); + final ExecutionAttemptID attemptId = sourceExecutionVertex.getCurrentExecutionAttempt().getAttemptId(); + scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED)); + testRestartBackoffTimeStrategy.setCanRestart(false); + + testExecutionSlotAllocator.enableAutoCompletePendingRequests(); + taskRestartExecutor.triggerScheduledTasks(); + + assertThat(testExecutionVertexOperations.getDeployedVertices(), containsInAnyOrder(sourceExecutionVertexId, sinkExecutionVertexId)); + assertThat(scheduler.requestJob().getState(), is(equalTo(JobStatus.RUNNING))); + } + + @Test + public void vertexIsResetBeforeRestarted() throws Exception { + final JobGraph jobGraph = singleNonParallelJobVertexJobGraph(); + + final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory(); + final DefaultScheduler scheduler = createScheduler(jobGraph, schedulingStrategyFactory); + final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy(); + final SchedulingTopology topology = schedulingStrategy.getSchedulingTopology(); + + startScheduling(scheduler); + + final SchedulingExecutionVertex onlySchedulingVertex = Iterables.getOnlyElement(topology.getVertices()); + schedulingStrategy.schedule(Collections.singleton(onlySchedulingVertex.getId())); + + final ArchivedExecutionVertex onlyExecutionVertex = Iterables.getOnlyElement(scheduler.requestJob().getAllExecutionVertices()); + final ExecutionAttemptID attemptId = onlyExecutionVertex.getCurrentExecutionAttempt().getAttemptId(); + scheduler.updateTaskExecutionState(new TaskExecutionState(jobGraph.getJobID(), attemptId, ExecutionState.FAILED)); + + taskRestartExecutor.triggerScheduledTasks(); + + assertThat(schedulingStrategy.getReceivedVerticesToRestart(), hasSize(1)); + assertThat(onlySchedulingVertex.getState(), is(equalTo(ExecutionState.CREATED))); + } + + @Test + public void scheduleOnlyIfVertexIsCreated() throws Exception { + final JobGraph jobGraph = singleNonParallelJobVertexJobGraph(); + + final TestSchedulingStrategy.Factory schedulingStrategyFactory = new TestSchedulingStrategy.Factory(); + final DefaultScheduler scheduler = createScheduler(jobGraph, schedulingStrategyFactory); + final TestSchedulingStrategy schedulingStrategy = schedulingStrategyFactory.getLastCreatedSchedulingStrategy(); + final SchedulingTopology topology = schedulingStrategy.getSchedulingTopology(); + + startScheduling(scheduler); + + final ExecutionVertexID onlySchedulingVertexId = Iterables.getOnlyElement(topology.getVertices()).getId(); + + // Schedule the vertex to get it to a non-CREATED state + schedulingStrategy.schedule(Collections.singleton(onlySchedulingVertexId)); + + // The scheduling of a non-CREATED vertex will result in IllegalStateException + try { + schedulingStrategy.schedule(Collections.singleton(onlySchedulingVertexId)); + fail("IllegalStateException should happen"); + } catch (IllegalStateException e) { + // expected exception + } + } + + private void waitForTermination(final DefaultScheduler scheduler) throws Exception { + scheduler.getTerminationFuture().get(TIMEOUT_MS, TimeUnit.MILLISECONDS); + } + + private static JobGraph singleNonParallelJobVertexJobGraph() { + final JobGraph jobGraph = new JobGraph(TEST_JOB_ID, "Testjob"); + jobGraph.setScheduleMode(ScheduleMode.EAGER); + final JobVertex vertex = new JobVertex("source"); + vertex.setInvokableClass(NoOpInvokable.class); + jobGraph.addVertex(vertex); + return jobGraph; + } + + private static JobGraph nonParallelSourceSinkJobGraph() { + final JobGraph jobGraph = new JobGraph(TEST_JOB_ID, "Testjob"); + jobGraph.setScheduleMode(ScheduleMode.EAGER); + + final JobVertex source = new JobVertex("source"); + source.setInvokableClass(NoOpInvokable.class); + jobGraph.addVertex(source); + + final JobVertex sink = new JobVertex("sink"); + sink.setInvokableClass(NoOpInvokable.class); + jobGraph.addVertex(sink); + + sink.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED); + + return jobGraph; + } + + private static JobVertex getOnlyJobVertex(final JobGraph jobGraph) { + final List sortedVertices = jobGraph.getVerticesSortedTopologicallyFromSources(); + Preconditions.checkState(sortedVertices.size() == 1); + return sortedVertices.get(0); + } + + private DefaultScheduler createSchedulerAndStartScheduling(final JobGraph jobGraph) { + final SchedulingStrategyFactory schedulingStrategyFactory = + jobGraph.getScheduleMode() == ScheduleMode.LAZY_FROM_SOURCES ? + new LazyFromSourcesSchedulingStrategy.Factory() : + new EagerSchedulingStrategy.Factory(); + + try { + final DefaultScheduler scheduler = createScheduler(jobGraph, schedulingStrategyFactory); + startScheduling(scheduler); + return scheduler; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private DefaultScheduler createScheduler( + final JobGraph jobGraph, + final SchedulingStrategyFactory schedulingStrategyFactory) throws Exception { + + return new DefaultScheduler( + log, + jobGraph, + VoidBackPressureStatsTracker.INSTANCE, + executor, + configuration, + new SimpleSlotProvider(TEST_JOB_ID, 0), + scheduledExecutorService, + taskRestartExecutor, + ClassLoader.getSystemClassLoader(), + new StandaloneCheckpointRecoveryFactory(), + Time.seconds(300), + VoidBlobWriter.getInstance(), + UnregisteredMetricGroups.createUnregisteredJobManagerJobMetricGroup(), + Time.seconds(300), + NettyShuffleMaster.INSTANCE, + NoOpPartitionTracker.INSTANCE, + schedulingStrategyFactory, + new RestartPipelinedRegionStrategy.Factory(), + testRestartBackoffTimeStrategy, + testExecutionVertexOperations, + executionVertexVersioner, + executionSlotAllocatorFactory); + } + + private void startScheduling(final SchedulerNG scheduler) { + scheduler.setMainThreadExecutor(ComponentMainThreadExecutorServiceAdapter.forMainThread()); + scheduler.startScheduling(); + } + +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionSlotAllocator.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionSlotAllocator.java new file mode 100644 index 0000000000000000000000000000000000000000..5960d7092cfbe00b01948fd61f5c50e304758eb4 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionSlotAllocator.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +import org.apache.flink.runtime.jobmaster.LogicalSlot; +import org.apache.flink.runtime.jobmaster.TestingLogicalSlotBuilder; +import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeoutException; + +import static org.apache.flink.util.Preconditions.checkState; + +/** + * Test {@link ExecutionSlotAllocator} implementation. + */ +public class TestExecutionSlotAllocator implements ExecutionSlotAllocator { + + private final Map pendingRequests = new HashMap<>(); + + private boolean autoCompletePendingRequests = true; + + @Override + public Collection allocateSlotsFor(final Collection schedulingRequirementsCollection) { + final List slotVertexAssignments = createSlotVertexAssignments(schedulingRequirementsCollection); + registerPendingRequests(slotVertexAssignments); + maybeCompletePendingRequests(); + return slotVertexAssignments; + } + + private void registerPendingRequests(final List slotVertexAssignments) { + for (SlotExecutionVertexAssignment slotVertexAssignment : slotVertexAssignments) { + pendingRequests.put(slotVertexAssignment.getExecutionVertexId(), slotVertexAssignment); + } + } + + private List createSlotVertexAssignments( + final Collection schedulingRequirementsCollection) { + + final List result = new ArrayList<>(); + for (ExecutionVertexSchedulingRequirements schedulingRequirements : schedulingRequirementsCollection) { + final ExecutionVertexID executionVertexId = schedulingRequirements.getExecutionVertexId(); + final CompletableFuture logicalSlotFuture = new CompletableFuture<>(); + result.add(new SlotExecutionVertexAssignment(executionVertexId, logicalSlotFuture)); + } + return result; + } + + private void maybeCompletePendingRequests() { + if (autoCompletePendingRequests) { + completePendingRequests(); + } + } + + public void completePendingRequests() { + final Collection vertexIds = new ArrayList<>(pendingRequests.keySet()); + vertexIds.forEach(this::completePendingRequest); + } + + public void completePendingRequest(final ExecutionVertexID executionVertexId) { + final SlotExecutionVertexAssignment slotVertexAssignment = removePendingRequest(executionVertexId); + checkState(slotVertexAssignment != null); + slotVertexAssignment + .getLogicalSlotFuture() + .complete(new TestingLogicalSlotBuilder().createTestingLogicalSlot()); + } + + private SlotExecutionVertexAssignment removePendingRequest(final ExecutionVertexID executionVertexId) { + return pendingRequests.remove(executionVertexId); + } + + public void timeoutPendingRequests() { + final Collection vertexIds = new ArrayList<>(pendingRequests.keySet()); + vertexIds.forEach(this::timeoutPendingRequest); + } + + public void timeoutPendingRequest(final ExecutionVertexID executionVertexId) { + final SlotExecutionVertexAssignment slotVertexAssignment = removePendingRequest(executionVertexId); + checkState(slotVertexAssignment != null); + slotVertexAssignment + .getLogicalSlotFuture() + .completeExceptionally(new TimeoutException()); + } + + public void enableAutoCompletePendingRequests() { + autoCompletePendingRequests = true; + } + + public void disableAutoCompletePendingRequests() { + autoCompletePendingRequests = false; + } + + @Override + public void cancel(final ExecutionVertexID executionVertexId) { + final SlotExecutionVertexAssignment slotVertexAssignment = removePendingRequest(executionVertexId); + if (slotVertexAssignment != null) { + slotVertexAssignment + .getLogicalSlotFuture() + .cancel(false); + } + } + + @Override + public CompletableFuture stop() { + return CompletableFuture.completedFuture(null); + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionSlotAllocatorFactory.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionSlotAllocatorFactory.java new file mode 100644 index 0000000000000000000000000000000000000000..e5d451ada81524b0142e40dfe4e8ab92a8282907 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionSlotAllocatorFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +/** + * Factory for {@link TestExecutionSlotAllocatorFactory}. + */ +public class TestExecutionSlotAllocatorFactory implements ExecutionSlotAllocatorFactory { + + private final TestExecutionSlotAllocator testExecutionSlotAllocator = new TestExecutionSlotAllocator(); + + @Override + public ExecutionSlotAllocator createInstance(final InputsLocationsRetriever ignored) { + return testExecutionSlotAllocator; + } + + public TestExecutionSlotAllocator getTestExecutionSlotAllocator() { + return testExecutionSlotAllocator; + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionVertexOperationsDecorator.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionVertexOperationsDecorator.java new file mode 100644 index 0000000000000000000000000000000000000000..50457519404490c5c6eacd524d78e30fa7dc4254 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/TestExecutionVertexOperationsDecorator.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.flink.runtime.scheduler; + +import org.apache.flink.runtime.JobException; +import org.apache.flink.runtime.executiongraph.ExecutionVertex; +import org.apache.flink.runtime.scheduler.strategy.ExecutionVertexID; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * {@link ExecutionVertexOperations} decorator that enables instrumentation of execution vertex + * operations for testing purposes. + */ +public class TestExecutionVertexOperationsDecorator implements ExecutionVertexOperations { + + private final ExecutionVertexOperations delegate; + + private final List deployedVertices = new ArrayList<>(); + + private boolean failDeploy; + + public TestExecutionVertexOperationsDecorator(final ExecutionVertexOperations delegate) { + this.delegate = checkNotNull(delegate); + } + + @Override + public void deploy(final ExecutionVertex executionVertex) throws JobException { + deployedVertices.add(executionVertex.getID()); + + if (failDeploy) { + throw new RuntimeException("Expected"); + } + + delegate.deploy(executionVertex); + } + + @Override + public CompletableFuture cancel(final ExecutionVertex executionVertex) { + return delegate.cancel(executionVertex); + } + + public void enableFailDeploy() { + failDeploy = true; + } + + public void disableFailDeploy() { + failDeploy = false; + } + + public List getDeployedVertices() { + return Collections.unmodifiableList(deployedVertices); + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adapter/ExecutionGraphToSchedulingTopologyAdapterTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adapter/ExecutionGraphToSchedulingTopologyAdapterTest.java index 5a925dae462c38831465ab10fcacc3b3119a5956..ab72e2c209f33b90c2546c4b6129a655c1dbe2c0 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adapter/ExecutionGraphToSchedulingTopologyAdapterTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/adapter/ExecutionGraphToSchedulingTopologyAdapterTest.java @@ -181,7 +181,7 @@ public class ExecutionGraphToSchedulingTopologyAdapterTest extends TestLogger { // since deep equality is verified later in the main loop // this DOES rely on an implicit assumption that the vertices objects returned by the topology are // identical to those stored in the partition - ExecutionVertexID originalId = new ExecutionVertexID(originalConsumer.getJobvertexId(), originalConsumer.getParallelSubtaskIndex()); + ExecutionVertexID originalId = originalConsumer.getID(); assertTrue(adaptedConsumers.stream().anyMatch(adaptedConsumer -> adaptedConsumer.getId().equals(originalId))); } } @@ -203,7 +203,7 @@ public class ExecutionGraphToSchedulingTopologyAdapterTest extends TestLogger { ExecutionVertex originalVertex, SchedulingExecutionVertex adaptedVertex) { assertEquals( - new ExecutionVertexID(originalVertex.getJobvertexId(), originalVertex.getParallelSubtaskIndex()), + originalVertex.getID(), adaptedVertex.getId()); assertEquals(originalVertex.getInputDependencyConstraint(), adaptedVertex.getInputDependencyConstraint()); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/strategy/TestSchedulingStrategy.java b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/strategy/TestSchedulingStrategy.java new file mode 100644 index 0000000000000000000000000000000000000000..eebec4fec184dd56e149d72d663f6c2639221d57 --- /dev/null +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/scheduler/strategy/TestSchedulingStrategy.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.runtime.scheduler.strategy; + +import org.apache.flink.runtime.execution.ExecutionState; +import org.apache.flink.runtime.io.network.partition.ResultPartitionID; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.scheduler.DeploymentOption; +import org.apache.flink.runtime.scheduler.ExecutionVertexDeploymentOption; +import org.apache.flink.runtime.scheduler.SchedulerOperations; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * {@link SchedulingStrategy} instance for tests. + */ +public class TestSchedulingStrategy implements SchedulingStrategy { + + private final SchedulerOperations schedulerOperations; + + private final SchedulingTopology schedulingTopology; + + private final DeploymentOption deploymentOption = new DeploymentOption(false); + + private Set receivedVerticesToRestart; + + public TestSchedulingStrategy( + final SchedulerOperations schedulerOperations, + final SchedulingTopology schedulingTopology) { + + this.schedulerOperations = checkNotNull(schedulerOperations); + this.schedulingTopology = checkNotNull(schedulingTopology); + } + + @Override + public void startScheduling() { + } + + @Override + public void restartTasks(final Set verticesToRestart) { + this.receivedVerticesToRestart = verticesToRestart; + } + + @Override + public void onExecutionStateChange(final ExecutionVertexID executionVertexId, final ExecutionState executionState) { + } + + @Override + public void onPartitionConsumable(final ExecutionVertexID executionVertexId, final ResultPartitionID resultPartitionId) { + } + + public void schedule(final Set verticesToSchedule) { + allocateSlotsAndDeploy(verticesToSchedule); + } + + public SchedulingTopology getSchedulingTopology() { + return schedulingTopology; + } + + public Set getReceivedVerticesToRestart() { + return receivedVerticesToRestart; + } + + private void allocateSlotsAndDeploy(final Set verticesToSchedule) { + final List executionVertexDeploymentOptions = + createExecutionVertexDeploymentOptions(verticesToSchedule); + schedulerOperations.allocateSlotsAndDeploy(executionVertexDeploymentOptions); + } + + private List createExecutionVertexDeploymentOptions( + final Collection vertices) { + + final List executionVertexDeploymentOptions = new ArrayList<>(vertices.size()); + for (ExecutionVertexID executionVertexID : vertices) { + executionVertexDeploymentOptions.add(new ExecutionVertexDeploymentOption(executionVertexID, deploymentOption)); + } + return executionVertexDeploymentOptions; + } + + /** + * The factory for creating {@link TestSchedulingStrategy}. + */ + public static class Factory implements SchedulingStrategyFactory { + + private TestSchedulingStrategy lastInstance; + + @Override + public SchedulingStrategy createInstance( + final SchedulerOperations schedulerOperations, + final SchedulingTopology schedulingTopology, + final JobGraph jobGraph) { + + lastInstance = new TestSchedulingStrategy(schedulerOperations, schedulingTopology); + return lastInstance; + } + + public TestSchedulingStrategy getLastCreatedSchedulingStrategy() { + return lastInstance; + } + } +} diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NettyShuffleEnvironmentConfigurationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NettyShuffleEnvironmentConfigurationTest.java index aee46898588684fc730c37d265f19c1409d8cdc5..a3e295f98e68e1fa8ef6fdc0ab756b68fa194381 100755 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NettyShuffleEnvironmentConfigurationTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NettyShuffleEnvironmentConfigurationTest.java @@ -247,15 +247,15 @@ public class NettyShuffleEnvironmentConfigurationTest extends TestLogger { config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, true); config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "10m"); // 10MB + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "10m"); // 10MB assertEquals(890, TaskManagerServices.calculateHeapSizeMB(1000, config)); config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.6f); assertEquals(390, TaskManagerServices.calculateHeapSizeMB(1000, config)); - config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE); // use fraction of given memory + config.removeConfig(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE); // use fraction of given memory config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, 0.1f); - config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, 0.1f); // 10% + config.setFloat(TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION, 0.1f); // 10% assertEquals(810, TaskManagerServices.calculateHeapSizeMB(1000, config)); } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java index ca83609a569cc4bdbe0cba8e4ea8230bc424ee7e..355784d3786a1a3ecba5c5ad778642eae474f8f2 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/NetworkBufferCalculationTest.java @@ -46,32 +46,32 @@ public class NetworkBufferCalculationTest extends TestLogger { final long networkBufMax = 1L << 30; // 1GB config = getConfig( - Long.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()), - TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(), + Long.valueOf(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.defaultValue()), + TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.defaultValue(), 0.1f, networkBufMin, networkBufMax, MemoryType.HEAP); assertEquals(100L << 20, NettyShuffleEnvironmentConfiguration.calculateNewNetworkBufferMemory(config, 900L << 20)); // 900MB config = getConfig( - Long.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()), - TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(), + Long.valueOf(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.defaultValue()), + TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.defaultValue(), 0.2f, networkBufMin, networkBufMax, MemoryType.HEAP); assertEquals(200L << 20, NettyShuffleEnvironmentConfiguration.calculateNewNetworkBufferMemory(config, 800L << 20)); // 800MB config = getConfig( - Long.valueOf(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()), - TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(), + Long.valueOf(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.defaultValue()), + TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.defaultValue(), 0.6f, networkBufMin, networkBufMax, MemoryType.HEAP); assertEquals(600L << 20, NettyShuffleEnvironmentConfiguration.calculateNewNetworkBufferMemory(config, 400L << 20)); // 400MB - config = getConfig(10, TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(), + config = getConfig(10, TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.defaultValue(), 0.1f, networkBufMin, networkBufMax, MemoryType.OFF_HEAP); assertEquals(100L << 20, NettyShuffleEnvironmentConfiguration.calculateNewNetworkBufferMemory(config, 890L << 20)); // 890MB - config = getConfig(10, TaskManagerOptions.MANAGED_MEMORY_FRACTION.defaultValue(), + config = getConfig(10, TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.defaultValue(), 0.6f, networkBufMin, networkBufMax, MemoryType.OFF_HEAP); assertEquals(615L << 20, NettyShuffleEnvironmentConfiguration.calculateNewNetworkBufferMemory(config, 400L << 20)); // 400MB @@ -84,8 +84,8 @@ public class NetworkBufferCalculationTest extends TestLogger { /** * Returns a configuration for the tests. * - * @param managedMemory see {@link TaskManagerOptions#MANAGED_MEMORY_SIZE} - * @param managedMemoryFraction see {@link TaskManagerOptions#MANAGED_MEMORY_FRACTION} + * @param managedMemory see {@link TaskManagerOptions#LEGACY_MANAGED_MEMORY_SIZE} + * @param managedMemoryFraction see {@link TaskManagerOptions#LEGACY_MANAGED_MEMORY_FRACTION} * @param networkBufFraction see {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_FRACTION} * @param networkBufMin see {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MIN} * @param networkBufMax see {@link NettyShuffleEnvironmentOptions#NETWORK_BUFFERS_MEMORY_MAX} @@ -103,8 +103,8 @@ public class NetworkBufferCalculationTest extends TestLogger { final Configuration configuration = new Configuration(); - configuration.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), managedMemory); - configuration.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(), managedMemoryFraction); + configuration.setLong(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE.key(), managedMemory); + configuration.setFloat(TaskManagerOptions.LEGACY_MANAGED_MEMORY_FRACTION.key(), managedMemoryFraction); configuration.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key(), networkBufFraction); configuration.setLong(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN.key(), networkBufMin); configuration.setLong(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX.key(), networkBufMax); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java index 71c2775374e10af2a7767bcd4e0815db47e0dbf2..630bb8ef510110bce3a46da018ec92d1996fa786 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskExecutorITCase.java @@ -36,12 +36,14 @@ import org.apache.flink.runtime.jobmaster.TestingAbstractInvokables; import org.apache.flink.runtime.minicluster.TestingMiniCluster; import org.apache.flink.runtime.minicluster.TestingMiniClusterConfiguration; import org.apache.flink.runtime.testutils.CommonTestUtils; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.util.TestLogger; import org.apache.flink.util.function.SupplierWithException; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.io.IOException; import java.time.Duration; @@ -56,6 +58,7 @@ import static org.junit.Assert.assertThat; /** * Integration tests for the {@link TaskExecutor}. */ +@Category(AlsoRunWithSchedulerNG.class) public class TaskExecutorITCase extends TestLogger { private static final Duration TESTING_TIMEOUT = Duration.ofMinutes(2L); @@ -157,7 +160,7 @@ public class TaskExecutorITCase extends TestLogger { private JobGraph createJobGraphWithRestartStrategy(int parallelism) throws IOException { final JobGraph jobGraph = createJobGraph(parallelism); final ExecutionConfig executionConfig = new ExecutionConfig(); - executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L)); + executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(2, 0L)); jobGraph.setExecutionConfig(executionConfig); return jobGraph; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunnerStartupTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunnerStartupTest.java index 58ef6a8f0898ae9054ae6da26bbdc08225d6e235..d2ace9562e7ae23e2609d2ba0a533e086d15a8a7 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunnerStartupTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskexecutor/TaskManagerRunnerStartupTest.java @@ -123,7 +123,7 @@ public class TaskManagerRunnerStartupTest extends TestLogger { cfg.setBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE, true); // something invalid - cfg.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "-42m"); + cfg.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "-42m"); try { startTaskManager( @@ -139,7 +139,7 @@ public class TaskManagerRunnerStartupTest extends TestLogger { // something ridiculously high final long memSize = (((long) Integer.MAX_VALUE - 1) * MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes()) >> 20; - cfg.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, memSize + "m"); + cfg.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, memSize + "m"); try { startTaskManager( diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java index 76969760d9ab8c3d006979cd967fa4de72a4399b..e3fda33f50777da2facf598ab055692d9a4ff8ed 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/taskmanager/TaskCancelAsyncProducerConsumerITCase.java @@ -40,11 +40,13 @@ import org.apache.flink.runtime.minicluster.MiniCluster; import org.apache.flink.runtime.testingUtils.TestingUtils; import org.apache.flink.runtime.testutils.MiniClusterResource; import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration; +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG; import org.apache.flink.types.LongValue; import org.apache.flink.util.TestLogger; import org.junit.ClassRule; import org.junit.Test; +import org.junit.experimental.categories.Category; import java.time.Duration; import java.util.Arrays; @@ -55,6 +57,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +@Category(AlsoRunWithSchedulerNG.class) public class TaskCancelAsyncProducerConsumerITCase extends TestLogger { // The Exceptions thrown by the producer/consumer Threads @@ -215,7 +218,7 @@ public class TaskCancelAsyncProducerConsumerITCase extends TestLogger { private final RecordWriter recordWriter; public ProducerThread(ResultPartitionWriter partitionWriter) { - this.recordWriter = new RecordWriterBuilder().build(partitionWriter); + this.recordWriter = new RecordWriterBuilder().build(partitionWriter); } @Override diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/MiniClusterResource.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/MiniClusterResource.java index eb63ceb44add8617852def20d29792f0a88cdb07..b648c22f67a24f7cb85610790572f71152eb8931 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/MiniClusterResource.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/MiniClusterResource.java @@ -159,8 +159,8 @@ public class MiniClusterResource extends ExternalResource { configuration.setBoolean(CoreOptions.FILESYTEM_DEFAULT_OVERRIDE, true); } - if (!configuration.contains(TaskManagerOptions.MANAGED_MEMORY_SIZE)) { - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE); + if (!configuration.contains(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE)) { + configuration.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, DEFAULT_MANAGED_MEMORY_SIZE); } // set rest and rpc port to 0 to avoid clashes with concurrent MiniClusters diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/InMemoryJobGraphStore.java b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestingJobGraphStore.java similarity index 35% rename from flink-runtime/src/test/java/org/apache/flink/runtime/testutils/InMemoryJobGraphStore.java rename to flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestingJobGraphStore.java index 31c3ea6eb85bdf069d4b12ec116383af424ff422..bdf6f0b56c67eb8ccb69ac1f2da80441c4a49af9 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/InMemoryJobGraphStore.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/testutils/TestingJobGraphStore.java @@ -24,6 +24,8 @@ import org.apache.flink.runtime.jobmanager.JobGraphStore; import org.apache.flink.util.Preconditions; import org.apache.flink.util.function.BiFunctionWithException; import org.apache.flink.util.function.FunctionWithException; +import org.apache.flink.util.function.ThrowingConsumer; +import org.apache.flink.util.function.ThrowingRunnable; import javax.annotation.Nullable; @@ -33,83 +35,88 @@ import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import static java.util.Objects.requireNonNull; - /** * In-Memory implementation of {@link JobGraphStore} for testing purposes. */ -public class InMemoryJobGraphStore implements JobGraphStore { +public class TestingJobGraphStore implements JobGraphStore { private final Map storedJobs = new HashMap<>(); - private boolean started; + private final ThrowingConsumer startConsumer; - private volatile FunctionWithException, Collection, ? extends Exception> jobIdsFunction; + private final ThrowingRunnable stopRunnable; - private volatile BiFunctionWithException, JobGraph, ? extends Exception> recoverJobGraphFunction; + private final FunctionWithException, Collection, ? extends Exception> jobIdsFunction; - public InMemoryJobGraphStore() { - jobIdsFunction = null; - recoverJobGraphFunction = null; - } + private final BiFunctionWithException, JobGraph, ? extends Exception> recoverJobGraphFunction; - public void setJobIdsFunction(FunctionWithException, Collection, ? extends Exception> jobIdsFunction) { - this.jobIdsFunction = Preconditions.checkNotNull(jobIdsFunction); - } + private final ThrowingConsumer putJobGraphConsumer; + + private final ThrowingConsumer removeJobGraphConsumer; - public void setRecoverJobGraphFunction(BiFunctionWithException, JobGraph, ? extends Exception> recoverJobGraphFunction) { - this.recoverJobGraphFunction = Preconditions.checkNotNull(recoverJobGraphFunction); + private final ThrowingConsumer releaseJobGraphConsumer; + + private boolean started; + + private TestingJobGraphStore( + ThrowingConsumer startConsumer, + ThrowingRunnable stopRunnable, + FunctionWithException, Collection, ? extends Exception> jobIdsFunction, + BiFunctionWithException, JobGraph, ? extends Exception> recoverJobGraphFunction, + ThrowingConsumer putJobGraphConsumer, + ThrowingConsumer removeJobGraphConsumer, + ThrowingConsumer releaseJobGraphConsumer) { + this.startConsumer = startConsumer; + this.stopRunnable = stopRunnable; + this.jobIdsFunction = jobIdsFunction; + this.recoverJobGraphFunction = recoverJobGraphFunction; + this.putJobGraphConsumer = putJobGraphConsumer; + this.removeJobGraphConsumer = removeJobGraphConsumer; + this.releaseJobGraphConsumer = releaseJobGraphConsumer; } @Override public synchronized void start(@Nullable JobGraphListener jobGraphListener) throws Exception { + startConsumer.accept(jobGraphListener); started = true; } @Override public synchronized void stop() throws Exception { + stopRunnable.run(); started = false; } @Override public synchronized JobGraph recoverJobGraph(JobID jobId) throws Exception { verifyIsStarted(); - - if (recoverJobGraphFunction != null) { - return recoverJobGraphFunction.apply(jobId, storedJobs); - } else { - return requireNonNull( - storedJobs.get(jobId), - "Job graph for job " + jobId + " does not exist"); - } + return recoverJobGraphFunction.apply(jobId, storedJobs); } @Override public synchronized void putJobGraph(JobGraph jobGraph) throws Exception { verifyIsStarted(); + putJobGraphConsumer.accept(jobGraph); storedJobs.put(jobGraph.getJobID(), jobGraph); } @Override public synchronized void removeJobGraph(JobID jobId) throws Exception { verifyIsStarted(); + removeJobGraphConsumer.accept(jobId); storedJobs.remove(jobId); } @Override - public void releaseJobGraph(JobID jobId) { + public synchronized void releaseJobGraph(JobID jobId) throws Exception { verifyIsStarted(); + releaseJobGraphConsumer.accept(jobId); } @Override public synchronized Collection getJobIds() throws Exception { verifyIsStarted(); - - if (jobIdsFunction != null) { - return jobIdsFunction.apply(storedJobs.keySet()); - } else { - return Collections.unmodifiableSet(new HashSet<>(storedJobs.keySet())); - } + return jobIdsFunction.apply(Collections.unmodifiableSet(new HashSet<>(storedJobs.keySet()))); } public synchronized boolean contains(JobID jobId) { @@ -120,4 +127,72 @@ public class InMemoryJobGraphStore implements JobGraphStore { Preconditions.checkState(started, "Not running. Forgot to call start()?"); } + public static Builder newBuilder() { + return new Builder(); + } + + public static class Builder { + private ThrowingConsumer startConsumer = ignored -> {}; + + private ThrowingRunnable stopRunnable = () -> {}; + + private FunctionWithException, Collection, ? extends Exception> jobIdsFunction = jobIds -> jobIds; + + private BiFunctionWithException, JobGraph, ? extends Exception> recoverJobGraphFunction = (jobId, jobs) -> jobs.get(jobId); + + private ThrowingConsumer putJobGraphConsumer = ignored -> {}; + + private ThrowingConsumer removeJobGraphConsumer = ignored -> {}; + + private ThrowingConsumer releaseJobGraphConsumer = ignored -> {}; + + private Builder() {} + + public Builder setStartConsumer(ThrowingConsumer startConsumer) { + this.startConsumer = startConsumer; + return this; + } + + public Builder setStopRunnable(ThrowingRunnable stopRunnable) { + this.stopRunnable = stopRunnable; + return this; + } + + public Builder setJobIdsFunction(FunctionWithException, Collection, ? extends Exception> jobIdsFunction) { + this.jobIdsFunction = jobIdsFunction; + return this; + } + + public Builder setRecoverJobGraphFunction(BiFunctionWithException, JobGraph, ? extends Exception> recoverJobGraphFunction) { + this.recoverJobGraphFunction = recoverJobGraphFunction; + return this; + } + + public Builder setPutJobGraphConsumer(ThrowingConsumer putJobGraphConsumer) { + this.putJobGraphConsumer = putJobGraphConsumer; + return this; + } + + public Builder setRemoveJobGraphConsumer(ThrowingConsumer removeJobGraphConsumer) { + this.removeJobGraphConsumer = removeJobGraphConsumer; + return this; + } + + public Builder setReleaseJobGraphConsumer(ThrowingConsumer releaseJobGraphConsumer) { + this.releaseJobGraphConsumer = releaseJobGraphConsumer; + return this; + } + + public TestingJobGraphStore build() { + return new TestingJobGraphStore( + startConsumer, + stopRunnable, + jobIdsFunction, + recoverJobGraphFunction, + putJobGraphConsumer, + removeJobGraphConsumer, + releaseJobGraphConsumer); + } + } + } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ThrowableClassifierTest.java b/flink-runtime/src/test/java/org/apache/flink/runtime/throwable/ThrowableClassifierTest.java similarity index 64% rename from flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ThrowableClassifierTest.java rename to flink-runtime/src/test/java/org/apache/flink/runtime/throwable/ThrowableClassifierTest.java index 517321a5c3b3e30972c13f8a7f2b02b034cb07dd..08abd5e274430522619815dd25e302867feb4520 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/executiongraph/ThrowableClassifierTest.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/throwable/ThrowableClassifierTest.java @@ -16,14 +16,11 @@ * limitations under the License. */ -package org.apache.flink.runtime.executiongraph; +package org.apache.flink.runtime.throwable; import org.apache.flink.runtime.execution.SuppressRestartsException; -import org.apache.flink.runtime.jobmanager.scheduler.NoResourceAvailableException; -import org.apache.flink.runtime.throwable.ThrowableAnnotation; -import org.apache.flink.runtime.throwable.ThrowableClassifier; -import org.apache.flink.runtime.throwable.ThrowableType; import org.apache.flink.util.TestLogger; + import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -31,17 +28,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; /** - * Test throwable classifier - * */ + * Test {@link ThrowableClassifier}. + */ public class ThrowableClassifierTest extends TestLogger { @Test public void testThrowableType_NonRecoverable() { assertEquals(ThrowableType.NonRecoverableError, ThrowableClassifier.getThrowableType(new SuppressRestartsException(new Exception("")))); - - assertEquals(ThrowableType.NonRecoverableError, - ThrowableClassifier.getThrowableType(new NoResourceAvailableException())); } @Test @@ -49,25 +43,25 @@ public class ThrowableClassifierTest extends TestLogger { assertEquals(ThrowableType.RecoverableError, ThrowableClassifier.getThrowableType(new Exception(""))); assertEquals(ThrowableType.RecoverableError, - ThrowableClassifier.getThrowableType(new ThrowableType_RecoverableFailure_Exception())); + ThrowableClassifier.getThrowableType(new TestRecoverableErrorException())); } @Test public void testThrowableType_EnvironmentError() { assertEquals(ThrowableType.EnvironmentError, - ThrowableClassifier.getThrowableType(new ThrowableType_EnvironmentError_Exception())); + ThrowableClassifier.getThrowableType(new TestEnvironmentErrorException())); } @Test public void testThrowableType_PartitionDataMissingError() { assertEquals(ThrowableType.PartitionDataMissingError, - ThrowableClassifier.getThrowableType(new ThrowableType_PartitionDataMissingError_Exception())); + ThrowableClassifier.getThrowableType(new TestPartitionDataMissingErrorException())); } @Test public void testThrowableType_InheritError() { assertEquals(ThrowableType.PartitionDataMissingError, - ThrowableClassifier.getThrowableType(new Sub_ThrowableType_PartitionDataMissingError_Exception())); + ThrowableClassifier.getThrowableType(new TestPartitionDataMissingErrorSubException())); } @Test @@ -79,40 +73,40 @@ public class ThrowableClassifierTest extends TestLogger { // no recoverable throwable type assertFalse(ThrowableClassifier.findThrowableOfThrowableType( - new ThrowableType_PartitionDataMissingError_Exception(), + new TestPartitionDataMissingErrorException(), ThrowableType.RecoverableError).isPresent()); // direct recoverable throwable assertTrue(ThrowableClassifier.findThrowableOfThrowableType( - new ThrowableType_RecoverableFailure_Exception(), + new TestRecoverableErrorException(), ThrowableType.RecoverableError).isPresent()); // nested recoverable throwable assertTrue(ThrowableClassifier.findThrowableOfThrowableType( - new Exception(new ThrowableType_RecoverableFailure_Exception()), + new Exception(new TestRecoverableErrorException()), ThrowableType.RecoverableError).isPresent()); // inherit recoverable throwable assertTrue(ThrowableClassifier.findThrowableOfThrowableType( - new Sub_ThrowableType_RecoverableFailure_Exception(), + new TestRecoverableFailureSubException(), ThrowableType.RecoverableError).isPresent()); } @ThrowableAnnotation(ThrowableType.PartitionDataMissingError) - private class ThrowableType_PartitionDataMissingError_Exception extends Exception { + private class TestPartitionDataMissingErrorException extends Exception { } @ThrowableAnnotation(ThrowableType.EnvironmentError) - private class ThrowableType_EnvironmentError_Exception extends Exception { + private class TestEnvironmentErrorException extends Exception { } @ThrowableAnnotation(ThrowableType.RecoverableError) - private class ThrowableType_RecoverableFailure_Exception extends Exception { + private class TestRecoverableErrorException extends Exception { } - private class Sub_ThrowableType_PartitionDataMissingError_Exception extends ThrowableType_PartitionDataMissingError_Exception { + private class TestPartitionDataMissingErrorSubException extends TestPartitionDataMissingErrorException { } - private class Sub_ThrowableType_RecoverableFailure_Exception extends ThrowableType_RecoverableFailure_Exception { + private class TestRecoverableFailureSubException extends TestRecoverableErrorException { } } diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingDispatcherGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingDispatcherGateway.java index 24c79f3c4e94574366585c98883ac76512896a6e..2ce36957a4022bae958386b96d048669bd38206c 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingDispatcherGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingDispatcherGateway.java @@ -135,7 +135,7 @@ public final class TestingDispatcherGateway extends TestingRestfulGateway implem /** * Builder for the {@link TestingDispatcherGateway}. */ - public static final class Builder extends TestingRestfulGateway.Builder { + public static final class Builder extends TestingRestfulGateway.AbstractBuilder { private Function> submitFunction; private Supplier>> listFunction; @@ -159,11 +159,16 @@ public final class TestingDispatcherGateway extends TestingRestfulGateway implem } @Override - public TestingRestfulGateway.Builder setRequestJobFunction(Function> requestJobFunction) { + public Builder setRequestJobFunction(Function> requestJobFunction) { // signature clash throw new UnsupportedOperationException("Use setRequestArchivedJobFunction() instead."); } + @Override + protected Builder self() { + return this; + } + public Builder setBlobServerPort(int blobServerPort) { this.blobServerPort = blobServerPort; return this; diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java index 1a3c6d78a1868d88a5a9b916a254fe9947311461..b309940025eb30a76eab28b93ca89940f1c4b806 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java +++ b/flink-runtime/src/test/java/org/apache/flink/runtime/webmonitor/TestingRestfulGateway.java @@ -197,14 +197,12 @@ public class TestingRestfulGateway implements RestfulGateway { return hostname; } - public static Builder newBuilder() { - return new Builder(); - } - /** - * Builder for the {@link TestingRestfulGateway}. + * Abstract builder class for {@link TestingRestfulGateway} and its subclasses. + * + * @param type of sub class */ - public static class Builder { + protected abstract static class AbstractBuilder { protected String address = LOCALHOST; protected String hostname = LOCALHOST; protected Function> cancelJobFunction; @@ -220,7 +218,7 @@ public class TestingRestfulGateway implements RestfulGateway { protected BiFunction> triggerSavepointFunction; protected BiFunction> stopWithSavepointFunction; - public Builder() { + protected AbstractBuilder() { cancelJobFunction = DEFAULT_CANCEL_JOB_FUNCTION; requestJobFunction = DEFAULT_REQUEST_JOB_FUNCTION; requestJobResultFunction = DEFAULT_REQUEST_JOB_RESULT_FUNCTION; @@ -234,72 +232,88 @@ public class TestingRestfulGateway implements RestfulGateway { stopWithSavepointFunction = DEFAULT_STOP_WITH_SAVEPOINT_FUNCTION; } - public Builder setAddress(String address) { + public T setAddress(String address) { this.address = address; - return this; + return self(); } - public Builder setHostname(String hostname) { + public T setHostname(String hostname) { this.hostname = hostname; - return this; + return self(); } - public Builder setRequestJobFunction(Function> requestJobFunction) { + public T setRequestJobFunction(Function> requestJobFunction) { this.requestJobFunction = requestJobFunction; - return this; + return self(); } - public Builder setRequestJobResultFunction(Function> requestJobResultFunction) { + public T setRequestJobResultFunction(Function> requestJobResultFunction) { this.requestJobResultFunction = requestJobResultFunction; - return this; + return self(); } - public Builder setRequestJobStatusFunction(Function> requestJobStatusFunction) { + public T setRequestJobStatusFunction(Function> requestJobStatusFunction) { this.requestJobStatusFunction = requestJobStatusFunction; - return this; + return self(); } - public Builder setRequestMultipleJobDetailsSupplier(Supplier> requestMultipleJobDetailsSupplier) { + public T setRequestMultipleJobDetailsSupplier(Supplier> requestMultipleJobDetailsSupplier) { this.requestMultipleJobDetailsSupplier = requestMultipleJobDetailsSupplier; - return this; + return self(); } - public Builder setRequestClusterOverviewSupplier(Supplier> requestClusterOverviewSupplier) { + public T setRequestClusterOverviewSupplier(Supplier> requestClusterOverviewSupplier) { this.requestClusterOverviewSupplier = requestClusterOverviewSupplier; - return this; + return self(); } - public Builder setRequestMetricQueryServiceGatewaysSupplier(Supplier>> requestMetricQueryServiceGatewaysSupplier) { + public T setRequestMetricQueryServiceGatewaysSupplier(Supplier>> requestMetricQueryServiceGatewaysSupplier) { this.requestMetricQueryServiceGatewaysSupplier = requestMetricQueryServiceGatewaysSupplier; - return this; + return self(); } - public Builder setRequestTaskManagerMetricQueryServiceGatewaysSupplier(Supplier>>> requestTaskManagerMetricQueryServiceGatewaysSupplier) { + public T setRequestTaskManagerMetricQueryServiceGatewaysSupplier(Supplier>>> requestTaskManagerMetricQueryServiceGatewaysSupplier) { this.requestTaskManagerMetricQueryServiceGatewaysSupplier = requestTaskManagerMetricQueryServiceGatewaysSupplier; - return this; + return self(); } - public Builder setRequestOperatorBackPressureStatsFunction(BiFunction> requestOeratorBackPressureStatsFunction) { + public T setRequestOperatorBackPressureStatsFunction(BiFunction> requestOeratorBackPressureStatsFunction) { this.requestOperatorBackPressureStatsFunction = requestOeratorBackPressureStatsFunction; - return this; + return self(); } - public Builder setCancelJobFunction(Function> cancelJobFunction) { + public T setCancelJobFunction(Function> cancelJobFunction) { this.cancelJobFunction = cancelJobFunction; - return this; + return self(); } - public Builder setTriggerSavepointFunction(BiFunction> triggerSavepointFunction) { + public T setTriggerSavepointFunction(BiFunction> triggerSavepointFunction) { this.triggerSavepointFunction = triggerSavepointFunction; - return this; + return self(); } - public Builder setStopWithSavepointFunction(BiFunction> stopWithSavepointFunction) { + public T setStopWithSavepointFunction(BiFunction> stopWithSavepointFunction) { this.stopWithSavepointFunction = stopWithSavepointFunction; + return self(); + } + + protected abstract T self(); + + public abstract TestingRestfulGateway build(); + } + + /** + * Builder for the {@link TestingRestfulGateway}. + */ + public static class Builder extends AbstractBuilder { + + @Override + protected Builder self() { return this; } - public TestingRestfulGateway build() { + @Override + public TestingRestfulGateway build() { return new TestingRestfulGateway( address, hostname, diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaUtilsTest.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaUtilsTest.scala index 3e20c44f3e0800a1bc141ec9dde5d6810bc75f76..bc7b709655a186496dcb3c21d5dca62b2f69b2b0 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaUtilsTest.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/akka/AkkaUtilsTest.scala @@ -207,4 +207,14 @@ class AkkaUtilsTest akkaConfig.getString("akka.remote.netty.tcp.hostname") should equal(NetUtils.unresolvedHostToNormalizedString(ipv6AddressString)) } + + test("getAkkaConfig should set startup timeout to be 10 times of ask timeout by default") { + val configuration = new Configuration() + configuration.setString(AkkaOptions.ASK_TIMEOUT.key(), "100ms") + + val akkaConfig = AkkaUtils.getAkkaConfig(configuration, Some(("localhost", 31337))) + + akkaConfig.getString("akka.remote.startup-timeout") should + equal("1000ms") + } } diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/Tasks.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/Tasks.scala index ea11f919dbef3118bfce319de87f8a4e2282a268..90ee2822c378cee317bde0ca32f4887a0a5a324d 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/Tasks.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/jobmanager/Tasks.scala @@ -35,8 +35,8 @@ object Tasks { classOf[IntValue], getEnvironment.getTaskManagerInfo.getTmpDirectories) - val writer = new RecordWriterBuilder().build( - getEnvironment.getWriter(0)).asInstanceOf[RecordWriter[IntValue]] + val writer = new RecordWriterBuilder[IntValue]().build( + getEnvironment.getWriter(0)) try { while (true) { diff --git a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala index fdc2c07ba7f6b0e220bac236fc3a236678941152..93aebcfea657cfc4056fb3f1bc56aafbb79807bb 100644 --- a/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala +++ b/flink-runtime/src/test/scala/org/apache/flink/runtime/testingUtils/TestingUtils.scala @@ -18,6 +18,7 @@ package org.apache.flink.runtime.testingUtils +import java.time.Duration import java.util import java.util.Collections import java.util.concurrent._ @@ -28,7 +29,6 @@ import org.apache.flink.api.common.time.Time import org.apache.flink.runtime.akka.AkkaUtils import org.apache.flink.runtime.concurrent.{ScheduledExecutor, ScheduledExecutorServiceAdapter} -import scala.concurrent.duration.{TimeUnit, _} import scala.concurrent.{ExecutionContext, ExecutionContextExecutor} import scala.language.postfixOps @@ -41,9 +41,9 @@ object TestingUtils { val testConfig = ConfigFactory.parseString(getDefaultTestingActorSystemConfigString) - val TESTING_DURATION = 2 minute + val TESTING_DURATION = Duration.ofMinutes(2L); - val TESTING_TIMEOUT = 1 minute + val TESTING_TIMEOUT = Duration.ofMinutes(1L); val TIMEOUT = Time.minutes(1L) diff --git a/flink-scala-shell/src/main/java/org/apache/flink/api/java/ScalaShellRemoteEnvironment.java b/flink-scala-shell/src/main/java/org/apache/flink/api/java/ScalaShellRemoteEnvironment.java index 6898a99a2e9ec4eba322f2bfb0f36fb0d4e7241b..e5451927422e0367060a3b1cecfeab336a6c4ad0 100644 --- a/flink-scala-shell/src/main/java/org/apache/flink/api/java/ScalaShellRemoteEnvironment.java +++ b/flink-scala-shell/src/main/java/org/apache/flink/api/java/ScalaShellRemoteEnvironment.java @@ -62,8 +62,8 @@ public class ScalaShellRemoteEnvironment extends RemoteEnvironment { final Plan p = createProgramPlan(jobName); final List allJarFiles = getUpdatedJarFiles(); - final PlanExecutor executor = PlanExecutor.createRemoteExecutor(host, port, clientConfiguration, allJarFiles, globalClasspaths); - lastJobExecutionResult = executor.executePlan(p); + final PlanExecutor executor = PlanExecutor.createRemoteExecutor(host, port, clientConfiguration); + lastJobExecutionResult = executor.executePlan(p, allJarFiles, globalClasspaths); return lastJobExecutionResult; } diff --git a/flink-scala-shell/src/test/java/org/apache/flink/api/java/FlinkILoopTest.java b/flink-scala-shell/src/test/java/org/apache/flink/api/java/FlinkILoopTest.java index eddf66c6d3738f1bd31006fe71ddca2a359bec5e..19bc2a0aa28934f88edcd5364649f965026faf05 100644 --- a/flink-scala-shell/src/test/java/org/apache/flink/api/java/FlinkILoopTest.java +++ b/flink-scala-shell/src/test/java/org/apache/flink/api/java/FlinkILoopTest.java @@ -19,8 +19,8 @@ package org.apache.flink.api.java; import org.apache.flink.api.common.JobExecutionResult; -import org.apache.flink.api.common.Plan; import org.apache.flink.api.common.PlanExecutor; +import org.apache.flink.api.dag.Pipeline; import org.apache.flink.api.java.io.DiscardingOutputFormat; import org.apache.flink.api.scala.FlinkILoop; import org.apache.flink.configuration.Configuration; @@ -39,6 +39,7 @@ import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; +import java.net.URL; import java.util.List; import scala.Option; @@ -68,18 +69,13 @@ public class FlinkILoopTest extends TestLogger { BDDMockito.given(PlanExecutor.createRemoteExecutor( Matchers.anyString(), Matchers.anyInt(), - Matchers.any(Configuration.class), - Matchers.any(java.util.List.class), - Matchers.any(java.util.List.class) + Matchers.any(Configuration.class) )).willAnswer(new Answer() { @Override public PlanExecutor answer(InvocationOnMock invocation) throws Throwable { testPlanExecutor.setHost((String) invocation.getArguments()[0]); testPlanExecutor.setPort((Integer) invocation.getArguments()[1]); testPlanExecutor.setConfiguration((Configuration) invocation.getArguments()[2]); - testPlanExecutor.setJars((List) invocation.getArguments()[3]); - testPlanExecutor.setGlobalClasspaths((List) invocation.getArguments()[4]); - return testPlanExecutor; } }); @@ -128,7 +124,8 @@ public class FlinkILoopTest extends TestLogger { private List globalClasspaths; @Override - public JobExecutionResult executePlan(Plan plan) throws Exception { + public JobExecutionResult executePlan( + Pipeline plan, List jarFiles, List globalClasspaths) throws Exception { return null; } diff --git a/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala b/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala index 8fd94e5d692db9eae0221c459621be216c1fa4cf..3c158a818f74e8748ec9aabc335ebf4c9e8a0230 100644 --- a/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala +++ b/flink-scala-shell/src/test/scala/org/apache/flink/api/scala/ScalaShellITCase.scala @@ -25,8 +25,10 @@ import org.apache.flink.runtime.clusterframework.BootstrapTools import org.apache.flink.runtime.minicluster.MiniCluster import org.apache.flink.runtime.testutils.{MiniClusterResource, MiniClusterResourceConfiguration} import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration +import org.apache.flink.testutils.junit.category.AlsoRunWithSchedulerNG import org.apache.flink.util.TestLogger import org.junit._ +import org.junit.experimental.categories.Category import org.junit.rules.TemporaryFolder import scala.tools.nsc.Settings @@ -441,6 +443,7 @@ class ScalaShellITCase extends TestLogger { } +@Category(Array(classOf[AlsoRunWithSchedulerNG])) object ScalaShellITCase { val configuration = new Configuration() diff --git a/flink-scala-shell/start-script/start-scala-shell.sh b/flink-scala-shell/start-script/start-scala-shell.sh index b6da81af7298690dd24ff3b8c8adb37a475de733..65b9045584b008cf1c4a72266d18d1e6cba30198 100644 --- a/flink-scala-shell/start-script/start-scala-shell.sh +++ b/flink-scala-shell/start-script/start-scala-shell.sh @@ -97,9 +97,9 @@ log_setting="-Dlog.file="$LOG" -Dlog4j.configuration=file:"$FLINK_CONF_DIR"/$LOG if ${EXTERNAL_LIB_FOUND} then - $JAVA_RUN -Dscala.color -cp "$FLINK_CLASSPATH" $log_setting org.apache.flink.api.scala.FlinkShell $@ --addclasspath "$EXT_CLASSPATH" + $JAVA_RUN -Dscala.color -cp "$FLINK_CLASSPATH" "$log_setting" org.apache.flink.api.scala.FlinkShell $@ --addclasspath "$EXT_CLASSPATH" else - $JAVA_RUN -Dscala.color -cp "$FLINK_CLASSPATH" $log_setting org.apache.flink.api.scala.FlinkShell $@ + $JAVA_RUN -Dscala.color -cp "$FLINK_CLASSPATH" "$log_setting" org.apache.flink.api.scala.FlinkShell $@ fi #restore echo diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java index b71006b31c41645b2966c890ba87787d0c0bb59a..a896cf2ed53a7b1fd57b381036122d93b68d95e6 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/LocalStreamEnvironment.java @@ -20,20 +20,15 @@ package org.apache.flink.streaming.api.environment; import org.apache.flink.annotation.Public; import org.apache.flink.api.common.InvalidProgramException; import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.PlanExecutor; import org.apache.flink.api.java.ExecutionEnvironment; import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.RestOptions; -import org.apache.flink.configuration.TaskManagerOptions; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.minicluster.MiniCluster; -import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; import org.apache.flink.streaming.api.graph.StreamGraph; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import javax.annotation.Nonnull; +import java.util.Collections; + /** * The LocalStreamEnvironment is a StreamExecutionEnvironment that runs the program locally, * multi-threaded, in the JVM where the environment is instantiated. It spawns an embedded @@ -45,8 +40,6 @@ import javax.annotation.Nonnull; @Public public class LocalStreamEnvironment extends StreamExecutionEnvironment { - private static final Logger LOG = LoggerFactory.getLogger(LocalStreamEnvironment.class); - private final Configuration configuration; /** @@ -83,42 +76,11 @@ public class LocalStreamEnvironment extends StreamExecutionEnvironment { */ @Override public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { - JobGraph jobGraph = streamGraph.getJobGraph(); - jobGraph.setAllowQueuedScheduling(true); - - Configuration configuration = new Configuration(); - configuration.addAll(jobGraph.getJobConfiguration()); - configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); - - // add (and override) the settings with what the user defined - configuration.addAll(this.configuration); - - if (!configuration.contains(RestOptions.BIND_PORT)) { - configuration.setString(RestOptions.BIND_PORT, "0"); - } - - int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism()); - - MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder() - .setConfiguration(configuration) - .setNumSlotsPerTaskManager(numSlotsPerTaskManager) - .build(); - - if (LOG.isInfoEnabled()) { - LOG.info("Running job on local embedded Flink mini cluster"); - } - - MiniCluster miniCluster = new MiniCluster(cfg); - try { - miniCluster.start(); - configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort()); - - return miniCluster.executeJobBlocking(jobGraph); - } - finally { + final PlanExecutor executor = PlanExecutor.createLocalExecutor(configuration); + return executor.executePlan(streamGraph, Collections.emptyList(), Collections.emptyList()); + } finally { transformations.clear(); - miniCluster.close(); } } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java index 0ba5bd6059520e5cd9cf222738bb8f2035cc00f4..2574b8e343f8e63b51a4e19d181826586a88e11c 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/RemoteStreamEnvironment.java @@ -22,9 +22,10 @@ import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.InvalidProgramException; import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.PlanExecutor; import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.client.ClientUtils; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.JobWithJars; import org.apache.flink.client.program.ProgramInvocationException; import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.configuration.Configuration; @@ -191,7 +192,7 @@ public class RemoteStreamEnvironment extends StreamExecutionEnvironment { try { URL jarFileUrl = new File(jarFile).getAbsoluteFile().toURI().toURL(); this.jarFiles.add(jarFileUrl); - JobWithJars.checkJarFile(jarFileUrl); + ClientUtils.checkJarFile(jarFileUrl); } catch (MalformedURLException e) { throw new IllegalArgumentException("JAR file path is invalid '" + jarFile + "'", e); } catch (IOException e) { @@ -225,7 +226,6 @@ public class RemoteStreamEnvironment extends StreamExecutionEnvironment { ) throws ProgramInvocationException { StreamGraph streamGraph = streamExecutionEnvironment.getStreamGraph(jobName); return executeRemotely(streamGraph, - streamExecutionEnvironment.getClass().getClassLoader(), streamExecutionEnvironment.getConfig(), jarFiles, host, @@ -242,7 +242,6 @@ public class RemoteStreamEnvironment extends StreamExecutionEnvironment { * @throws ProgramInvocationException */ private static JobExecutionResult executeRemotely(StreamGraph streamGraph, - ClassLoader envClassLoader, ExecutionConfig executionConfig, List jarFiles, String host, @@ -255,8 +254,6 @@ public class RemoteStreamEnvironment extends StreamExecutionEnvironment { LOG.info("Running remotely at {}:{}", host, port); } - ClassLoader userCodeClassLoader = JobWithJars.buildUserCodeClassLoader(jarFiles, globalClasspaths, envClassLoader); - Configuration configuration = new Configuration(); configuration.addAll(clientConfiguration); @@ -270,25 +267,27 @@ public class RemoteStreamEnvironment extends StreamExecutionEnvironment { client = new RestClusterClient<>(configuration, "RemoteStreamEnvironment"); } catch (Exception e) { - throw new ProgramInvocationException("Cannot establish connection to JobManager: " + e.getMessage(), - streamGraph.getJobGraph().getJobID(), e); + throw new ProgramInvocationException("Cannot establish connection to JobManager: " + e.getMessage(), e); } - if (savepointRestoreSettings == null) { - savepointRestoreSettings = SavepointRestoreSettings.none(); + if (savepointRestoreSettings != null) { + streamGraph.setSavepointRestoreSettings(savepointRestoreSettings); } try { - return client.run(streamGraph, jarFiles, globalClasspaths, userCodeClassLoader, savepointRestoreSettings) - .getJobExecutionResult(); + final PlanExecutor executor = PlanExecutor.createRemoteExecutor( + host, + port, + clientConfiguration); + + return executor.executePlan(streamGraph, jarFiles, globalClasspaths).getJobExecutionResult(); } catch (ProgramInvocationException e) { throw e; } catch (Exception e) { String term = e.getMessage() == null ? "." : (": " + e.getMessage()); - throw new ProgramInvocationException("The program execution failed" + term, - streamGraph.getJobGraph().getJobID(), e); + throw new ProgramInvocationException("The program execution failed" + term, e); } finally { try { @@ -318,7 +317,6 @@ public class RemoteStreamEnvironment extends StreamExecutionEnvironment { @Deprecated protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List jarFiles) throws ProgramInvocationException { return executeRemotely(streamGraph, - this.getClass().getClassLoader(), getConfig(), jarFiles, host, diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java index 6ee45411f63b3a9d02e9e916aff6dd8e87d0b606..b2db19b41481ac9224ced9f867866f9e50831188 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamContextEnvironment.java @@ -19,7 +19,10 @@ package org.apache.flink.streaming.api.environment; import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.client.ClientUtils; +import org.apache.flink.client.FlinkPipelineTranslationUtil; import org.apache.flink.client.program.ContextEnvironment; +import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.streaming.api.graph.StreamGraph; /** @@ -43,9 +46,20 @@ public class StreamContextEnvironment extends StreamExecutionEnvironment { public JobExecutionResult execute(StreamGraph streamGraph) throws Exception { transformations.clear(); - // execute the programs - return ctx.getClient() - .run(streamGraph, ctx.getJars(), ctx.getClasspaths(), ctx.getUserCodeClassLoader(), ctx.getSavepointRestoreSettings()) + JobGraph jobGraph = FlinkPipelineTranslationUtil.getJobGraph( + streamGraph, + ctx.getClient().getFlinkConfiguration(), + getParallelism()); + + ClientUtils.addJarFiles(jobGraph, ctx.getJars()); + jobGraph.setClasspaths(ctx.getClasspaths()); + + // running from the CLI will override the savepoint restore settings + jobGraph.setSavepointRestoreSettings(ctx.getSavepointRestoreSettings()); + + return ctx + .getClient() + .submitJob(jobGraph, ctx.getUserCodeClassLoader()) .getJobExecutionResult(); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java index bb1a4cfe0bca68557f6143e63475d4aa5cac6812..54ef3e292367e50e16168d73fc72cd552695189c 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/environment/StreamPlanEnvironment.java @@ -57,7 +57,7 @@ public class StreamPlanEnvironment extends StreamExecutionEnvironment { transformations.clear(); if (env instanceof OptimizerPlanEnvironment) { - ((OptimizerPlanEnvironment) env).setPlan(streamGraph); + ((OptimizerPlanEnvironment) env).setPipeline(streamGraph); } throw new OptimizerPlanEnvironment.ProgramAbortException(); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java index 75692dde1c9a9447506c80548c3a0cfed9c25537..ed7e9264249ac02d4863818422fc998e177cfe4e 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/functions/sink/filesystem/StreamingFileSink.java @@ -110,7 +110,7 @@ public class StreamingFileSink private final long bucketCheckInterval; - private final StreamingFileSink.BucketsBuilder bucketsBuilder; + private final StreamingFileSink.BucketsBuilder> bucketsBuilder; // --------------------------- runtime fields ----------------------------- @@ -125,11 +125,24 @@ public class StreamingFileSink private transient ListState maxPartCountersState; /** - * Creates a new {@code StreamingFileSink} that writes files to the given base directory. + * Creates a new {@code StreamingFileSink} that writes files in row-based format to the given base directory. */ protected StreamingFileSink( - final StreamingFileSink.BucketsBuilder bucketsBuilder, - final long bucketCheckInterval) { + final RowFormatBuilder> bucketsBuilder, + final long bucketCheckInterval) { + + Preconditions.checkArgument(bucketCheckInterval > 0L); + + this.bucketsBuilder = Preconditions.checkNotNull(bucketsBuilder); + this.bucketCheckInterval = bucketCheckInterval; + } + + /** + * Creates a new {@code StreamingFileSink} that writes files in bulk-encoded format to the given base directory. + */ + protected StreamingFileSink( + final BulkFormatBuilder> bucketsBuilder, + final long bucketCheckInterval) { Preconditions.checkArgument(bucketCheckInterval > 0L); @@ -149,7 +162,7 @@ public class StreamingFileSink * @return The builder where the remaining of the configuration parameters for the sink can be configured. * In order to instantiate the sink, call {@link RowFormatBuilder#build()} after specifying the desired parameters. */ - public static StreamingFileSink.RowFormatBuilder forRowFormat( + public static StreamingFileSink.RowFormatBuilder> forRowFormat( final Path basePath, final Encoder encoder) { return new StreamingFileSink.RowFormatBuilder<>(basePath, encoder, new DateTimeBucketAssigner<>()); } @@ -162,7 +175,7 @@ public class StreamingFileSink * @return The builder where the remaining of the configuration parameters for the sink can be configured. * In order to instantiate the sink, call {@link RowFormatBuilder#build()} after specifying the desired parameters. */ - public static StreamingFileSink.BulkFormatBuilder forBulkFormat( + public static StreamingFileSink.BulkFormatBuilder> forBulkFormat( final Path basePath, final BulkWriter.Factory writerFactory) { return new StreamingFileSink.BulkFormatBuilder<>(basePath, writerFactory, new DateTimeBucketAssigner<>()); } @@ -170,10 +183,17 @@ public class StreamingFileSink /** * The base abstract class for the {@link RowFormatBuilder} and {@link BulkFormatBuilder}. */ - protected abstract static class BucketsBuilder implements Serializable { + private abstract static class BucketsBuilder> implements Serializable { private static final long serialVersionUID = 1L; + protected static final long DEFAULT_BUCKET_CHECK_INTERVAL = 60L * 1000L; + + @SuppressWarnings("unchecked") + protected T self() { + return (T) this; + } + abstract Buckets createBuckets(final int subtaskIndex) throws IOException; } @@ -181,31 +201,31 @@ public class StreamingFileSink * A builder for configuring the sink for row-wise encoding formats. */ @PublicEvolving - public static class RowFormatBuilder extends StreamingFileSink.BucketsBuilder { + public static class RowFormatBuilder> extends StreamingFileSink.BucketsBuilder { private static final long serialVersionUID = 1L; - private final long bucketCheckInterval; + private long bucketCheckInterval; private final Path basePath; - private final Encoder encoder; + private Encoder encoder; - private final BucketAssigner bucketAssigner; + private BucketAssigner bucketAssigner; - private final RollingPolicy rollingPolicy; + private RollingPolicy rollingPolicy; - private final BucketFactory bucketFactory; + private BucketFactory bucketFactory; - private final String partFilePrefix; + private String partFilePrefix; - private final String partFileSuffix; + private String partFileSuffix; - RowFormatBuilder(Path basePath, Encoder encoder, BucketAssigner bucketAssigner) { - this(basePath, encoder, bucketAssigner, DefaultRollingPolicy.builder().build(), 60L * 1000L, new DefaultBucketFactoryImpl<>(), PartFileConfig.DEFAULT_PART_PREFIX, PartFileConfig.DEFAULT_PART_SUFFIX); + protected RowFormatBuilder(Path basePath, Encoder encoder, BucketAssigner bucketAssigner) { + this(basePath, encoder, bucketAssigner, DefaultRollingPolicy.builder().build(), DEFAULT_BUCKET_CHECK_INTERVAL, new DefaultBucketFactoryImpl<>(), PartFileConfig.DEFAULT_PART_PREFIX, PartFileConfig.DEFAULT_PART_SUFFIX); } - private RowFormatBuilder( + protected RowFormatBuilder( Path basePath, Encoder encoder, BucketAssigner assigner, @@ -224,44 +244,38 @@ public class StreamingFileSink this.partFileSuffix = Preconditions.checkNotNull(partFileSuffix); } - /** - * Creates a new builder instance with the specified bucket check interval. The interval specifies how often - * time based {@link RollingPolicy}s will be checked/executed for the open buckets. - * @param interval Time interval in milliseconds - * @return A new builder with the check interval set. - */ - public StreamingFileSink.RowFormatBuilder withBucketCheckInterval(final long interval) { - return new RowFormatBuilder<>(basePath, encoder, bucketAssigner, rollingPolicy, interval, bucketFactory, partFilePrefix, partFileSuffix); + public long getBucketCheckInterval() { + return bucketCheckInterval; } - /** - * Creates a new builder instance with the specified {@link BucketAssigner}. - * @param assigner @{@link BucketAssigner} to be used. - * @return A new builder with the assigner set. - */ - public StreamingFileSink.RowFormatBuilder withBucketAssigner(final BucketAssigner assigner) { - return new RowFormatBuilder<>(basePath, encoder, Preconditions.checkNotNull(assigner), rollingPolicy, bucketCheckInterval, bucketFactory, partFilePrefix, partFileSuffix); + public T withBucketCheckInterval(final long interval) { + this.bucketCheckInterval = interval; + return self(); } - /** - * Creates a new builder instance with the specified {@link RollingPolicy} set for the bucketing logic. - * @param policy {@link RollingPolicy} to be applied - * @return A new builder with the check interval set. - */ - public StreamingFileSink.RowFormatBuilder withRollingPolicy(final RollingPolicy policy) { - return new RowFormatBuilder<>(basePath, encoder, bucketAssigner, Preconditions.checkNotNull(policy), bucketCheckInterval, bucketFactory, partFilePrefix, partFileSuffix); + public T withBucketAssigner(final BucketAssigner assigner) { + this.bucketAssigner = Preconditions.checkNotNull(assigner); + return self(); } - public StreamingFileSink.RowFormatBuilder withBucketAssignerAndPolicy(final BucketAssigner assigner, final RollingPolicy policy) { - return new RowFormatBuilder<>(basePath, encoder, Preconditions.checkNotNull(assigner), Preconditions.checkNotNull(policy), bucketCheckInterval, new DefaultBucketFactoryImpl<>(), partFilePrefix, partFileSuffix); + public T withRollingPolicy(final RollingPolicy policy) { + this.rollingPolicy = Preconditions.checkNotNull(policy); + return self(); } - public StreamingFileSink.RowFormatBuilder withPartFilePrefix(final String partPrefix) { - return new RowFormatBuilder<>(basePath, encoder, bucketAssigner, rollingPolicy, bucketCheckInterval, bucketFactory, partPrefix, partFileSuffix); + public T withPartFilePrefix(final String partPrefix) { + this.partFilePrefix = partPrefix; + return self(); } - public StreamingFileSink.RowFormatBuilder withPartFileSuffix(final String partSuffix) { - return new RowFormatBuilder<>(basePath, encoder, bucketAssigner, rollingPolicy, bucketCheckInterval, bucketFactory, partFilePrefix, partSuffix); + public T withPartFileSuffix(final String partSuffix) { + this.partFileSuffix = partSuffix; + return self(); + } + + public StreamingFileSink.RowFormatBuilder> withNewBucketAssignerAndPolicy(final BucketAssigner assigner, final RollingPolicy policy) { + Preconditions.checkState(bucketFactory.getClass() == DefaultBucketFactoryImpl.class, "newBuilderWithBucketAssignerAndPolicy() cannot be called after specifying a customized bucket factory"); + return new RowFormatBuilder<>(basePath, encoder, Preconditions.checkNotNull(assigner), Preconditions.checkNotNull(policy), bucketCheckInterval, new DefaultBucketFactoryImpl<>(), partFilePrefix, partFileSuffix); } /** Creates the actual sink. */ @@ -269,6 +283,12 @@ public class StreamingFileSink return new StreamingFileSink<>(this, bucketCheckInterval); } + @VisibleForTesting + T withBucketFactory(final BucketFactory factory) { + this.bucketFactory = Preconditions.checkNotNull(factory); + return self(); + } + @Override Buckets createBuckets(int subtaskIndex) throws IOException { return new Buckets<>( @@ -280,40 +300,35 @@ public class StreamingFileSink subtaskIndex, new PartFileConfig(partFilePrefix, partFileSuffix)); } - - @VisibleForTesting - StreamingFileSink.RowFormatBuilder withBucketFactory(final BucketFactory factory) { - return new RowFormatBuilder<>(basePath, encoder, bucketAssigner, rollingPolicy, bucketCheckInterval, Preconditions.checkNotNull(factory), partFilePrefix, partFileSuffix); - } } /** * A builder for configuring the sink for bulk-encoding formats, e.g. Parquet/ORC. */ @PublicEvolving - public static class BulkFormatBuilder extends StreamingFileSink.BucketsBuilder { + public static class BulkFormatBuilder> extends StreamingFileSink.BucketsBuilder { private static final long serialVersionUID = 1L; - private final long bucketCheckInterval; + private long bucketCheckInterval; private final Path basePath; - private final BulkWriter.Factory writerFactory; + private BulkWriter.Factory writerFactory; - private final BucketAssigner bucketAssigner; + private BucketAssigner bucketAssigner; - private final BucketFactory bucketFactory; + private BucketFactory bucketFactory; - private final String partFilePrefix; + private String partFilePrefix; - private final String partFileSuffix; + private String partFileSuffix; - BulkFormatBuilder(Path basePath, BulkWriter.Factory writerFactory, BucketAssigner assigner) { - this(basePath, writerFactory, assigner, 60L * 1000L, new DefaultBucketFactoryImpl<>(), PartFileConfig.DEFAULT_PART_PREFIX, PartFileConfig.DEFAULT_PART_SUFFIX); + protected BulkFormatBuilder(Path basePath, BulkWriter.Factory writerFactory, BucketAssigner assigner) { + this(basePath, writerFactory, assigner, DEFAULT_BUCKET_CHECK_INTERVAL, new DefaultBucketFactoryImpl<>(), PartFileConfig.DEFAULT_PART_PREFIX, PartFileConfig.DEFAULT_PART_SUFFIX); } - private BulkFormatBuilder( + protected BulkFormatBuilder( Path basePath, BulkWriter.Factory writerFactory, BucketAssigner assigner, @@ -330,34 +345,39 @@ public class StreamingFileSink this.partFileSuffix = Preconditions.checkNotNull(partFileSuffix); } - /** - * Currently bulk formats always use the {@link OnCheckpointRollingPolicy} therefore this settings does - * not have any effect. - */ - public StreamingFileSink.BulkFormatBuilder withBucketCheckInterval(long interval) { - return new BulkFormatBuilder<>(basePath, writerFactory, bucketAssigner, interval, bucketFactory, partFilePrefix, partFileSuffix); + public long getBucketCheckInterval() { + return bucketCheckInterval; } - /** - * Creates a new builder instance with the specified {@link BucketAssigner}. - * @param assigner @{@link BucketAssigner} to be used. - * @return A new builder with the assigner set. - */ - public StreamingFileSink.BulkFormatBuilder withBucketAssigner(BucketAssigner assigner) { - return new BulkFormatBuilder<>(basePath, writerFactory, Preconditions.checkNotNull(assigner), bucketCheckInterval, new DefaultBucketFactoryImpl<>(), partFilePrefix, partFileSuffix); + public T withBucketCheckInterval(long interval) { + this.bucketCheckInterval = interval; + return self(); + } + + public T withBucketAssigner(BucketAssigner assigner) { + this.bucketAssigner = Preconditions.checkNotNull(assigner); + return self(); } @VisibleForTesting - StreamingFileSink.BulkFormatBuilder withBucketFactory(final BucketFactory factory) { - return new BulkFormatBuilder<>(basePath, writerFactory, bucketAssigner, bucketCheckInterval, Preconditions.checkNotNull(factory), partFilePrefix, partFileSuffix); + T withBucketFactory(final BucketFactory factory) { + this.bucketFactory = Preconditions.checkNotNull(factory); + return self(); } - public StreamingFileSink.BulkFormatBuilder withPartFilePrefix(final String partPrefix) { - return new BulkFormatBuilder<>(basePath, writerFactory, bucketAssigner, bucketCheckInterval, bucketFactory, partPrefix, partFileSuffix); + public T withPartFilePrefix(final String partPrefix) { + this.partFilePrefix = partPrefix; + return self(); + } + + public T withPartFileSuffix(final String partSuffix) { + this.partFileSuffix = partSuffix; + return self(); } - public StreamingFileSink.BulkFormatBuilder withPartFileSuffix(final String partSuffix) { - return new BulkFormatBuilder<>(basePath, writerFactory, bucketAssigner, bucketCheckInterval, bucketFactory, partFilePrefix, partSuffix); + public StreamingFileSink.BulkFormatBuilder> withNewBucketAssigner(final BucketAssigner assigner) { + Preconditions.checkState(bucketFactory.getClass() == DefaultBucketFactoryImpl.class, "newBuilderWithBucketAssigner() cannot be called after specifying a customized bucket factory"); + return new BulkFormatBuilder<>(basePath, writerFactory, Preconditions.checkNotNull(assigner), bucketCheckInterval, new DefaultBucketFactoryImpl<>(), partFilePrefix, partFileSuffix); } /** Creates the actual sink. */ diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java index a4f83c1af8c966fb9abdcdbb8337b04e3a08137f..5384541030d3611c4d9bbfc9e62b66f3d84ad29f 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamConfig.java @@ -58,6 +58,9 @@ public class StreamConfig implements Serializable { // Config Keys // ------------------------------------------------------------------------ + @VisibleForTesting + public static final String SERIALIZEDUDF = "serializedUDF"; + private static final String NUMBER_OF_OUTPUTS = "numberOfOutputs"; private static final String NUMBER_OF_INPUTS = "numberOfInputs"; private static final String CHAINED_OUTPUTS = "chainedOutputs"; @@ -67,7 +70,6 @@ public class StreamConfig implements Serializable { private static final String VERTEX_NAME = "vertexID"; private static final String ITERATION_ID = "iterationId"; private static final String OUTPUT_SELECTOR_WRAPPER = "outputSelectorWrapper"; - private static final String SERIALIZEDUDF = "serializedUDF"; private static final String BUFFER_TIMEOUT = "bufferTimeout"; private static final String TYPE_SERIALIZER_IN_1 = "typeSerializer_in_1"; private static final String TYPE_SERIALIZER_IN_2 = "typeSerializer_in_2"; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java index f4741fb315a7f24f2b18ccbd3a82b9fcb61407b5..a8b322f81efd63c182582a439c39a6e119de3fd9 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraph.java @@ -26,12 +26,13 @@ import org.apache.flink.api.common.io.OutputFormat; import org.apache.flink.api.common.operators.ResourceSpec; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.dag.Pipeline; import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.tuple.Tuple3; import org.apache.flink.api.java.typeutils.MissingTypeInfo; -import org.apache.flink.optimizer.plan.StreamingPlan; import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.runtime.jobgraph.ScheduleMode; import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable; import org.apache.flink.runtime.state.StateBackend; @@ -71,7 +72,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull; * */ @Internal -public class StreamGraph extends StreamingPlan { +public class StreamGraph implements Pipeline { private static final Logger LOG = LoggerFactory.getLogger(StreamGraph.class); @@ -83,6 +84,7 @@ public class StreamGraph extends StreamingPlan { private final ExecutionConfig executionConfig; private final CheckpointConfig checkpointConfig; + private SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.none(); private ScheduleMode scheduleMode; @@ -110,9 +112,10 @@ public class StreamGraph extends StreamingPlan { private StateBackend stateBackend; private Set> iterationSourceSinkPairs; - public StreamGraph(ExecutionConfig executionConfig, CheckpointConfig checkpointConfig) { + public StreamGraph(ExecutionConfig executionConfig, CheckpointConfig checkpointConfig, SavepointRestoreSettings savepointRestoreSettings) { this.executionConfig = checkNotNull(executionConfig); this.checkpointConfig = checkNotNull(checkpointConfig); + this.savepointRestoreSettings = checkNotNull(savepointRestoreSettings); // create an empty new stream graph. clear(); @@ -141,6 +144,14 @@ public class StreamGraph extends StreamingPlan { return checkpointConfig; } + public void setSavepointRestoreSettings(SavepointRestoreSettings savepointRestoreSettings) { + this.savepointRestoreSettings = savepointRestoreSettings; + } + + public SavepointRestoreSettings getSavepointRestoreSettings() { + return savepointRestoreSettings; + } + public String getJobName() { return jobName; } @@ -715,14 +726,19 @@ public class StreamGraph extends StreamingPlan { } /** - * Gets the assembled {@link JobGraph} with a given job id. + * Gets the assembled {@link JobGraph} with a random {@link JobID}. + */ + public JobGraph getJobGraph() { + return getJobGraph(null); + } + + /** + * Gets the assembled {@link JobGraph} with a specified {@link JobID}. */ - @Override public JobGraph getJobGraph(@Nullable JobID jobID) { return StreamingJobGraphGenerator.createJobGraph(this, jobID); } - @Override public String getStreamingPlanAsJSON() { try { return new JSONGenerator(this).getJSON(); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java index bd6a4b54b328796abb750ab73423e9c64fbbe7cc..9a4eafd967e071c95baaa7b9ad196fea715c4c09 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphGenerator.java @@ -24,6 +24,7 @@ import org.apache.flink.api.common.cache.DistributedCache; import org.apache.flink.api.common.typeutils.TypeSerializer; import org.apache.flink.api.dag.Transformation; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.runtime.jobgraph.ScheduleMode; import org.apache.flink.runtime.state.KeyGroupRangeAssignment; import org.apache.flink.runtime.state.StateBackend; @@ -107,6 +108,8 @@ public class StreamGraphGenerator { private final CheckpointConfig checkpointConfig; + private SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.none(); + private StateBackend stateBackend; private boolean chaining = true; @@ -193,8 +196,12 @@ public class StreamGraphGenerator { return this; } + public void setSavepointRestoreSettings(SavepointRestoreSettings savepointRestoreSettings) { + this.savepointRestoreSettings = savepointRestoreSettings; + } + public StreamGraph generate() { - streamGraph = new StreamGraph(executionConfig, checkpointConfig); + streamGraph = new StreamGraph(executionConfig, checkpointConfig, savepointRestoreSettings); streamGraph.setStateBackend(stateBackend); streamGraph.setChaining(chaining); streamGraph.setScheduleMode(scheduleMode); diff --git a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FaultyJobGraphStore.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphTranslator.java similarity index 39% rename from flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FaultyJobGraphStore.java rename to flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphTranslator.java index 3ce028c929bf83831fc9f6cab9f3a94a4957e9eb..4a43adbb2ed0230d5e42e31659b85d34d622e4d6 100644 --- a/flink-runtime/src/test/java/org/apache/flink/runtime/dispatcher/FaultyJobGraphStore.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamGraphTranslator.java @@ -14,51 +14,56 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ -package org.apache.flink.runtime.dispatcher; +package org.apache.flink.streaming.api.graph; -import org.apache.flink.api.common.JobID; +import org.apache.flink.api.dag.Pipeline; +import org.apache.flink.client.FlinkPipelineTranslator; +import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.testutils.InMemoryJobGraphStore; -import javax.annotation.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.flink.util.Preconditions.checkArgument; /** - * {@link InMemoryJobGraphStore} implementation which can throw artifical errors for - * testing purposes. + * {@link FlinkPipelineTranslator} for DataStream API {@link StreamGraph StreamGraphs}. + * + *

    Note: this is used through reflection in + * {@link org.apache.flink.client.FlinkPipelineTranslationUtil}. */ -final class FaultyJobGraphStore extends InMemoryJobGraphStore { - - @Nullable - private Exception recoveryFailure = null; +@SuppressWarnings("unused") +public class StreamGraphTranslator implements FlinkPipelineTranslator { - @Nullable - private Exception removalFailure = null; + private static final Logger LOG = LoggerFactory.getLogger(StreamGraphTranslator.class); - void setRecoveryFailure(@Nullable Exception recoveryFailure) { - this.recoveryFailure = recoveryFailure; - } + @Override + public JobGraph translateToJobGraph( + Pipeline pipeline, + Configuration optimizerConfiguration, + int defaultParallelism) { + checkArgument(pipeline instanceof StreamGraph, + "Given pipeline is not a DataStream StreamGraph."); - void setRemovalFailure(@Nullable Exception removalFailure) { - this.removalFailure = removalFailure; + StreamGraph streamGraph = (StreamGraph) pipeline; + return streamGraph.getJobGraph(null); } @Override - public synchronized JobGraph recoverJobGraph(JobID jobId) throws Exception { - if (recoveryFailure != null) { - throw recoveryFailure; - } else { - return super.recoverJobGraph(jobId); - } + public String translateToJSONExecutionPlan(Pipeline pipeline) { + checkArgument(pipeline instanceof StreamGraph, + "Given pipeline is not a DataStream StreamGraph."); + + StreamGraph streamGraph = (StreamGraph) pipeline; + + return streamGraph.getStreamingPlanAsJSON(); } @Override - public synchronized void removeJobGraph(JobID jobId) throws Exception { - if (removalFailure != null) { - throw removalFailure; - } else { - super.removeJobGraph(jobId); - } + public boolean canTranslate(Pipeline pipeline) { + return pipeline instanceof StreamGraph; } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java index 58b4989dd7bd39cfd10ea37ad049349e8693ee73..83a81031526deb794ee995cbc718143525ca3062 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGenerator.java @@ -166,6 +166,8 @@ public class StreamingJobGraphGenerator { configureCheckpointing(); + jobGraph.setSavepointRestoreSettings(streamGraph.getSavepointRestoreSettings()); + JobGraphGenerator.addUserArtifactEntries(streamGraph.getUserArtifacts(), jobGraph); // set the ExecutionConfig last when it has been finalized diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceReaderOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceReaderOperator.java new file mode 100644 index 0000000000000000000000000000000000000000..8b73d966520a3b8dca4e31d202e28e254a10379a --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/SourceReaderOperator.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.operators; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput; + +/** + * Base source operator only used for integrating the source reader which is proposed by FLIP-27. It implements + * the interface of {@link PushingAsyncDataInput} for naturally compatible with one input processing in runtime + * stack. + * + *

    Note: We are expecting this to be changed to the concrete class once SourceReader interface is introduced. + * + * @param The output type of the operator + */ +@Internal +public abstract class SourceReaderOperator extends AbstractStreamOperator implements PushingAsyncDataInput { +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java index ded272c5d3544cde06f5c32ffb89f932b3cbd901..fded06129e270adcbc924b38cb7b39e38665d78c 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/StreamOperator.java @@ -54,6 +54,10 @@ public interface StreamOperator extends CheckpointListener, KeyContext, Dis * This method is called immediately before any elements are processed, it should contain the * operator's initialization logic. * + * @implSpec In case of recovery, this method needs to ensure that all recovered data is processed before passing + * back control, so that the order of elements is ensured during the recovery of an operator chain (operators + * are opened from the tail operator to the head operator). + * * @throws java.lang.Exception An exception in this method causes the operator to fail. */ void open() throws Exception; diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimestampedCollector.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimestampedCollector.java index e0e09694f17c23ee902e8853f359d0f46a6e6483..0c705103b8c49fe9444417357e681c744b9a3687 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimestampedCollector.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/TimestampedCollector.java @@ -19,11 +19,13 @@ package org.apache.flink.streaming.api.operators; import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.util.Collector; +import org.apache.flink.util.OutputTag; /** - * Wrapper around an {@link Output} for user functions that expect a {@link Collector}. + * Wrapper around an {@link Output} for user functions that expect a {@link Output}. * Before giving the {@link TimestampedCollector} to a user function you must set * the timestamp that should be attached to emitted elements. Most operators * would set the timestamp of the incoming @@ -32,7 +34,7 @@ import org.apache.flink.util.Collector; * @param The type of the elements that can be emitted. */ @Internal -public class TimestampedCollector implements Collector { +public final class TimestampedCollector implements Output { private final Output> output; @@ -71,4 +73,19 @@ public class TimestampedCollector implements Collector { public void close() { output.close(); } + + @Override + public void emitWatermark(Watermark mark) { + output.emitWatermark(mark); + } + + @Override + public void collect(OutputTag outputTag, StreamRecord record) { + output.collect(outputTag, record); + } + + @Override + public void emitLatencyMarker(LatencyMarker latencyMarker) { + output.emitLatencyMarker(latencyMarker); + } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java index 6fb9691f7f89c5b51e7258802bfaa00c081774f9..85161aacee1f8085357b66479eff2d0a8df8afe3 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperator.java @@ -34,29 +34,25 @@ import org.apache.flink.streaming.api.operators.BoundedOneInput; import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.operators.TimestampedCollector; import org.apache.flink.streaming.api.operators.async.queue.OrderedStreamElementQueue; import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue; -import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueueEntry; -import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.streaming.api.operators.async.queue.UnorderedStreamElementQueue; -import org.apache.flink.streaming.api.operators.async.queue.WatermarkQueueEntry; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamElement; import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback; import org.apache.flink.streaming.runtime.tasks.StreamTask; import org.apache.flink.streaming.runtime.tasks.mailbox.execution.MailboxExecutor; -import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.Preconditions; import javax.annotation.Nonnull; import java.util.Collection; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.Collections; +import java.util.Optional; import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; /** * The {@link AsyncWaitOperator} allows to asynchronously process incoming stream records. For that @@ -80,7 +76,7 @@ import java.util.concurrent.TimeUnit; @Internal public class AsyncWaitOperator extends AbstractUdfStreamOperator> - implements OneInputStreamOperator, OperatorActions, BoundedOneInput { + implements OneInputStreamOperator, BoundedOneInput { private static final long serialVersionUID = 1L; private static final String STATE_NAME = "_async_wait_operator_state_"; @@ -94,7 +90,7 @@ public class AsyncWaitOperator /** Timeout for the async collectors. */ private final long timeout; - protected transient Object checkpointingLock; + private transient Object checkpointingLock; /** {@link TypeSerializer} for inputs while making snapshots. */ private transient StreamElementSerializer inStreamElementSerializer; @@ -102,23 +98,14 @@ public class AsyncWaitOperator /** Recovered input stream elements. */ private transient ListState recoveredStreamElements; - /** Queue to store the currently in-flight stream elements into. */ - private transient StreamElementQueue queue; - - /** Pending stream element which could not yet added to the queue. */ - private transient StreamElementQueueEntry pendingStreamElementQueueEntry; - - private transient ExecutorService executor; - - /** Emitter for the completed stream element queue entries. */ - private transient Emitter emitter; - - /** Thread running the emitter. */ - private transient Thread emitterThread; + /** Queue, into which to store the currently in-flight stream elements. */ + private transient StreamElementQueue queue; /** Mailbox executor used to yield while waiting for buffers to empty. */ private final transient MailboxExecutor mailboxExecutor; + private transient TimestampedCollector timestampedCollector; + public AsyncWaitOperator( @Nonnull AsyncFunction asyncFunction, long timeout, @@ -150,43 +137,24 @@ public class AsyncWaitOperator this.inStreamElementSerializer = new StreamElementSerializer<>( getOperatorConfig().getTypeSerializerIn1(getUserCodeClassloader())); - // create the operators executor for the complete operations of the queue entries - this.executor = Executors.newSingleThreadExecutor(); - switch (outputMode) { case ORDERED: - queue = new OrderedStreamElementQueue( - capacity, - executor, - this); + queue = new OrderedStreamElementQueue<>(capacity); break; case UNORDERED: - queue = new UnorderedStreamElementQueue( - capacity, - executor, - this); + queue = new UnorderedStreamElementQueue<>(capacity); break; default: throw new IllegalStateException("Unknown async mode: " + outputMode + '.'); } + + this.timestampedCollector = new TimestampedCollector<>(output); } @Override public void open() throws Exception { super.open(); - // create the emitter - this.emitter = new Emitter<>(checkpointingLock, mailboxExecutor, output, queue, this); - - // start the emitter thread - this.emitterThread = new Thread(emitter, "AsyncIO-Emitter-Thread (" + getOperatorName() + ')'); - emitterThread.setDaemon(true); - emitterThread.start(); - - // process stream elements from state, since the Emit thread will start as soon as all - // elements from previous state are in the StreamElementQueue, we have to make sure that the - // order to open all operators in the operator chain proceeds from the tail operator to the - // head operator. if (recoveredStreamElements != null) { for (StreamElement element : recoveredStreamElements.get()) { if (element.isRecord()) { @@ -210,40 +178,33 @@ public class AsyncWaitOperator @Override public void processElement(StreamRecord element) throws Exception { - final StreamRecordQueueEntry streamRecordBufferEntry = new StreamRecordQueueEntry<>(element); + // add element first to the queue + final ResultFuture entry = addToWorkQueue(element); + final ResultHandler resultHandler = new ResultHandler(element, entry); + + // register a timeout for the entry if timeout is configured if (timeout > 0L) { - // register a timeout for this AsyncStreamRecordBufferEntry - long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime(); + final long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime(); - final ScheduledFuture timerFuture = getProcessingTimeService().registerTimer( + final ScheduledFuture timeoutTimer = getProcessingTimeService().registerTimer( timeoutTimestamp, - new ProcessingTimeCallback() { - @Override - public void onProcessingTime(long timestamp) throws Exception { - userFunction.timeout(element.getValue(), streamRecordBufferEntry); - } - }); - - // Cancel the timer once we've completed the stream record buffer entry. This will remove - // the register trigger task - streamRecordBufferEntry.onComplete( - (StreamElementQueueEntry> value) -> { - timerFuture.cancel(true); - }, - executor); - } + timestamp -> userFunction.timeout(element.getValue(), resultHandler)); - addAsyncBufferEntry(streamRecordBufferEntry); + resultHandler.setTimeoutTimer(timeoutTimer); + } - userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry); + userFunction.asyncInvoke(element.getValue(), resultHandler); } @Override public void processWatermark(Watermark mark) throws Exception { - WatermarkQueueEntry watermarkBufferEntry = new WatermarkQueueEntry(mark); + addToWorkQueue(mark); - addAsyncBufferEntry(watermarkBufferEntry); + // watermarks are always completed + // if there is no prior element, we can directly emit them + // this also avoids watermarks being held back until the next element has been processed + outputCompletedElement(); } @Override @@ -254,17 +215,8 @@ public class AsyncWaitOperator getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer)); partitionableState.clear(); - Collection> values = queue.values(); - try { - for (StreamElementQueueEntry value : values) { - partitionableState.add(value.getStreamElement()); - } - - // add the pending stream element queue entry if the stream element queue is currently full - if (pendingStreamElementQueueEntry != null) { - partitionableState.add(pendingStreamElementQueueEntry.getStreamElement()); - } + partitionableState.addAll(queue.values()); } catch (Exception e) { partitionableState.clear(); @@ -293,140 +245,137 @@ public class AsyncWaitOperator waitInFlightInputsFinished(); } finally { - Exception exception = null; - - try { - super.close(); - } catch (InterruptedException interrupted) { - exception = interrupted; - - Thread.currentThread().interrupt(); - } catch (Exception e) { - exception = e; - } - - try { - // terminate the emitter, the emitter thread and the executor - stopResources(true); - } catch (InterruptedException interrupted) { - exception = ExceptionUtils.firstOrSuppressed(interrupted, exception); - - Thread.currentThread().interrupt(); - } catch (Exception e) { - exception = ExceptionUtils.firstOrSuppressed(e, exception); - } - - if (exception != null) { - LOG.warn("Errors occurred while closing the AsyncWaitOperator.", exception); - } + super.close(); } } - @Override - public void dispose() throws Exception { - Exception exception = null; - - try { - super.dispose(); - } catch (InterruptedException interrupted) { - exception = interrupted; + /** + * Add the given stream element to the operator's stream element queue. This operation blocks until the element + * has been added. + * + *

    Between two insertion attempts, this method yields the execution to the mailbox, such that events as well + * as asynchronous results can be processed. + * + * @param streamElement to add to the operator's queue + * @throws InterruptedException if the current thread has been interrupted while yielding to mailbox + * @return a handle that allows to set the result of the async computation for the given element. + */ + private ResultFuture addToWorkQueue(StreamElement streamElement) throws InterruptedException { + assert(Thread.holdsLock(checkpointingLock)); - Thread.currentThread().interrupt(); - } catch (Exception e) { - exception = e; + Optional> queueEntry; + while (!(queueEntry = queue.tryPut(streamElement)).isPresent()) { + mailboxExecutor.yield(); } - try { - stopResources(false); - } catch (InterruptedException interrupted) { - exception = ExceptionUtils.firstOrSuppressed(interrupted, exception); + return queueEntry.get(); + } - Thread.currentThread().interrupt(); - } catch (Exception e) { - exception = ExceptionUtils.firstOrSuppressed(e, exception); - } + private void waitInFlightInputsFinished() throws InterruptedException { + assert(Thread.holdsLock(checkpointingLock)); - if (exception != null) { - throw exception; + while (!queue.isEmpty()) { + mailboxExecutor.yield(); } } /** - * Close the operator's resources. They include the emitter thread and the executor to run - * the queue's complete operation. + * Outputs one completed element. Watermarks are always completed if it's their turn to be processed. * - * @param waitForShutdown is true if the method should wait for the resources to be freed; - * otherwise false. - * @throws InterruptedException if current thread has been interrupted + *

    This method will be called from {@link #processWatermark(Watermark)} and from a mail processing the result + * of an async function call. */ - private void stopResources(boolean waitForShutdown) throws InterruptedException { - emitter.stop(); - emitterThread.interrupt(); - - executor.shutdown(); - - if (waitForShutdown) { - try { - if (!executor.awaitTermination(365L, TimeUnit.DAYS)) { - executor.shutdownNow(); - } - } catch (InterruptedException e) { - executor.shutdownNow(); - - Thread.currentThread().interrupt(); + private void outputCompletedElement() { + if (queue.hasCompletedElements()) { + // emit only one element to not block the mailbox thread unnecessarily + synchronized (checkpointingLock) { + queue.emitCompletedElement(timestampedCollector); } - - /* - * FLINK-5638: If we have the checkpoint lock we might have to free it for a while so - * that the emitter thread can complete/react to the interrupt signal. - */ - if (Thread.holdsLock(checkpointingLock)) { - while (emitterThread.isAlive()) { - checkpointingLock.wait(100L); - } + // if there are more completed elements, emit them with subsequent mails + if (queue.hasCompletedElements()) { + mailboxExecutor.execute(this::outputCompletedElement); } - - emitterThread.join(); - } else { - executor.shutdownNow(); } } /** - * Add the given stream element queue entry to the operator's stream element queue. This - * operation blocks until the element has been added. - * - *

    For that it tries to put the element into the queue and if not successful then it waits on - * the checkpointing lock. The checkpointing lock is also used by the {@link Emitter} to output - * elements. The emitter is also responsible for notifying this method if the queue has capacity - * left again, by calling notifyAll on the checkpointing lock. - * - * @param streamElementQueueEntry to add to the operator's queue - * @param Type of the stream element queue entry's result - * @throws InterruptedException if the current thread has been interrupted + * A handler for the results of a specific input record. */ - private void addAsyncBufferEntry(StreamElementQueueEntry streamElementQueueEntry) throws InterruptedException { - assert(Thread.holdsLock(checkpointingLock)); - - pendingStreamElementQueueEntry = streamElementQueueEntry; + private class ResultHandler implements ResultFuture { + /** + * Optional timeout timer used to signal the timeout to the AsyncFunction. + */ + private ScheduledFuture timeoutTimer; + /** + * Record for which this result handler exists. Used only to report errors. + */ + private final StreamRecord inputRecord; + /** + * The handle received from the queue to update the entry. Should only be used to inject the result; + * exceptions are handled here. + */ + private final ResultFuture resultFuture; + /** + * A guard against ill-written AsyncFunction. Additional (parallel) invokations of + * {@link #complete(Collection)} or {@link #completeExceptionally(Throwable)} will be ignored. This guard + * also helps for cases where proper results and timeouts happen at the same time. + */ + private final AtomicBoolean completed = new AtomicBoolean(false); + + ResultHandler(StreamRecord inputRecord, ResultFuture resultFuture) { + this.inputRecord = inputRecord; + this.resultFuture = resultFuture; + } - while (!queue.tryPut(streamElementQueueEntry)) { - mailboxExecutor.yield(); + void setTimeoutTimer(ScheduledFuture timeoutTimer) { + this.timeoutTimer = timeoutTimer; } - pendingStreamElementQueueEntry = null; - } + @Override + public void complete(Collection results) { + Preconditions.checkNotNull(results, "Results must not be null, use empty collection to emit nothing"); - private void waitInFlightInputsFinished() throws InterruptedException { - assert(Thread.holdsLock(checkpointingLock)); + // already completed (exceptionally or with previous complete call from ill-written AsyncFunction), so + // ignore additional result + if (!completed.compareAndSet(false, true)) { + return; + } - while (!queue.isEmpty()) { - mailboxExecutor.yield(); + processInMailbox(results); } - } - @Override - public void failOperator(Throwable throwable) { - getContainingTask().getEnvironment().failExternally(throwable); + private void processInMailbox(Collection results) { + // move further processing into the mailbox thread + mailboxExecutor.execute(() -> { + // Cancel the timer once we've completed the stream record buffer entry. This will remove the registered + // timer task + if (timeoutTimer != null) { + // canceling in mailbox thread avoids https://issues.apache.org/jira/browse/FLINK-13635 + timeoutTimer.cancel(true); + } + + // update the queue entry with the result + resultFuture.complete(results); + // now output all elements from the queue that have been completed (in the correct order) + outputCompletedElement(); + }); + } + + @Override + public void completeExceptionally(Throwable error) { + // already completed, so ignore exception + if (!completed.compareAndSet(false, true)) { + return; + } + + // signal failure through task + getContainingTask().getEnvironment().failExternally(new Exception( + "Could not complete the stream element: " + inputRecord + '.', + error)); + + // complete with empty result, so that we remove timer and move ahead processing (to leave potentially + // blocking section in #addToWorkQueue or #waitInFlightInputsFinished) + processInMailbox(Collections.emptyList()); + } } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java deleted file mode 100644 index 16e47f29bc6f74c90f21a25603ef9f360bb3c9e7..0000000000000000000000000000000000000000 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/Emitter.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.streaming.api.operators.async; - -import org.apache.flink.annotation.Internal; -import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.streaming.api.operators.TimestampedCollector; -import org.apache.flink.streaming.api.operators.async.queue.AsyncCollectionResult; -import org.apache.flink.streaming.api.operators.async.queue.AsyncResult; -import org.apache.flink.streaming.api.operators.async.queue.AsyncWatermarkResult; -import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue; -import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.runtime.tasks.mailbox.execution.MailboxExecutor; -import org.apache.flink.util.Preconditions; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nonnull; - -import java.util.Collection; - -/** - * Runnable responsible for consuming elements from the given queue and outputting them to the - * given output/timestampedCollector. - * - * @param Type of the output elements - */ -@Internal -public class Emitter implements Runnable { - - private static final Logger LOG = LoggerFactory.getLogger(Emitter.class); - - /** Lock to hold before outputting. */ - private final Object checkpointLock; - - /** Output for the watermark elements. */ - private final Output> output; - - /** Executor for mailbox. */ - private final MailboxExecutor executor; - - /** Queue to consume the async results from. */ - private final StreamElementQueue streamElementQueue; - - private final OperatorActions operatorActions; - - /** Output for stream records. */ - private final TimestampedCollector timestampedCollector; - - private volatile boolean running; - - public Emitter( - final Object checkpointLock, - final @Nonnull MailboxExecutor executor, - final @Nonnull Output> output, - final @Nonnull StreamElementQueue streamElementQueue, - final @Nonnull OperatorActions operatorActions) { - - this.checkpointLock = Preconditions.checkNotNull(checkpointLock, "checkpointLock"); - this.executor = Preconditions.checkNotNull(executor, "executor"); - this.output = Preconditions.checkNotNull(output, "output"); - this.streamElementQueue = Preconditions.checkNotNull(streamElementQueue, "streamElementQueue"); - this.operatorActions = Preconditions.checkNotNull(operatorActions, "operatorActions"); - - this.timestampedCollector = new TimestampedCollector<>(this.output); - this.running = true; - } - - @Override - public void run() { - try { - while (running) { - LOG.debug("Wait for next completed async stream element result."); - AsyncResult asyncResult = streamElementQueue.peekBlockingly(); - executor.submit(() -> { - try { - output(asyncResult); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }).get(); - } - } catch (InterruptedException e) { - if (running) { - operatorActions.failOperator(e); - } else { - // Thread got interrupted which means that it should shut down - LOG.debug("Emitter thread got interrupted, shutting down."); - } - } catch (Throwable t) { - operatorActions.failOperator(new Exception("AsyncWaitOperator's emitter caught an " + - "unexpected throwable.", t)); - } - } - - /** - * Executed as a mail in the mailbox thread. Output needs to be guarded with checkpoint lock (for the time being). - * - * @param asyncResult the result to output. - */ - private void output(AsyncResult asyncResult) throws InterruptedException { - if (asyncResult.isWatermark()) { - AsyncWatermarkResult asyncWatermarkResult = asyncResult.asWatermark(); - - LOG.debug("Output async watermark."); - - synchronized (checkpointLock) { - output.emitWatermark(asyncWatermarkResult.getWatermark()); - // remove the peeked element from the async collector buffer so that it is no longer - // checkpointed - streamElementQueue.poll(); - } - } else { - AsyncCollectionResult streamRecordResult = asyncResult.asResultCollection(); - - if (streamRecordResult.hasTimestamp()) { - timestampedCollector.setAbsoluteTimestamp(streamRecordResult.getTimestamp()); - } else { - timestampedCollector.eraseTimestamp(); - } - - LOG.debug("Output async stream element collection result."); - - synchronized (checkpointLock) { - try { - Collection resultCollection = streamRecordResult.get(); - - if (resultCollection != null) { - for (OUT result : resultCollection) { - timestampedCollector.collect(result); - } - } - } catch (Exception e) { - operatorActions.failOperator( - new Exception("An async function call terminated with an exception. " + - "Failing the AsyncWaitOperator.", e)); - } - - // remove the peeked element from the async collector buffer so that it is no longer - // checkpointed - streamElementQueue.poll(); - } - } - } - - public void stop() { - running = false; - } -} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java index 51338090d06265c80494a352d9cedbd94b78e002..e0c680ca800826eb49c715f86ff3f5aa83f4af4d 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueue.java @@ -19,117 +19,66 @@ package org.apache.flink.streaming.api.operators.async.queue; import org.apache.flink.annotation.Internal; -import org.apache.flink.streaming.api.operators.async.OperatorActions; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.streaming.api.operators.TimestampedCollector; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamElement; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayDeque; -import java.util.Arrays; -import java.util.Collection; -import java.util.concurrent.Executor; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Queue; /** - * Ordered {@link StreamElementQueue} implementation. The ordered stream element queue emits + * Ordered {@link StreamElementQueue} implementation. The ordered stream element queue provides * asynchronous results in the order in which the {@link StreamElementQueueEntry} have been added * to the queue. Thus, even if the completion order can be arbitrary, the output order strictly * follows the insertion order (element cannot overtake each other). */ @Internal -public class OrderedStreamElementQueue implements StreamElementQueue { +public final class OrderedStreamElementQueue implements StreamElementQueue { private static final Logger LOG = LoggerFactory.getLogger(OrderedStreamElementQueue.class); /** Capacity of this queue. */ private final int capacity; - /** Executor to run the onCompletion callback. */ - private final Executor executor; - - /** Operator actions to signal a failure to the operator. */ - private final OperatorActions operatorActions; - - /** Lock and conditions for the blocking queue. */ - private final ReentrantLock lock; - private final Condition notFull; - private final Condition headIsCompleted; - /** Queue for the inserted StreamElementQueueEntries. */ - private final ArrayDeque> queue; - - public OrderedStreamElementQueue( - int capacity, - Executor executor, - OperatorActions operatorActions) { + private final Queue> queue; + public OrderedStreamElementQueue(int capacity) { Preconditions.checkArgument(capacity > 0, "The capacity must be larger than 0."); - this.capacity = capacity; - - this.executor = Preconditions.checkNotNull(executor, "executor"); - - this.operatorActions = Preconditions.checkNotNull(operatorActions, "operatorActions"); - - this.lock = new ReentrantLock(false); - this.headIsCompleted = lock.newCondition(); - this.notFull = lock.newCondition(); + this.capacity = capacity; this.queue = new ArrayDeque<>(capacity); } @Override - public AsyncResult peekBlockingly() throws InterruptedException { - lock.lockInterruptibly(); - - try { - while (queue.isEmpty() || !queue.peek().isDone()) { - headIsCompleted.await(); - } - - LOG.debug("Peeked head element from ordered stream element queue with filling degree " + - "({}/{}).", queue.size(), capacity); - - return queue.peek(); - } finally { - lock.unlock(); - } + public boolean hasCompletedElements() { + return !queue.isEmpty() && queue.peek().isDone(); } @Override - public AsyncResult poll() throws InterruptedException { - lock.lockInterruptibly(); - - try { - while (queue.isEmpty() || !queue.peek().isDone()) { - headIsCompleted.await(); - } - - notFull.signalAll(); - - LOG.debug("Polled head element from ordered stream element queue. New filling degree " + - "({}/{}).", queue.size() - 1, capacity); - - return queue.poll(); - } finally { - lock.unlock(); + public void emitCompletedElement(TimestampedCollector output) { + if (hasCompletedElements()) { + final StreamElementQueueEntry head = queue.poll(); + head.emitResult(output); } } @Override - public Collection> values() throws InterruptedException { - lock.lockInterruptibly(); - - try { - StreamElementQueueEntry[] array = new StreamElementQueueEntry[queue.size()]; - - array = queue.toArray(array); - - return Arrays.asList(array); - } finally { - lock.unlock(); + public List values() { + List list = new ArrayList<>(this.queue.size()); + for (StreamElementQueueEntry e : queue) { + list.add(e.getInputElement()); } + return list; } @Override @@ -143,88 +92,31 @@ public class OrderedStreamElementQueue implements StreamElementQueue { } @Override - public void put(StreamElementQueueEntry streamElementQueueEntry) throws InterruptedException { - lock.lockInterruptibly(); - - try { - while (queue.size() >= capacity) { - notFull.await(); - } + public Optional> tryPut(StreamElement streamElement) { + if (queue.size() < capacity) { + StreamElementQueueEntry queueEntry = createEntry(streamElement); - addEntry(streamElementQueueEntry); - } finally { - lock.unlock(); - } - } + queue.add(queueEntry); - @Override - public boolean tryPut(StreamElementQueueEntry streamElementQueueEntry) throws InterruptedException { - lock.lockInterruptibly(); - - try { - if (queue.size() < capacity) { - addEntry(streamElementQueueEntry); - - LOG.debug("Put element into ordered stream element queue. New filling degree " + - "({}/{}).", queue.size(), capacity); + LOG.debug("Put element into ordered stream element queue. New filling degree " + + "({}/{}).", queue.size(), capacity); - return true; - } else { - LOG.debug("Failed to put element into ordered stream element queue because it " + - "was full ({}/{}).", queue.size(), capacity); + return Optional.of(queueEntry); + } else { + LOG.debug("Failed to put element into ordered stream element queue because it " + + "was full ({}/{}).", queue.size(), capacity); - return false; - } - } finally { - lock.unlock(); + return Optional.empty(); } } - /** - * Add the given {@link StreamElementQueueEntry} to the queue. Additionally, this method - * registers a onComplete callback which is triggered once the given queue entry is completed. - * - * @param streamElementQueueEntry to be inserted - * @param Type of the stream element queue entry's result - */ - private void addEntry(StreamElementQueueEntry streamElementQueueEntry) { - assert(lock.isHeldByCurrentThread()); - - queue.addLast(streamElementQueueEntry); - - streamElementQueueEntry.onComplete( - (StreamElementQueueEntry value) -> { - try { - onCompleteHandler(value); - } catch (InterruptedException e) { - // we got interrupted. This indicates a shutdown of the executor - LOG.debug("AsyncBufferEntry could not be properly completed because the " + - "executor thread has been interrupted.", e); - } catch (Throwable t) { - operatorActions.failOperator(new Exception("Could not complete the " + - "stream element queue entry: " + value + '.', t)); - } - }, - executor); - } - - /** - * Check if the completed {@link StreamElementQueueEntry} is the current head. If this is the - * case, then notify the consumer thread about a new consumable entry. - * - * @param streamElementQueueEntry which has been completed - * @throws InterruptedException if the current thread is interrupted - */ - private void onCompleteHandler(StreamElementQueueEntry streamElementQueueEntry) throws InterruptedException { - lock.lockInterruptibly(); - - try { - if (!queue.isEmpty() && queue.peek().isDone()) { - LOG.debug("Signal ordered stream element queue has completed head element."); - headIsCompleted.signalAll(); - } - } finally { - lock.unlock(); + private StreamElementQueueEntry createEntry(StreamElement streamElement) { + if (streamElement.isRecord()) { + return new StreamRecordQueueEntry<>((StreamRecord) streamElement); + } + if (streamElement.isWatermark()) { + return new WatermarkQueueEntry<>((Watermark) streamElement); } + throw new UnsupportedOperationException("Cannot enqueue " + streamElement); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueue.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueue.java index e02b8b03ba659a39151d67c24033a286c2cd6244..d896e8b0b6fa9b08956d38c72215550a4e529eab 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueue.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueue.java @@ -19,68 +19,56 @@ package org.apache.flink.streaming.api.operators.async.queue; import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.streaming.api.operators.TimestampedCollector; import org.apache.flink.streaming.api.operators.async.AsyncWaitOperator; +import org.apache.flink.streaming.runtime.streamrecord.StreamElement; -import java.util.Collection; +import java.util.List; +import java.util.Optional; /** - * Interface for blocking stream element queues for the {@link AsyncWaitOperator}. + * Interface for stream element queues for the {@link AsyncWaitOperator}. */ @Internal -public interface StreamElementQueue { +public interface StreamElementQueue { /** - * Put the given element in the queue if capacity is left. If not, then block until this is - * the case. + * Tries to put the given element in the queue. This operation succeeds if the queue has capacity left and fails if + * the queue is full. * - * @param streamElementQueueEntry to be put into the queue - * @param Type of the entries future value - * @throws InterruptedException if the calling thread has been interrupted while waiting to - * insert the given element - */ - void put(StreamElementQueueEntry streamElementQueueEntry) throws InterruptedException; - - /** - * Try to put the given element in the queue. This operation succeeds if the queue has capacity - * left and fails if the queue is full. + *

    This method returns a handle to the inserted element that allows to set the result of the computation. * - * @param streamElementQueueEntry to be inserted - * @param Type of the entries future value - * @return True if the entry could be inserted; otherwise false - * @throws InterruptedException if the calling thread has been interrupted while waiting to - * insert the given element + * @param streamElement the element to be inserted. + * @return A handle to the element if successful or {@link Optional#empty()} otherwise. */ - boolean tryPut(StreamElementQueueEntry streamElementQueueEntry) throws InterruptedException; + Optional> tryPut(StreamElement streamElement); /** - * Peek at the head of the queue and return the first completed {@link AsyncResult}. This - * operation is a blocking operation and only returns once a completed async result has been - * found. + * Emits one completed element from the head of this queue into the given output. * - * @return Completed {@link AsyncResult} - * @throws InterruptedException if the current thread has been interrupted while waiting for a - * completed async result. + *

    Will not emit any element if no element has been completed (check {@link #hasCompletedElements()} before entering + * any critical section). + * + * @param output the output into which to emit */ - AsyncResult peekBlockingly() throws InterruptedException; + void emitCompletedElement(TimestampedCollector output); /** - * Poll the first completed {@link AsyncResult} from the head of this queue. This operation is - * blocking and only returns once a completed async result has been found. + * Checks if there is at least one completed head element. * - * @return Completed {@link AsyncResult} which has been removed from the queue - * @throws InterruptedException if the current thread has been interrupted while waiting for a - * completed async result. + * @return True if there is a completed head element. */ - AsyncResult poll() throws InterruptedException; + boolean hasCompletedElements(); /** - * Return the collection of {@link StreamElementQueueEntry} currently contained in this queue. + * Returns the collection of {@link StreamElement} currently contained in this queue for checkpointing. + * + *

    This includes all non-emitted, completed and non-completed elements. * - * @return Collection of currently contained {@link StreamElementQueueEntry}. - * @throws InterruptedException if the current thread has been interrupted while retrieving the - * stream element queue entries of this queue. + * @return List of currently contained {@link StreamElement}. */ - Collection> values() throws InterruptedException; + List values(); /** * True if the queue is empty; otherwise false. diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java index c59e012c86f06d5726d8987b7746a3ab08c008f9..84fd7116bd6dd8cc83198b10c7241d93c2adc597 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueEntry.java @@ -19,79 +19,46 @@ package org.apache.flink.streaming.api.operators.async.queue; import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.streaming.api.operators.TimestampedCollector; import org.apache.flink.streaming.runtime.streamrecord.StreamElement; -import org.apache.flink.util.Preconditions; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.function.Consumer; +import javax.annotation.Nonnull; /** - * Entry class for the {@link StreamElementQueue}. The stream element queue entry stores the - * {@link StreamElement} for which the stream element queue entry has been instantiated. - * Furthermore, it allows to register callbacks for when the queue entry is completed. - * - * @param Type of the result + * An entry for the {@link StreamElementQueue}. The stream element queue entry stores the {@link StreamElement} for + * which the stream element queue entry has been instantiated. + * Furthermore, it allows to set the result of a completed entry through {@link ResultFuture}. */ @Internal -public abstract class StreamElementQueueEntry implements AsyncResult { - - private final StreamElement streamElement; - - public StreamElementQueueEntry(StreamElement streamElement) { - this.streamElement = Preconditions.checkNotNull(streamElement); - } - - public StreamElement getStreamElement() { - return streamElement; - } +interface StreamElementQueueEntry extends ResultFuture { /** * True if the stream element queue entry has been completed; otherwise false. * * @return True if the stream element queue entry has been completed; otherwise false. */ - public boolean isDone() { - return getFuture().isDone(); - } + boolean isDone(); /** - * Register the given complete function to be called once this queue entry has been completed. + * Emits the results associated with this queue entry. * - * @param completeFunction to call when the queue entry has been completed - * @param executor to run the complete function + * @param output the output into which to emit. */ - public void onComplete( - final Consumer> completeFunction, - Executor executor) { - final StreamElementQueueEntry thisReference = this; - - getFuture().whenCompleteAsync( - // call the complete function for normal completion as well as exceptional completion - // see FLINK-6435 - (value, throwable) -> completeFunction.accept(thisReference), - executor); - } + void emitResult(TimestampedCollector output); - protected abstract CompletableFuture getFuture(); - - @Override - public final boolean isWatermark() { - return AsyncWatermarkResult.class.isAssignableFrom(getClass()); - } - - @Override - public final boolean isResultCollection() { - return AsyncCollectionResult.class.isAssignableFrom(getClass()); - } - - @Override - public final AsyncWatermarkResult asWatermark() { - return (AsyncWatermarkResult) this; - } + /** + * The input element for this queue entry, for which the calculation is performed asynchronously. + * + * @return the input element. + */ + @Nonnull StreamElement getInputElement(); + /** + * Not supported. Exceptions must be handled in the AsyncWaitOperator. + */ @Override - public final AsyncCollectionResult asResultCollection() { - return (AsyncCollectionResult) this; + default void completeExceptionally(Throwable error) { + throw new UnsupportedOperationException("This result future should only be used to set completed results."); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java index 796b44f91012ab76c359787f0f95b6dfbb9054c2..bfb46d69635cc7fd0ab701ca2d2f8ee2371ddbba 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/StreamRecordQueueEntry.java @@ -21,65 +21,53 @@ package org.apache.flink.streaming.api.operators.async.queue; import org.apache.flink.annotation.Internal; import org.apache.flink.streaming.api.functions.async.AsyncFunction; import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.streaming.api.operators.TimestampedCollector; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.util.Preconditions; + +import javax.annotation.Nonnull; import java.util.Collection; -import java.util.concurrent.CompletableFuture; /** * {@link StreamElementQueueEntry} implementation for {@link StreamRecord}. This class also acts * as the {@link ResultFuture} implementation which is given to the {@link AsyncFunction}. The * async function completes this class with a collection of results. * - * @param Type of the asynchronous collection result + * @param Type of the asynchronous collection result. */ @Internal -public class StreamRecordQueueEntry extends StreamElementQueueEntry> - implements AsyncCollectionResult, ResultFuture { - - /** Timestamp information. */ - private final boolean hasTimestamp; - private final long timestamp; - - /** Future containing the collection result. */ - private final CompletableFuture> resultFuture; - - public StreamRecordQueueEntry(StreamRecord streamRecord) { - super(streamRecord); - - hasTimestamp = streamRecord.hasTimestamp(); - timestamp = streamRecord.getTimestamp(); +class StreamRecordQueueEntry implements StreamElementQueueEntry { + @Nonnull + private final StreamRecord inputRecord; - resultFuture = new CompletableFuture<>(); - } + private Collection completedElements; - @Override - public boolean hasTimestamp() { - return hasTimestamp; + StreamRecordQueueEntry(StreamRecord inputRecord) { + this.inputRecord = Preconditions.checkNotNull(inputRecord); } @Override - public long getTimestamp() { - return timestamp; + public boolean isDone() { + return completedElements != null; } + @Nonnull @Override - public Collection get() throws Exception { - return resultFuture.get(); + public StreamRecord getInputElement() { + return inputRecord; } @Override - protected CompletableFuture> getFuture() { - return resultFuture; + public void emitResult(TimestampedCollector output) { + output.setTimestamp(inputRecord); + for (OUT r : completedElements) { + output.collect(r); + } } @Override public void complete(Collection result) { - resultFuture.complete(result); - } - - @Override - public void completeExceptionally(Throwable error) { - resultFuture.completeExceptionally(error); + this.completedElements = Preconditions.checkNotNull(result); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java index 687ea7ef72d00fa5e7a93b1454ddbd2df93e1929..77a5185b07730bd7ed34726aeeef7dcbe0e6dd43 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueue.java @@ -19,287 +19,261 @@ package org.apache.flink.streaming.api.operators.async.queue; import org.apache.flink.annotation.Internal; -import org.apache.flink.streaming.api.operators.async.OperatorActions; +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.streaming.api.operators.TimestampedCollector; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamElement; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayDeque; -import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; +import java.util.Deque; import java.util.HashSet; -import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Queue; import java.util.Set; -import java.util.concurrent.Executor; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.ReentrantLock; /** - * Unordered implementation of the {@link StreamElementQueue}. The unordered stream element queue - * emits asynchronous results as soon as they are completed. Additionally it maintains the - * watermark-stream record order. This means that no stream record can be overtaken by a watermark - * and no watermark can overtake a stream record. However, stream records falling in the same - * segment between two watermarks can overtake each other (their emission order is not guaranteed). + * Unordered implementation of the {@link StreamElementQueue}. The unordered stream element queue provides + * asynchronous results as soon as they are completed. Additionally, it maintains the watermark-stream record order. + * + *

    Elements can be logically grouped into different segments separated by watermarks. A segment needs to be + * completely emitted before entries from a following segment are emitted. Thus, no stream record can be overtaken + * by a watermark and no watermark can overtake a stream record. + * However, stream records falling in the same segment between two watermarks can overtake each other (their emission + * order is not guaranteed). */ @Internal -public class UnorderedStreamElementQueue implements StreamElementQueue { +public final class UnorderedStreamElementQueue implements StreamElementQueue { private static final Logger LOG = LoggerFactory.getLogger(UnorderedStreamElementQueue.class); /** Capacity of this queue. */ private final int capacity; - /** Executor to run the onComplete callbacks. */ - private final Executor executor; - - /** OperatorActions to signal the owning operator a failure. */ - private final OperatorActions operatorActions; - - /** Queue of uncompleted stream element queue entries segmented by watermarks. */ - private final ArrayDeque>> uncompletedQueue; - - /** Queue of completed stream element queue entries. */ - private final ArrayDeque> completedQueue; - - /** First (chronologically oldest) uncompleted set of stream element queue entries. */ - private Set> firstSet; + /** Queue of queue entries segmented by watermarks. */ + private final Deque> segments; - // Last (chronologically youngest) uncompleted set of stream element queue entries. New - // stream element queue entries are inserted into this set. - private Set> lastSet; - private volatile int numberEntries; - - /** Locks and conditions for the blocking queue. */ - private final ReentrantLock lock; - private final Condition notFull; - private final Condition hasCompletedEntries; - - public UnorderedStreamElementQueue( - int capacity, - Executor executor, - OperatorActions operatorActions) { + private int numberOfEntries; + public UnorderedStreamElementQueue(int capacity) { Preconditions.checkArgument(capacity > 0, "The capacity must be larger than 0."); - this.capacity = capacity; - - this.executor = Preconditions.checkNotNull(executor, "executor"); - this.operatorActions = Preconditions.checkNotNull(operatorActions, "operatorActions"); - - this.uncompletedQueue = new ArrayDeque<>(capacity); - this.completedQueue = new ArrayDeque<>(capacity); - - this.firstSet = new HashSet<>(capacity); - this.lastSet = firstSet; - - this.numberEntries = 0; - - this.lock = new ReentrantLock(); - this.notFull = lock.newCondition(); - this.hasCompletedEntries = lock.newCondition(); + this.capacity = capacity; + // most likely scenario are 4 segments + this.segments = new ArrayDeque<>(4); + this.numberOfEntries = 0; } @Override - public void put(StreamElementQueueEntry streamElementQueueEntry) throws InterruptedException { - lock.lockInterruptibly(); - - try { - while (numberEntries >= capacity) { - notFull.await(); + public Optional> tryPut(StreamElement streamElement) { + if (size() < capacity) { + StreamElementQueueEntry queueEntry; + if (streamElement.isRecord()) { + queueEntry = addRecord((StreamRecord) streamElement); + } else if (streamElement.isWatermark()) { + queueEntry = addWatermark((Watermark) streamElement); + } else { + throw new UnsupportedOperationException("Cannot enqueue " + streamElement); } - addEntry(streamElementQueueEntry); - } finally { - lock.unlock(); - } - } - - @Override - public boolean tryPut(StreamElementQueueEntry streamElementQueueEntry) throws InterruptedException { - lock.lockInterruptibly(); - - try { - if (numberEntries < capacity) { - addEntry(streamElementQueueEntry); + numberOfEntries++; - LOG.debug("Put element into unordered stream element queue. New filling degree " + - "({}/{}).", numberEntries, capacity); + LOG.debug("Put element into unordered stream element queue. New filling degree " + + "({}/{}).", size(), capacity); - return true; - } else { - LOG.debug("Failed to put element into unordered stream element queue because it " + - "was full ({}/{}).", numberEntries, capacity); + return Optional.of(queueEntry); + } else { + LOG.debug("Failed to put element into unordered stream element queue because it " + + "was full ({}/{}).", size(), capacity); - return false; - } - } finally { - lock.unlock(); + return Optional.empty(); } } - @Override - public AsyncResult peekBlockingly() throws InterruptedException { - lock.lockInterruptibly(); + private StreamElementQueueEntry addRecord(StreamRecord record) { + // ensure that there is at least one segment + Segment lastSegment; + if (segments.isEmpty()) { + lastSegment = addSegment(capacity); + } else { + lastSegment = segments.getLast(); + } - try { - while (completedQueue.isEmpty()) { - hasCompletedEntries.await(); - } + // entry is bound to segment to notify it easily upon completion + StreamElementQueueEntry queueEntry = new SegmentedStreamRecordQueueEntry<>(record, lastSegment); + lastSegment.add(queueEntry); + return queueEntry; + } - LOG.debug("Peeked head element from unordered stream element queue with filling degree " + - "({}/{}).", numberEntries, capacity); + private Segment addSegment(int capacity) { + Segment newSegment = new Segment(capacity); + segments.addLast(newSegment); + return newSegment; + } - return completedQueue.peek(); - } finally { - lock.unlock(); + private StreamElementQueueEntry addWatermark(Watermark watermark) { + Segment watermarkSegment; + if (!segments.isEmpty() && segments.getLast().isEmpty()) { + // reuse already existing segment if possible (completely drained) or the new segment added at the end of + // this method for two succeeding watermarks + watermarkSegment = segments.getLast(); + } else { + watermarkSegment = addSegment(1); } - } - @Override - public AsyncResult poll() throws InterruptedException { - lock.lockInterruptibly(); + StreamElementQueueEntry watermarkEntry = new WatermarkQueueEntry<>(watermark); + watermarkSegment.add(watermarkEntry); - try { - while (completedQueue.isEmpty()) { - hasCompletedEntries.await(); - } + // add a new segment for actual elements + addSegment(capacity); + return watermarkEntry; + } - numberEntries--; - notFull.signalAll(); + @Override + public boolean hasCompletedElements() { + return !this.segments.isEmpty() && this.segments.getFirst().hasCompleted(); + } - LOG.debug("Polled element from unordered stream element queue. New filling degree " + - "({}/{}).", numberEntries, capacity); + @Override + public void emitCompletedElement(TimestampedCollector output) { + if (segments.isEmpty()) { + return; + } + final Segment currentSegment = segments.getFirst(); + numberOfEntries -= currentSegment.emitCompleted(output); - return completedQueue.poll(); - } finally { - lock.unlock(); + // remove any segment if there are further segments, if not leave it as an optimization even if empty + if (segments.size() > 1 && currentSegment.isEmpty()) { + segments.pop(); } } @Override - public Collection> values() throws InterruptedException { - lock.lockInterruptibly(); - - try { - StreamElementQueueEntry[] array = new StreamElementQueueEntry[numberEntries]; - - array = completedQueue.toArray(array); - - int counter = completedQueue.size(); - - for (StreamElementQueueEntry entry: firstSet) { - array[counter] = entry; - counter++; - } - - for (Set> asyncBufferEntries : uncompletedQueue) { - - for (StreamElementQueueEntry streamElementQueueEntry : asyncBufferEntries) { - array[counter] = streamElementQueueEntry; - counter++; - } - } - - return Arrays.asList(array); - } finally { - lock.unlock(); + public List values() { + List list = new ArrayList<>(); + for (Segment s : segments) { + s.addPendingElements(list); } + return list; } @Override public boolean isEmpty() { - return numberEntries == 0; + return numberOfEntries == 0; } @Override public int size() { - return numberEntries; + return numberOfEntries; } /** - * Callback for onComplete events for the given stream element queue entry. Whenever a queue - * entry is completed, it is checked whether this entry belongs to the first set. If this is the - * case, then the element is added to the completed entries queue from where it can be consumed. - * If the first set becomes empty, then the next set is polled from the uncompleted entries - * queue. Completed entries from this new set are then added to the completed entries queue. - * - * @param streamElementQueueEntry which has been completed - * @throws InterruptedException if the current thread has been interrupted while performing the - * on complete callback. + * An entry that notifies the respective segment upon completion. */ - public void onCompleteHandler(StreamElementQueueEntry streamElementQueueEntry) throws InterruptedException { - lock.lockInterruptibly(); + static class SegmentedStreamRecordQueueEntry extends StreamRecordQueueEntry { + private final Segment segment; - try { - if (firstSet.remove(streamElementQueueEntry)) { - completedQueue.offer(streamElementQueueEntry); + SegmentedStreamRecordQueueEntry(StreamRecord inputRecord, Segment segment) { + super(inputRecord); + this.segment = segment; + } - while (firstSet.isEmpty() && firstSet != lastSet) { - firstSet = uncompletedQueue.poll(); + @Override + public void complete(Collection result) { + super.complete(result); + segment.completed(this); + } + } - Iterator> it = firstSet.iterator(); + /** + * A segment is a collection of queue entries that can be completed in arbitrary order. + * + *

    All elements from one segment must be emitted before any element of the next segment is emitted. + */ + static class Segment { + /** Unfinished input elements. */ + private final Set> incompleteElements; - while (it.hasNext()) { - StreamElementQueueEntry bufferEntry = it.next(); + /** Undrained finished elements. */ + private final Queue> completedElements; - if (bufferEntry.isDone()) { - completedQueue.offer(bufferEntry); - it.remove(); - } - } - } + Segment(int initialCapacity) { + incompleteElements = new HashSet<>(initialCapacity); + completedElements = new ArrayDeque<>(initialCapacity); + } - LOG.debug("Signal unordered stream element queue has completed entries."); - hasCompletedEntries.signalAll(); + /** + * Signals that an entry finished computation. + */ + void completed(StreamElementQueueEntry elementQueueEntry) { + // adding only to completed queue if not completed before + // there may be a real result coming after a timeout result, which is updated in the queue entry but + // the entry is not re-added to the complete queue + if (incompleteElements.remove(elementQueueEntry)) { + completedElements.add(elementQueueEntry); } - } finally { - lock.unlock(); } - } - /** - * Add the given stream element queue entry to the current last set if it is not a watermark. - * If it is a watermark, then stop adding to the current last set, insert the watermark into its - * own set and add a new last set. - * - * @param streamElementQueueEntry to be inserted - * @param Type of the stream element queue entry's result - */ - private void addEntry(StreamElementQueueEntry streamElementQueueEntry) { - assert(lock.isHeldByCurrentThread()); + /** + * True if there are no incomplete elements and all complete elements have been consumed. + */ + boolean isEmpty() { + return incompleteElements.isEmpty() && completedElements.isEmpty(); + } - if (streamElementQueueEntry.isWatermark()) { - lastSet = new HashSet<>(capacity); + /** + * True if there is at least one completed elements, such that {@link #emitCompleted(TimestampedCollector)} + * will actually output an element. + */ + boolean hasCompleted() { + return !completedElements.isEmpty(); + } - if (firstSet.isEmpty()) { - firstSet.add(streamElementQueueEntry); - } else { - Set> watermarkSet = new HashSet<>(1); - watermarkSet.add(streamElementQueueEntry); - uncompletedQueue.offer(watermarkSet); + /** + * Adds the segmentd input elements for checkpointing including completed but not yet emitted elements. + */ + void addPendingElements(List results) { + for (StreamElementQueueEntry element : completedElements) { + results.add(element.getInputElement()); } - uncompletedQueue.offer(lastSet); - } else { - lastSet.add(streamElementQueueEntry); + for (StreamElementQueueEntry element : incompleteElements) { + results.add(element.getInputElement()); + } + } + + /** + * Pops one completed elements into the given output. Because an input element may produce an arbitrary + * number of output elements, there is no correlation between the size of the collection and the popped + * elements. + * + * @return the number of popped input elements. + */ + int emitCompleted(TimestampedCollector output) { + final StreamElementQueueEntry completedEntry = completedElements.poll(); + if (completedEntry == null) { + return 0; + } + completedEntry.emitResult(output); + return 1; } - streamElementQueueEntry.onComplete( - (StreamElementQueueEntry value) -> { - try { - onCompleteHandler(value); - } catch (InterruptedException e) { - // The accept executor thread got interrupted. This is probably cause by - // the shutdown of the executor. - LOG.debug("AsyncBufferEntry could not be properly completed because the " + - "executor thread has been interrupted.", e); - } catch (Throwable t) { - operatorActions.failOperator(new Exception("Could not complete the " + - "stream element queue entry: " + value + '.', t)); - } - }, - executor); - - numberEntries++; + /** + * Adds the given entry to this segment. If the element is completed (watermark), it is directly moved into the + * completed queue. + */ + void add(StreamElementQueueEntry queueEntry) { + if (queueEntry.isDone()) { + completedElements.add(queueEntry); + } else { + incompleteElements.add(queueEntry); + } + } } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/WatermarkQueueEntry.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/WatermarkQueueEntry.java index c9c6e7480c7b80b8163e75a005a55248e086623b..153f462027258b36eff7d00a717a18ee14549a0b 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/WatermarkQueueEntry.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/WatermarkQueueEntry.java @@ -19,31 +19,44 @@ package org.apache.flink.streaming.api.operators.async.queue; import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.operators.TimestampedCollector; import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.util.Preconditions; -import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +import java.util.Collection; /** * {@link StreamElementQueueEntry} implementation for the {@link Watermark}. */ @Internal -public class WatermarkQueueEntry extends StreamElementQueueEntry implements AsyncWatermarkResult { +class WatermarkQueueEntry implements StreamElementQueueEntry { + @Nonnull + private final Watermark watermark; - private final CompletableFuture future; + WatermarkQueueEntry(Watermark watermark) { + this.watermark = Preconditions.checkNotNull(watermark); + } - public WatermarkQueueEntry(Watermark watermark) { - super(watermark); + @Override + public void emitResult(TimestampedCollector output) { + output.emitWatermark(watermark); + } - this.future = CompletableFuture.completedFuture(watermark); + @Nonnull + @Override + public Watermark getInputElement() { + return watermark; } @Override - public Watermark getWatermark() { - return (Watermark) getStreamElement(); + public boolean isDone() { + return true; } @Override - protected CompletableFuture getFuture() { - return future; + public void complete(Collection result) { + throw new IllegalStateException("Cannot complete a watermark."); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/AbstractDataOutput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/AbstractDataOutput.java new file mode 100644 index 0000000000000000000000000000000000000000..7ef94e8cad9101248b07c04469d4a991955aa34d --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/AbstractDataOutput.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.runtime.io; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.runtime.streamstatus.StreamStatus; +import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Base class for all data outputs. It implements the unified way of emitting stream status + * for both network and source outputs. + * + * @param The output type + */ +@Internal +public abstract class AbstractDataOutput implements PushingAsyncDataInput.DataOutput { + + /** The maintainer toggles the current stream status. */ + protected final StreamStatusMaintainer streamStatusMaintainer; + + protected final Object lock; + + public AbstractDataOutput(StreamStatusMaintainer streamStatusMaintainer, Object lock) { + this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer); + this.lock = checkNotNull(lock); + } + + @Override + public void emitStreamStatus(StreamStatus streamStatus) { + synchronized (lock) { + streamStatusMaintainer.toggleStreamStatus(streamStatus); + } + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointedInputGate.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointedInputGate.java index 8e82a005a56a8aec47fae7bbb9df4da3a5df5266..63dd16b26753a56ecb8d91aa2c4a9b5b915af459 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointedInputGate.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/CheckpointedInputGate.java @@ -18,7 +18,7 @@ package org.apache.flink.streaming.runtime.io; import org.apache.flink.annotation.Internal; -import org.apache.flink.runtime.io.AsyncDataInput; +import org.apache.flink.runtime.io.PullingAsyncDataInput; import org.apache.flink.runtime.io.network.api.CancelCheckpointMarker; import org.apache.flink.runtime.io.network.api.CheckpointBarrier; import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent; @@ -42,7 +42,7 @@ import static org.apache.flink.util.Preconditions.checkNotNull; * {@link CheckpointBarrier} from the {@link InputGate}. */ @Internal -public class CheckpointedInputGate implements AsyncDataInput { +public class CheckpointedInputGate implements PullingAsyncDataInput { private static final Logger LOG = LoggerFactory.getLogger(CheckpointedInputGate.class); diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/InputStatus.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/InputStatus.java new file mode 100644 index 0000000000000000000000000000000000000000..a2d8d29a4114949aa27e26f65ce3b1162024071a --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/InputStatus.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.runtime.io; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput; + +/** + * An {@link InputStatus} indicates one input state which might be currently + * available, not available or already finished. It is returned while calling + * {@link PushingAsyncDataInput#emitNext(DataOutput)}. + */ +@Internal +public enum InputStatus { + + /** + * Indicator that more data is available and the input can be called immediately again + * to emit more data. + */ + MORE_AVAILABLE, + + /** + * Indicator that no data is currently available, but more data will be available in the + * future again. + */ + NOTHING_AVAILABLE, + + /** + * Indicator that the input has reached the end of data. + */ + END_OF_INPUT +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/PushingAsyncDataInput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/PushingAsyncDataInput.java new file mode 100644 index 0000000000000000000000000000000000000000..7fcd047e0aedc17e74e16325ff739ec166816b47 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/PushingAsyncDataInput.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.runtime.io; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.runtime.io.AvailabilityProvider; +import org.apache.flink.runtime.io.PullingAsyncDataInput; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.streamstatus.StreamStatus; + +/** + * The variant of {@link PullingAsyncDataInput} that is defined for handling both network + * input and source input in a unified way via {@link #emitNext(DataOutput)} instead + * of returning {@code Optional.empty()} via {@link PullingAsyncDataInput#pollNext()}. + */ +@Internal +public interface PushingAsyncDataInput extends AvailabilityProvider { + + /** + * Pushes the next element to the output from current data input, and returns + * the input status to indicate whether there are more available data in + * current input. + * + *

    This method should be non blocking. + */ + InputStatus emitNext(DataOutput output) throws Exception; + + /** + * Basic data output interface used in emitting the next element from data input. + * + * @param The type encapsulated with the stream record. + */ + interface DataOutput { + + void emitRecord(StreamRecord streamRecord) throws Exception; + + void emitWatermark(Watermark watermark) throws Exception; + + void emitStreamStatus(StreamStatus streamStatus) throws Exception; + + void emitLatencyMarker(LatencyMarker latencyMarker) throws Exception; + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java index 1de31bf6df4ec885d460523a9114e5bf1f2728b1..3c070e28eff5ee42a2c52bcb82796bbd50dcda10 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamInputProcessor.java @@ -19,7 +19,7 @@ package org.apache.flink.streaming.runtime.io; import org.apache.flink.annotation.Internal; -import org.apache.flink.runtime.io.AvailabilityListener; +import org.apache.flink.runtime.io.AvailabilityProvider; import java.io.Closeable; @@ -27,11 +27,11 @@ import java.io.Closeable; * Interface for processing records by {@link org.apache.flink.streaming.runtime.tasks.StreamTask}. */ @Internal -public interface StreamInputProcessor extends AvailabilityListener, Closeable { +public interface StreamInputProcessor extends AvailabilityProvider, Closeable { /** - * @return true if {@link StreamInputProcessor} estimates that more records can be processed - * immediately. Otherwise false, which means that there are no more records available at the - * moment and the caller should check {@link #isFinished()} and/or {@link #isAvailable()}. + * @return input status to estimate whether more records can be processed immediately or not. + * If there are no more records available at the moment and the caller should check finished + * state and/or {@link #isAvailable()}. */ - boolean processInput() throws Exception; + InputStatus processInput() throws Exception; } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamOneInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamOneInputProcessor.java index 28aea1910f7caa31115631c7b925d260307940ca..ddeaf7be04abb9ae16570a0b15e7e79396d61bad 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamOneInputProcessor.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamOneInputProcessor.java @@ -19,23 +19,8 @@ package org.apache.flink.streaming.runtime.io; import org.apache.flink.annotation.Internal; -import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.metrics.Counter; -import org.apache.flink.runtime.io.disk.iomanager.IOManager; -import org.apache.flink.runtime.io.network.partition.consumer.InputGate; -import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup; -import org.apache.flink.streaming.api.CheckpointingMode; -import org.apache.flink.streaming.api.operators.OneInputStreamOperator; -import org.apache.flink.streaming.api.watermark.Watermark; -import org.apache.flink.streaming.runtime.metrics.WatermarkGauge; -import org.apache.flink.streaming.runtime.streamrecord.StreamElement; -import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.runtime.streamstatus.StatusWatermarkValve; -import org.apache.flink.streaming.runtime.streamstatus.StreamStatus; -import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput; import org.apache.flink.streaming.runtime.tasks.OperatorChain; -import org.apache.flink.streaming.runtime.tasks.StreamTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,20 +29,10 @@ import java.io.IOException; import java.util.concurrent.CompletableFuture; import static org.apache.flink.util.Preconditions.checkNotNull; -import static org.apache.flink.util.Preconditions.checkState; /** * Input reader for {@link org.apache.flink.streaming.runtime.tasks.OneInputStreamTask}. * - *

    This internally uses a {@link StatusWatermarkValve} to keep track of {@link Watermark} and - * {@link StreamStatus} events, and forwards them to event subscribers once the - * {@link StatusWatermarkValve} determines the {@link Watermark} from all inputs has advanced, or - * that a {@link StreamStatus} needs to be propagated downstream to denote a status change. - * - *

    Forwarding elements, watermarks, or status status elements must be protected by synchronizing - * on the given lock object. This ensures that we don't call methods on a - * {@link OneInputStreamOperator} concurrently with the timer callback or other things. - * * @param The type of the record that can be read with this record reader. */ @Internal @@ -65,73 +40,23 @@ public final class StreamOneInputProcessor implements StreamInputProcessor { private static final Logger LOG = LoggerFactory.getLogger(StreamOneInputProcessor.class); - private final StreamTaskInput input; + private final StreamTaskInput input; + private final DataOutput output; private final Object lock; private final OperatorChain operatorChain; - // ---------------- Status and Watermark Valve ------------------ - - /** Valve that controls how watermarks and stream statuses are forwarded. */ - private StatusWatermarkValve statusWatermarkValve; - - private final StreamStatusMaintainer streamStatusMaintainer; - - private final OneInputStreamOperator streamOperator; - - // ---------------- Metrics ------------------ - - private final WatermarkGauge watermarkGauge; - private final Counter numRecordsIn; - - @SuppressWarnings("unchecked") public StreamOneInputProcessor( - InputGate[] inputGates, - TypeSerializer inputSerializer, - StreamTask checkpointedTask, - CheckpointingMode checkpointMode, + StreamTaskInput input, + DataOutput output, Object lock, - IOManager ioManager, - Configuration taskManagerConfig, - StreamStatusMaintainer streamStatusMaintainer, - OneInputStreamOperator streamOperator, - TaskIOMetricGroup metrics, - WatermarkGauge watermarkGauge, - String taskName, - OperatorChain operatorChain, - Counter numRecordsIn) throws IOException { - - InputGate inputGate = InputGateUtil.createInputGate(inputGates); - - CheckpointedInputGate barrierHandler = InputProcessorUtil.createCheckpointedInputGate( - checkpointedTask, - checkpointMode, - ioManager, - inputGate, - taskManagerConfig, - taskName); - this.input = new StreamTaskNetworkInput(barrierHandler, inputSerializer, ioManager, 0); + OperatorChain operatorChain) { + this.input = checkNotNull(input); + this.output = checkNotNull(output); this.lock = checkNotNull(lock); - - this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer); - this.streamOperator = checkNotNull(streamOperator); - - this.statusWatermarkValve = new StatusWatermarkValve( - inputGate.getNumberOfInputChannels(), - new ForwardingValveOutputHandler(streamOperator, lock)); - - this.watermarkGauge = watermarkGauge; - metrics.gauge("checkpointAlignmentTime", barrierHandler::getAlignmentDurationNanos); - this.operatorChain = checkNotNull(operatorChain); - this.numRecordsIn = checkNotNull(numRecordsIn); - } - - @Override - public boolean isFinished() { - return input.isFinished(); } @Override @@ -140,81 +65,20 @@ public final class StreamOneInputProcessor implements StreamInputProcessor { } @Override - public boolean processInput() throws Exception { - StreamElement recordOrMark = input.pollNextNullable(); - if (recordOrMark != null) { - int channel = input.getLastChannel(); - checkState(channel != StreamTaskInput.UNSPECIFIED); - - processElement(recordOrMark, channel); - } - checkFinished(); - - return recordOrMark != null; - } - - private void processElement(StreamElement recordOrMark, int channel) throws Exception { - if (recordOrMark.isRecord()) { - // now we can do the actual processing - StreamRecord record = recordOrMark.asRecord(); - synchronized (lock) { - numRecordsIn.inc(); - streamOperator.setKeyContextElement1(record); - streamOperator.processElement(record); - } - } - else if (recordOrMark.isWatermark()) { - // handle watermark - statusWatermarkValve.inputWatermark(recordOrMark.asWatermark(), channel); - } else if (recordOrMark.isStreamStatus()) { - // handle stream status - statusWatermarkValve.inputStreamStatus(recordOrMark.asStreamStatus(), channel); - } else if (recordOrMark.isLatencyMarker()) { - // handle latency marker - synchronized (lock) { - streamOperator.processLatencyMarker(recordOrMark.asLatencyMarker()); - } - } else { - throw new UnsupportedOperationException("Unknown type of StreamElement"); - } - } + public InputStatus processInput() throws Exception { + InputStatus status = input.emitNext(output); - private void checkFinished() throws Exception { - if (input.isFinished()) { + if (status == InputStatus.END_OF_INPUT) { synchronized (lock) { operatorChain.endInput(1); } } + + return status; } @Override public void close() throws IOException { input.close(); } - - private class ForwardingValveOutputHandler implements StatusWatermarkValve.ValveOutputHandler { - private final OneInputStreamOperator operator; - private final Object lock; - - private ForwardingValveOutputHandler(final OneInputStreamOperator operator, final Object lock) { - this.operator = checkNotNull(operator); - this.lock = checkNotNull(lock); - } - - @Override - public void handleWatermark(Watermark watermark) throws Exception { - synchronized (lock) { - watermarkGauge.setCurrentWatermark(watermark.getTimestamp()); - operator.processWatermark(watermark); - } - } - - @SuppressWarnings("unchecked") - @Override - public void handleStreamStatus(StreamStatus streamStatus) { - synchronized (lock) { - streamStatusMaintainer.toggleStreamStatus(streamStatus); - } - } - } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskInput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskInput.java index 19fd76513f818b49bf5b27156e9b881e83ebb72d..484d9cd8d883c10cd2b1c722afec41b5d2097b0e 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskInput.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskInput.java @@ -18,8 +18,6 @@ package org.apache.flink.streaming.runtime.io; import org.apache.flink.annotation.Internal; -import org.apache.flink.runtime.io.NullableAsyncDataInput; -import org.apache.flink.streaming.runtime.streamrecord.StreamElement; import java.io.Closeable; @@ -27,15 +25,9 @@ import java.io.Closeable; * Basic interface for inputs of stream operators. */ @Internal -public interface StreamTaskInput extends NullableAsyncDataInput, Closeable { +public interface StreamTaskInput extends PushingAsyncDataInput, Closeable { int UNSPECIFIED = -1; - /** - * @return channel index of last returned {@link StreamElement}, or {@link #UNSPECIFIED} if - * it is unspecified. - */ - int getLastChannel(); - /** * Returns the input index of this input. */ diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInput.java index 0bd6e624aa69cb5fb95516869b9b4a5ef5cc1dde..ecb26ac061c25aa9a8f9d47b1e2b212396317a72 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInput.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInput.java @@ -31,22 +31,33 @@ import org.apache.flink.runtime.io.network.buffer.Buffer; import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent; import org.apache.flink.runtime.plugable.DeserializationDelegate; import org.apache.flink.runtime.plugable.NonReusingDeserializationDelegate; +import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamElement; import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer; - -import javax.annotation.Nullable; +import org.apache.flink.streaming.runtime.streamstatus.StatusWatermarkValve; +import org.apache.flink.streaming.runtime.streamstatus.StreamStatus; import java.io.IOException; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import static org.apache.flink.util.Preconditions.checkNotNull; import static org.apache.flink.util.Preconditions.checkState; /** * Implementation of {@link StreamTaskInput} that wraps an input from network taken from {@link CheckpointedInputGate}. + * + *

    This internally uses a {@link StatusWatermarkValve} to keep track of {@link Watermark} and + * {@link StreamStatus} events, and forwards them to event subscribers once the + * {@link StatusWatermarkValve} determines the {@link Watermark} from all inputs has advanced, or + * that a {@link StreamStatus} needs to be propagated downstream to denote a status change. + * + *

    Forwarding elements, watermarks, or status status elements must be protected by synchronizing + * on the given lock object. This ensures that we don't call methods on a + * {@link StreamInputProcessor} concurrently with the timer callback or other things. */ @Internal -public final class StreamTaskNetworkInput implements StreamTaskInput { +public final class StreamTaskNetworkInput implements StreamTaskInput { private final CheckpointedInputGate checkpointedInputGate; @@ -54,19 +65,21 @@ public final class StreamTaskNetworkInput implements StreamTaskInput { private final RecordDeserializer>[] recordDeserializers; + /** Valve that controls how watermarks and stream statuses are forwarded. */ + private final StatusWatermarkValve statusWatermarkValve; + private final int inputIndex; private int lastChannel = UNSPECIFIED; private RecordDeserializer> currentRecordDeserializer = null; - private boolean isFinished = false; - @SuppressWarnings("unchecked") public StreamTaskNetworkInput( CheckpointedInputGate checkpointedInputGate, TypeSerializer inputSerializer, IOManager ioManager, + StatusWatermarkValve statusWatermarkValve, int inputIndex) { this.checkpointedInputGate = checkpointedInputGate; this.deserializationDelegate = new NonReusingDeserializationDelegate<>( @@ -79,6 +92,7 @@ public final class StreamTaskNetworkInput implements StreamTaskInput { ioManager.getSpillingDirectoriesPaths()); } + this.statusWatermarkValve = checkNotNull(statusWatermarkValve); this.inputIndex = inputIndex; } @@ -86,7 +100,7 @@ public final class StreamTaskNetworkInput implements StreamTaskInput { StreamTaskNetworkInput( CheckpointedInputGate checkpointedInputGate, TypeSerializer inputSerializer, - IOManager ioManager, + StatusWatermarkValve statusWatermarkValve, int inputIndex, RecordDeserializer>[] recordDeserializers) { @@ -94,12 +108,12 @@ public final class StreamTaskNetworkInput implements StreamTaskInput { this.deserializationDelegate = new NonReusingDeserializationDelegate<>( new StreamElementSerializer<>(inputSerializer)); this.recordDeserializers = recordDeserializers; + this.statusWatermarkValve = statusWatermarkValve; this.inputIndex = inputIndex; } @Override - @Nullable - public StreamElement pollNextNullable() throws Exception { + public InputStatus emitNext(DataOutput output) throws Exception { while (true) { // get the stream element from the deserializer @@ -111,7 +125,8 @@ public final class StreamTaskNetworkInput implements StreamTaskInput { } if (result.isFullRecord()) { - return deserializationDelegate.getInstance(); + processElement(deserializationDelegate.getInstance(), output); + return InputStatus.MORE_AVAILABLE; } } @@ -120,20 +135,35 @@ public final class StreamTaskNetworkInput implements StreamTaskInput { processBufferOrEvent(bufferOrEvent.get()); } else { if (checkpointedInputGate.isFinished()) { - isFinished = true; checkState(checkpointedInputGate.isAvailable().isDone(), "Finished BarrierHandler should be available"); if (!checkpointedInputGate.isEmpty()) { throw new IllegalStateException("Trailing data in checkpoint barrier handler."); } + return InputStatus.END_OF_INPUT; } - return null; + return InputStatus.NOTHING_AVAILABLE; } } } + private void processElement(StreamElement recordOrMark, DataOutput output) throws Exception { + if (recordOrMark.isRecord()){ + output.emitRecord(recordOrMark.asRecord()); + } else if (recordOrMark.isWatermark()) { + statusWatermarkValve.inputWatermark(recordOrMark.asWatermark(), lastChannel); + } else if (recordOrMark.isLatencyMarker()) { + output.emitLatencyMarker(recordOrMark.asLatencyMarker()); + } else if (recordOrMark.isStreamStatus()) { + statusWatermarkValve.inputStreamStatus(recordOrMark.asStreamStatus(), lastChannel); + } else { + throw new UnsupportedOperationException("Unknown type of StreamElement"); + } + } + private void processBufferOrEvent(BufferOrEvent bufferOrEvent) throws IOException { if (bufferOrEvent.isBuffer()) { lastChannel = bufferOrEvent.getChannelIndex(); + checkState(lastChannel != StreamTaskInput.UNSPECIFIED); currentRecordDeserializer = recordDeserializers[lastChannel]; checkState(currentRecordDeserializer != null, "currentRecordDeserializer has already been released"); @@ -154,21 +184,11 @@ public final class StreamTaskNetworkInput implements StreamTaskInput { } } - @Override - public int getLastChannel() { - return lastChannel; - } - @Override public int getInputIndex() { return inputIndex; } - @Override - public boolean isFinished() { - return isFinished; - } - @Override public CompletableFuture isAvailable() { if (currentRecordDeserializer != null) { diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskSourceInput.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskSourceInput.java new file mode 100644 index 0000000000000000000000000000000000000000..46878248e9c127b4b76c12cb40a2ce946108bb26 --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTaskSourceInput.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.runtime.io; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.streaming.api.operators.SourceReaderOperator; +import org.apache.flink.util.IOUtils; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * Implementation of {@link StreamTaskInput} that reads data from the {@link SourceReaderOperator} + * and returns the {@link InputStatus} to indicate whether the source state is available, + * unavailable or finished. + */ +@Internal +public final class StreamTaskSourceInput implements StreamTaskInput { + + private final SourceReaderOperator operator; + + public StreamTaskSourceInput(SourceReaderOperator operator) { + this.operator = checkNotNull(operator); + } + + @Override + public InputStatus emitNext(DataOutput output) throws Exception { + return operator.emitNext(output); + } + + @Override + public CompletableFuture isAvailable() { + return operator.isAvailable(); + } + + /** + * This method is invalid and never called by the one/source input processor. + */ + @Override + public int getInputIndex() { + return -1; + } + + @Override + public void close() { + IOUtils.closeQuietly(operator::close); + } +} + diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java index df5d186a098e512df80528441c3db8faa3579eeb..a18defe8eda4e6a3afd01695a77bfee39a905390 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/io/StreamTwoInputProcessor.java @@ -20,30 +20,26 @@ package org.apache.flink.streaming.runtime.io; import org.apache.flink.annotation.Internal; import org.apache.flink.api.common.typeutils.TypeSerializer; -import org.apache.flink.configuration.Configuration; import org.apache.flink.metrics.Counter; import org.apache.flink.runtime.io.disk.iomanager.IOManager; -import org.apache.flink.runtime.io.network.partition.consumer.InputGate; -import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput; import org.apache.flink.streaming.runtime.metrics.WatermarkGauge; -import org.apache.flink.streaming.runtime.streamrecord.StreamElement; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.streamstatus.StatusWatermarkValve; import org.apache.flink.streaming.runtime.streamstatus.StreamStatus; import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; import org.apache.flink.streaming.runtime.tasks.OperatorChain; -import org.apache.flink.streaming.runtime.tasks.StreamTask; import org.apache.flink.streaming.runtime.tasks.TwoInputStreamTask; import org.apache.flink.util.ExceptionUtils; +import org.apache.flink.util.function.ThrowingConsumer; import java.io.IOException; -import java.util.Collection; import java.util.concurrent.CompletableFuture; import static org.apache.flink.util.Preconditions.checkNotNull; -import static org.apache.flink.util.Preconditions.checkState; /** * Input reader for {@link TwoInputStreamTask}. @@ -54,97 +50,105 @@ import static org.apache.flink.util.Preconditions.checkState; @Internal public final class StreamTwoInputProcessor implements StreamInputProcessor { - private final TwoInputStreamOperator streamOperator; - private final TwoInputSelectionHandler inputSelectionHandler; private final Object lock; - private final StreamTaskInput input1; - private final StreamTaskInput input2; + private final StreamTaskInput input1; + private final StreamTaskInput input2; private final OperatorChain operatorChain; - /** - * Valves that control how watermarks and stream statuses from the 2 inputs are forwarded. - */ - private final StatusWatermarkValve statusWatermarkValve1; - private final StatusWatermarkValve statusWatermarkValve2; + private final DataOutput output1; + private final DataOutput output2; + + /** Input status to keep track for determining whether the input is finished or not. */ + private InputStatus firstInputStatus = InputStatus.MORE_AVAILABLE; + private InputStatus secondInputStatus = InputStatus.MORE_AVAILABLE; /** * Stream status for the two inputs. We need to keep track for determining when * to forward stream status changes downstream. */ - private StreamStatus firstStatus; - private StreamStatus secondStatus; - - private int lastReadInputIndex; + private StreamStatus firstStatus = StreamStatus.ACTIVE; + private StreamStatus secondStatus = StreamStatus.ACTIVE; - private final Counter numRecordsIn; + /** Always try to read from the first input. */ + private int lastReadInputIndex = 1; private boolean isPrepared; public StreamTwoInputProcessor( - Collection inputGates1, - Collection inputGates2, - TypeSerializer inputSerializer1, - TypeSerializer inputSerializer2, - StreamTask streamTask, - CheckpointingMode checkpointingMode, - Object lock, - IOManager ioManager, - Configuration taskManagerConfig, - StreamStatusMaintainer streamStatusMaintainer, - TwoInputStreamOperator streamOperator, - TwoInputSelectionHandler inputSelectionHandler, - WatermarkGauge input1WatermarkGauge, - WatermarkGauge input2WatermarkGauge, - String taskName, - OperatorChain operatorChain, - Counter numRecordsIn) throws IOException { - - this.streamOperator = checkNotNull(streamOperator); - this.inputSelectionHandler = checkNotNull(inputSelectionHandler); + CheckpointedInputGate[] checkpointedInputGates, + TypeSerializer inputSerializer1, + TypeSerializer inputSerializer2, + Object lock, + IOManager ioManager, + StreamStatusMaintainer streamStatusMaintainer, + TwoInputStreamOperator streamOperator, + TwoInputSelectionHandler inputSelectionHandler, + WatermarkGauge input1WatermarkGauge, + WatermarkGauge input2WatermarkGauge, + OperatorChain operatorChain, + Counter numRecordsIn) { this.lock = checkNotNull(lock); + this.inputSelectionHandler = checkNotNull(inputSelectionHandler); - InputGate unionedInputGate1 = InputGateUtil.createInputGate(inputGates1.toArray(new InputGate[0])); - InputGate unionedInputGate2 = InputGateUtil.createInputGate(inputGates2.toArray(new InputGate[0])); - - // create a Input instance for each input - CheckpointedInputGate[] checkpointedInputGates = InputProcessorUtil.createCheckpointedInputGatePair( - streamTask, - checkpointingMode, + this.output1 = new StreamTaskNetworkOutput<>( + streamOperator, + record -> processRecord1(record, streamOperator, numRecordsIn), + lock, + streamStatusMaintainer, + input1WatermarkGauge, + 0); + this.output2 = new StreamTaskNetworkOutput<>( + streamOperator, + record -> processRecord2(record, streamOperator, numRecordsIn), + lock, + streamStatusMaintainer, + input2WatermarkGauge, + 1); + + this.input1 = new StreamTaskNetworkInput<>( + checkpointedInputGates[0], + inputSerializer1, + ioManager, + new StatusWatermarkValve(checkpointedInputGates[0].getNumberOfInputChannels(), output1), + 0); + this.input2 = new StreamTaskNetworkInput<>( + checkpointedInputGates[1], + inputSerializer2, ioManager, - unionedInputGate1, - unionedInputGate2, - taskManagerConfig, - taskName); - checkState(checkpointedInputGates.length == 2); - this.input1 = new StreamTaskNetworkInput(checkpointedInputGates[0], inputSerializer1, ioManager, 0); - this.input2 = new StreamTaskNetworkInput(checkpointedInputGates[1], inputSerializer2, ioManager, 1); - - this.statusWatermarkValve1 = new StatusWatermarkValve( - unionedInputGate1.getNumberOfInputChannels(), - new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input1WatermarkGauge, 0)); - this.statusWatermarkValve2 = new StatusWatermarkValve( - unionedInputGate2.getNumberOfInputChannels(), - new ForwardingValveOutputHandler(streamOperator, lock, streamStatusMaintainer, input2WatermarkGauge, 1)); + new StatusWatermarkValve(checkpointedInputGates[1].getNumberOfInputChannels(), output2), + 1); this.operatorChain = checkNotNull(operatorChain); - this.numRecordsIn = checkNotNull(numRecordsIn); + } + + private void processRecord1( + StreamRecord record, + TwoInputStreamOperator streamOperator, + Counter numRecordsIn) throws Exception { - this.firstStatus = StreamStatus.ACTIVE; - this.secondStatus = StreamStatus.ACTIVE; + streamOperator.setKeyContextElement1(record); + streamOperator.processElement1(record); + postProcessRecord(numRecordsIn); + } - this.lastReadInputIndex = 1; // always try to read from the first input + private void processRecord2( + StreamRecord record, + TwoInputStreamOperator streamOperator, + Counter numRecordsIn) throws Exception { - this.isPrepared = false; + streamOperator.setKeyContextElement2(record); + streamOperator.processElement2(record); + postProcessRecord(numRecordsIn); } - @Override - public boolean isFinished() { - return input1.isFinished() && input2.isFinished(); + private void postProcessRecord(Counter numRecordsIn) { + numRecordsIn.inc(); + inputSelectionHandler.nextSelection(); } @Override @@ -158,43 +162,46 @@ public final class StreamTwoInputProcessor implements StreamInputProce } @Override - public boolean processInput() throws Exception { - if (!isPrepared) { + public InputStatus processInput() throws Exception { + int readingInputIndex; + if (isPrepared) { + readingInputIndex = selectNextReadingInputIndex(); + assert readingInputIndex != -1; + } else { // the preparations here are not placed in the constructor because all work in it // must be executed after all operators are opened. - prepareForProcessing(); + readingInputIndex = selectFirstReadingInputIndex(); + if (readingInputIndex == -1) { + return InputStatus.NOTHING_AVAILABLE; + } } - int readingInputIndex = selectNextReadingInputIndex(); - if (readingInputIndex == -1) { - return false; - } lastReadInputIndex = readingInputIndex; - StreamElement recordOrMark; if (readingInputIndex == 0) { - recordOrMark = input1.pollNextNullable(); - if (recordOrMark != null) { - processElement1(recordOrMark, input1.getLastChannel()); - } - checkFinished(input1, lastReadInputIndex); + firstInputStatus = input1.emitNext(output1); + checkFinished(firstInputStatus, lastReadInputIndex); } else { - recordOrMark = input2.pollNextNullable(); - if (recordOrMark != null) { - processElement2(recordOrMark, input2.getLastChannel()); - } - checkFinished(input2, lastReadInputIndex); + secondInputStatus = input2.emitNext(output2); + checkFinished(secondInputStatus, lastReadInputIndex); } - if (recordOrMark == null) { - inputSelectionHandler.setUnavailableInput(readingInputIndex); - } + return getInputStatus(); + } - return recordOrMark != null; + private int selectFirstReadingInputIndex() throws IOException { + // Note: the first call to nextSelection () on the operator must be made after this operator + // is opened to ensure that any changes about the input selection in its open() + // method take effect. + inputSelectionHandler.nextSelection(); + + isPrepared = true; + + return selectNextReadingInputIndex(); } - private void checkFinished(StreamTaskInput input, int inputIndex) throws Exception { - if (input.isFinished()) { + private void checkFinished(InputStatus status, int inputIndex) throws Exception { + if (status == InputStatus.END_OF_INPUT) { synchronized (lock) { operatorChain.endInput(getInputId(inputIndex)); inputSelectionHandler.nextSelection(); @@ -202,6 +209,24 @@ public final class StreamTwoInputProcessor implements StreamInputProce } } + private InputStatus getInputStatus() { + if (firstInputStatus == InputStatus.END_OF_INPUT && secondInputStatus == InputStatus.END_OF_INPUT) { + return InputStatus.END_OF_INPUT; + } + + if (inputSelectionHandler.areAllInputsSelected()) { + if (firstInputStatus == InputStatus.MORE_AVAILABLE || secondInputStatus == InputStatus.MORE_AVAILABLE) { + return InputStatus.MORE_AVAILABLE; + } else { + return InputStatus.NOTHING_AVAILABLE; + } + } + + InputStatus selectedStatus = inputSelectionHandler.isFirstInputSelected() ? firstInputStatus : secondInputStatus; + InputStatus otherStatus = inputSelectionHandler.isFirstInputSelected() ? secondInputStatus : firstInputStatus; + return selectedStatus == InputStatus.END_OF_INPUT ? otherStatus : selectedStatus; + } + @Override public void close() throws IOException { IOException ex = null; @@ -233,8 +258,6 @@ public final class StreamTwoInputProcessor implements StreamInputProce // to avoid starvation, if the input selection is ALL and availableInputsMask is not ALL, // always try to check and set the availability of another input - // TODO: because this can be a costly operation (checking volatile inside CompletableFuture` - // this might be optimized to only check once per processed NetworkBuffer if (inputSelectionHandler.shouldSetAvailableForAnotherInput()) { checkAndSetAvailable(1 - readingInputIndex); } @@ -246,91 +269,47 @@ public final class StreamTwoInputProcessor implements StreamInputProce if (inputSelectionHandler.areAllInputsSelected()) { return; } - if (inputSelectionHandler.isFirstInputSelected() && input1.isFinished()) { + if (inputSelectionHandler.isFirstInputSelected() && firstInputStatus == InputStatus.END_OF_INPUT) { throw new IOException("Can not make a progress: only first input is selected but it is already finished"); } - if (inputSelectionHandler.isSecondInputSelected() && input2.isFinished()) { + if (inputSelectionHandler.isSecondInputSelected() && secondInputStatus == InputStatus.END_OF_INPUT) { throw new IOException("Can not make a progress: only second input is selected but it is already finished"); } } private void updateAvailability() { - if (!input1.isFinished() && input1.isAvailable() == AVAILABLE) { - inputSelectionHandler.setAvailableInput(input1.getInputIndex()); - } - if (!input2.isFinished() && input2.isAvailable() == AVAILABLE) { - inputSelectionHandler.setAvailableInput(input2.getInputIndex()); - } - } - - private void processElement1(StreamElement recordOrMark, int channel) throws Exception { - if (recordOrMark.isRecord()) { - StreamRecord record = recordOrMark.asRecord(); - synchronized (lock) { - numRecordsIn.inc(); - streamOperator.setKeyContextElement1(record); - streamOperator.processElement1(record); - inputSelectionHandler.nextSelection(); - } - } - else if (recordOrMark.isWatermark()) { - statusWatermarkValve1.inputWatermark(recordOrMark.asWatermark(), channel); - } else if (recordOrMark.isStreamStatus()) { - statusWatermarkValve1.inputStreamStatus(recordOrMark.asStreamStatus(), channel); - } else if (recordOrMark.isLatencyMarker()) { - synchronized (lock) { - streamOperator.processLatencyMarker1(recordOrMark.asLatencyMarker()); - } - } else { - throw new UnsupportedOperationException("Unknown type of StreamElement on input1"); - } + updateAvailability(firstInputStatus, input1); + updateAvailability(secondInputStatus, input2); } - private void processElement2(StreamElement recordOrMark, int channel) throws Exception { - if (recordOrMark.isRecord()) { - StreamRecord record = recordOrMark.asRecord(); - synchronized (lock) { - numRecordsIn.inc(); - streamOperator.setKeyContextElement2(record); - streamOperator.processElement2(record); - inputSelectionHandler.nextSelection(); - } - } - else if (recordOrMark.isWatermark()) { - statusWatermarkValve2.inputWatermark(recordOrMark.asWatermark(), channel); - } else if (recordOrMark.isStreamStatus()) { - statusWatermarkValve2.inputStreamStatus(recordOrMark.asStreamStatus(), channel); - } else if (recordOrMark.isLatencyMarker()) { - synchronized (lock) { - streamOperator.processLatencyMarker2(recordOrMark.asLatencyMarker()); - } + private void updateAvailability(InputStatus status, StreamTaskInput input) { + if (status == InputStatus.MORE_AVAILABLE || (status != InputStatus.END_OF_INPUT && input.isAvailable() == AVAILABLE)) { + inputSelectionHandler.setAvailableInput(input.getInputIndex()); } else { - throw new UnsupportedOperationException("Unknown type of StreamElement on input2"); + inputSelectionHandler.setUnavailableInput(input.getInputIndex()); } } - private void prepareForProcessing() { - // Note: the first call to nextSelection () on the operator must be made after this operator - // is opened to ensure that any changes about the input selection in its open() - // method take effect. - inputSelectionHandler.nextSelection(); - - isPrepared = true; - } - private void checkAndSetAvailable(int inputIndex) { - StreamTaskInput input = getInput(inputIndex); - if (!input.isFinished() && input.isAvailable().isDone()) { + InputStatus status = (inputIndex == 0 ? firstInputStatus : secondInputStatus); + if (status == InputStatus.END_OF_INPUT) { + return; + } + CompletableFuture inputAvailable = getInput(inputIndex).isAvailable(); + // TODO: inputAvailable.isDone() can be a costly operation (checking volatile). If one of + // the input is constantly available and another is not, we will be checking this volatile + // once per every record. This might be optimized to only check once per processed NetworkBuffer + if (inputAvailable == AVAILABLE || inputAvailable.isDone()) { inputSelectionHandler.setAvailableInput(inputIndex); } } private CompletableFuture isAnyInputAvailable() { - if (input1.isFinished()) { - return input2.isFinished() ? AVAILABLE : input2.isAvailable(); + if (firstInputStatus == InputStatus.END_OF_INPUT) { + return input2.isAvailable(); } - if (input2.isFinished()) { + if (secondInputStatus == InputStatus.END_OF_INPUT) { return input1.isAvailable(); } @@ -349,37 +328,46 @@ public final class StreamTwoInputProcessor implements StreamInputProce return inputIndex + 1; } - private class ForwardingValveOutputHandler implements StatusWatermarkValve.ValveOutputHandler { + /** + * The network data output implementation used for processing stream elements + * from {@link StreamTaskNetworkInput} in two input selective processor. + */ + private class StreamTaskNetworkOutput extends AbstractDataOutput { private final TwoInputStreamOperator operator; - private final Object lock; - - private final StreamStatusMaintainer streamStatusMaintainer; + /** The function way is only used for frequent record processing as for JIT optimization. */ + private final ThrowingConsumer, Exception> recordConsumer; private final WatermarkGauge inputWatermarkGauge; + /** The input index to indicate how to process elements by two input operator. */ private final int inputIndex; - private ForwardingValveOutputHandler( - TwoInputStreamOperator operator, - Object lock, - StreamStatusMaintainer streamStatusMaintainer, - WatermarkGauge inputWatermarkGauge, - int inputIndex) { + private StreamTaskNetworkOutput( + TwoInputStreamOperator operator, + ThrowingConsumer, Exception> recordConsumer, + Object lock, + StreamStatusMaintainer streamStatusMaintainer, + WatermarkGauge inputWatermarkGauge, + int inputIndex) { + super(streamStatusMaintainer, lock); this.operator = checkNotNull(operator); - this.lock = checkNotNull(lock); - - this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer); - - this.inputWatermarkGauge = inputWatermarkGauge; - + this.recordConsumer = checkNotNull(recordConsumer); + this.inputWatermarkGauge = checkNotNull(inputWatermarkGauge); this.inputIndex = inputIndex; } @Override - public void handleWatermark(Watermark watermark) throws Exception { + public void emitRecord(StreamRecord record) throws Exception { + synchronized (lock) { + recordConsumer.accept(record); + } + } + + @Override + public void emitWatermark(Watermark watermark) throws Exception { synchronized (lock) { inputWatermarkGauge.setCurrentWatermark(watermark.getTimestamp()); if (inputIndex == 0) { @@ -391,7 +379,7 @@ public final class StreamTwoInputProcessor implements StreamInputProce } @Override - public void handleStreamStatus(StreamStatus streamStatus) { + public void emitStreamStatus(StreamStatus streamStatus) { synchronized (lock) { final StreamStatus anotherStreamStatus; if (inputIndex == 0) { @@ -414,5 +402,16 @@ public final class StreamTwoInputProcessor implements StreamInputProce } } } + + @Override + public void emitLatencyMarker(LatencyMarker latencyMarker) throws Exception { + synchronized (lock) { + if (inputIndex == 0) { + operator.processLatencyMarker1(latencyMarker); + } else { + operator.processLatencyMarker2(latencyMarker); + } + } + } } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java index e69d41aedbd5b463f043adeef1c93d84eff13778..43704d57062a362d1e897ef6a904438000e24fc9 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValve.java @@ -21,6 +21,7 @@ package org.apache.flink.streaming.runtime.streamstatus; import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput; import org.apache.flink.util.Preconditions; import static org.apache.flink.util.Preconditions.checkArgument; @@ -29,24 +30,13 @@ import static org.apache.flink.util.Preconditions.checkNotNull; /** * A {@code StatusWatermarkValve} embodies the logic of how {@link Watermark} and {@link StreamStatus} are propagated to * downstream outputs, given a set of one or multiple input channels that continuously receive them. Usages of this - * class need to define the number of input channels that the valve needs to handle, as well as provide a customized - * implementation of {@link ValveOutputHandler}, which is called by the valve only when it determines a new watermark or - * stream status can be propagated. + * class need to define the number of input channels that the valve needs to handle, as well as provide a implementation of + * {@link DataOutput}, which is called by the valve only when it determines a new watermark or stream status can be propagated. */ @Internal public class StatusWatermarkValve { - /** - * Usages of {@code StatusWatermarkValve} should implement a {@code ValveOutputHandler} - * to handle watermark and stream status outputs from the valve. - */ - public interface ValveOutputHandler { - void handleWatermark(Watermark watermark) throws Exception; - - void handleStreamStatus(StreamStatus streamStatus) throws Exception; - } - - private final ValveOutputHandler outputHandler; + private final DataOutput output; // ------------------------------------------------------------------------ // Runtime state for watermark & stream status output determination @@ -68,9 +58,9 @@ public class StatusWatermarkValve { * Returns a new {@code StatusWatermarkValve}. * * @param numInputChannels the number of input channels that this valve will need to handle - * @param outputHandler the customized output handler for the valve + * @param output the customized output handler for the valve */ - public StatusWatermarkValve(int numInputChannels, ValveOutputHandler outputHandler) { + public StatusWatermarkValve(int numInputChannels, DataOutput output) { checkArgument(numInputChannels > 0); this.channelStatuses = new InputChannelStatus[numInputChannels]; for (int i = 0; i < numInputChannels; i++) { @@ -80,7 +70,7 @@ public class StatusWatermarkValve { channelStatuses[i].isWatermarkAligned = true; } - this.outputHandler = checkNotNull(outputHandler); + this.output = checkNotNull(output); this.lastOutputWatermark = Long.MIN_VALUE; this.lastOutputStreamStatus = StreamStatus.ACTIVE; @@ -88,7 +78,7 @@ public class StatusWatermarkValve { /** * Feed a {@link Watermark} into the valve. If the input triggers the valve to output a new Watermark, - * {@link ValveOutputHandler#handleWatermark(Watermark)} will be called to process the new Watermark. + * {@link DataOutput#emitWatermark(Watermark)} will be called to process the new Watermark. * * @param watermark the watermark to feed to the valve * @param channelIndex the index of the channel that the fed watermark belongs to (index starting from 0) @@ -115,8 +105,8 @@ public class StatusWatermarkValve { /** * Feed a {@link StreamStatus} into the valve. This may trigger the valve to output either a new Stream Status, - * for which {@link ValveOutputHandler#handleStreamStatus(StreamStatus)} will be called, or a new Watermark, - * for which {@link ValveOutputHandler#handleWatermark(Watermark)} will be called. + * for which {@link DataOutput#emitStreamStatus(StreamStatus)} will be called, or a new Watermark, + * for which {@link DataOutput#emitWatermark(Watermark)} will be called. * * @param streamStatus the stream status to feed to the valve * @param channelIndex the index of the channel that the fed stream status belongs to (index starting from 0) @@ -144,7 +134,7 @@ public class StatusWatermarkValve { } lastOutputStreamStatus = StreamStatus.IDLE; - outputHandler.handleStreamStatus(lastOutputStreamStatus); + output.emitStreamStatus(lastOutputStreamStatus); } else if (channelStatuses[channelIndex].watermark == lastOutputWatermark) { // if the watermark of the channel that just became idle equals the last output // watermark (the previous overall min watermark), we may be able to find a new @@ -165,7 +155,7 @@ public class StatusWatermarkValve { // status because at least one of the input channels is now active if (lastOutputStreamStatus.isIdle()) { lastOutputStreamStatus = StreamStatus.ACTIVE; - outputHandler.handleStreamStatus(lastOutputStreamStatus); + output.emitStreamStatus(lastOutputStreamStatus); } } } @@ -186,7 +176,7 @@ public class StatusWatermarkValve { // from some remaining aligned channel, and is also larger than the last output watermark if (hasAlignedChannels && newMinWatermark > lastOutputWatermark) { lastOutputWatermark = newMinWatermark; - outputHandler.handleWatermark(new Watermark(lastOutputWatermark)); + output.emitWatermark(new Watermark(lastOutputWatermark)); } } @@ -199,7 +189,7 @@ public class StatusWatermarkValve { if (maxWatermark > lastOutputWatermark) { lastOutputWatermark = maxWatermark; - outputHandler.handleWatermark(new Watermark(lastOutputWatermark)); + output.emitWatermark(new Watermark(lastOutputWatermark)); } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java index b7981b74a864f800351558f2eb1bb880ca964cd6..2610661d6c767a3328126c9bc9276568ce97c732 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/OneInputStreamTask.java @@ -21,16 +21,34 @@ package org.apache.flink.streaming.runtime.tasks; import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.metrics.Counter; import org.apache.flink.runtime.execution.Environment; import org.apache.flink.runtime.io.network.partition.consumer.InputGate; import org.apache.flink.runtime.metrics.MetricNames; +import org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup; import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.io.AbstractDataOutput; +import org.apache.flink.streaming.runtime.io.CheckpointedInputGate; +import org.apache.flink.streaming.runtime.io.InputGateUtil; +import org.apache.flink.streaming.runtime.io.InputProcessorUtil; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput; import org.apache.flink.streaming.runtime.io.StreamOneInputProcessor; +import org.apache.flink.streaming.runtime.io.StreamTaskInput; +import org.apache.flink.streaming.runtime.io.StreamTaskNetworkInput; import org.apache.flink.streaming.runtime.metrics.WatermarkGauge; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.streamstatus.StatusWatermarkValve; +import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; import javax.annotation.Nullable; +import java.io.IOException; + +import static org.apache.flink.util.Preconditions.checkNotNull; + /** * A {@link StreamTask} for executing a {@link OneInputStreamOperator}. */ @@ -68,31 +86,107 @@ public class OneInputStreamTask extends StreamTask inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); int numberOfInputs = configuration.getNumberOfInputs(); if (numberOfInputs > 0) { - InputGate[] inputGates = getEnvironment().getAllInputGates(); + CheckpointedInputGate inputGate = createCheckpointedInputGate(); + TaskIOMetricGroup taskIOMetricGroup = getEnvironment().getMetricGroup().getIOMetricGroup(); + taskIOMetricGroup.gauge("checkpointAlignmentTime", inputGate::getAlignmentDurationNanos); + DataOutput output = createDataOutput(); + StreamTaskInput input = createTaskInput(inputGate, output); inputProcessor = new StreamOneInputProcessor<>( - inputGates, - inSerializer, - this, - configuration.getCheckpointMode(), + input, + output, getCheckpointLock(), - getEnvironment().getIOManager(), - getEnvironment().getTaskManagerInfo().getConfiguration(), - getStreamStatusMaintainer(), - headOperator, - getEnvironment().getMetricGroup().getIOMetricGroup(), - inputWatermarkGauge, - getTaskNameWithSubtaskAndId(), - operatorChain, - setupNumRecordsInCounter(headOperator)); + operatorChain); } headOperator.getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge); // wrap watermark gauge since registered metrics must be unique getEnvironment().getMetricGroup().gauge(MetricNames.IO_CURRENT_INPUT_WATERMARK, this.inputWatermarkGauge::getValue); } + + private CheckpointedInputGate createCheckpointedInputGate() throws IOException { + InputGate[] inputGates = getEnvironment().getAllInputGates(); + InputGate inputGate = InputGateUtil.createInputGate(inputGates); + + return InputProcessorUtil.createCheckpointedInputGate( + this, + configuration.getCheckpointMode(), + getEnvironment().getIOManager(), + inputGate, + getEnvironment().getTaskManagerInfo().getConfiguration(), + getTaskNameWithSubtaskAndId()); + } + + private DataOutput createDataOutput() { + return new StreamTaskNetworkOutput<>( + headOperator, + getStreamStatusMaintainer(), + getCheckpointLock(), + inputWatermarkGauge, + setupNumRecordsInCounter(headOperator)); + } + + private StreamTaskInput createTaskInput(CheckpointedInputGate inputGate, DataOutput output) { + int numberOfInputChannels = inputGate.getNumberOfInputChannels(); + StatusWatermarkValve statusWatermarkValve = new StatusWatermarkValve(numberOfInputChannels, output); + + TypeSerializer inSerializer = configuration.getTypeSerializerIn1(getUserCodeClassLoader()); + return new StreamTaskNetworkInput<>( + inputGate, + inSerializer, + getEnvironment().getIOManager(), + statusWatermarkValve, + 0); + } + + /** + * The network data output implementation used for processing stream elements + * from {@link StreamTaskNetworkInput} in one input processor. + */ + private static class StreamTaskNetworkOutput extends AbstractDataOutput { + + private final OneInputStreamOperator operator; + + private final WatermarkGauge watermarkGauge; + private final Counter numRecordsIn; + + private StreamTaskNetworkOutput( + OneInputStreamOperator operator, + StreamStatusMaintainer streamStatusMaintainer, + Object lock, + WatermarkGauge watermarkGauge, + Counter numRecordsIn) { + super(streamStatusMaintainer, lock); + + this.operator = checkNotNull(operator); + this.watermarkGauge = checkNotNull(watermarkGauge); + this.numRecordsIn = checkNotNull(numRecordsIn); + } + + @Override + public void emitRecord(StreamRecord record) throws Exception { + synchronized (lock) { + numRecordsIn.inc(); + operator.setKeyContextElement1(record); + operator.processElement(record); + } + } + + @Override + public void emitWatermark(Watermark watermark) throws Exception { + synchronized (lock) { + watermarkGauge.setCurrentWatermark(watermark.getTimestamp()); + operator.processWatermark(watermark); + } + } + + @Override + public void emitLatencyMarker(LatencyMarker latencyMarker) throws Exception { + synchronized (lock) { + operator.processLatencyMarker(latencyMarker); + } + } + } } diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceReaderStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceReaderStreamTask.java new file mode 100644 index 0000000000000000000000000000000000000000..57434d37d9fc89c38ae7be00714a3b1c6a30379f --- /dev/null +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/SourceReaderStreamTask.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.runtime.tasks; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.runtime.execution.Environment; +import org.apache.flink.streaming.api.operators.Output; +import org.apache.flink.streaming.api.operators.SourceReaderOperator; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.io.AbstractDataOutput; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput; +import org.apache.flink.streaming.runtime.io.StreamOneInputProcessor; +import org.apache.flink.streaming.runtime.io.StreamTaskInput; +import org.apache.flink.streaming.runtime.io.StreamTaskSourceInput; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A subclass of {@link StreamTask} for executing the {@link SourceReaderOperator}. + */ +@Internal +public class SourceReaderStreamTask extends StreamTask> { + + public SourceReaderStreamTask(Environment env) { + super(env); + } + + @Override + public void init() { + StreamTaskInput input = new StreamTaskSourceInput<>(headOperator); + DataOutput output = new StreamTaskSourceOutput<>( + operatorChain.getChainEntryPoint(), + getStreamStatusMaintainer(), + getCheckpointLock()); + + inputProcessor = new StreamOneInputProcessor<>( + input, + output, + getCheckpointLock(), + operatorChain); + } + + /** + * Implementation of {@link DataOutput} that wraps a specific {@link Output} to emit + * stream elements for {@link SourceReaderOperator}. + */ + private static class StreamTaskSourceOutput extends AbstractDataOutput { + + private final Output> output; + + StreamTaskSourceOutput( + Output> output, + StreamStatusMaintainer streamStatusMaintainer, + Object lock) { + super(streamStatusMaintainer, lock); + + this.output = checkNotNull(output); + } + + @Override + public void emitRecord(StreamRecord streamRecord) { + synchronized (lock) { + output.collect(streamRecord); + } + } + + @Override + public void emitLatencyMarker(LatencyMarker latencyMarker) { + synchronized (lock) { + output.emitLatencyMarker(latencyMarker); + } + } + + @Override + public void emitWatermark(Watermark watermark) { + synchronized (lock) { + output.emitWatermark(watermark); + } + } + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java index 313beb401e459bef4f81d587441b5576a64c29f6..61b1bce3801a5b60af2ac88657898cd794e52450 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/StreamTask.java @@ -58,6 +58,7 @@ import org.apache.flink.streaming.api.operators.OperatorSnapshotFutures; import org.apache.flink.streaming.api.operators.StreamOperator; import org.apache.flink.streaming.api.operators.StreamTaskStateInitializer; import org.apache.flink.streaming.api.operators.StreamTaskStateInitializerImpl; +import org.apache.flink.streaming.runtime.io.InputStatus; import org.apache.flink.streaming.runtime.io.RecordWriterOutput; import org.apache.flink.streaming.runtime.io.StreamInputProcessor; import org.apache.flink.streaming.runtime.partitioner.ConfigurableStreamPartitioner; @@ -199,9 +200,6 @@ public abstract class StreamTask> /** Thread pool for async snapshot workers. */ private ExecutorService asyncOperationsThreadPool; - /** Handler for exceptions during checkpointing in the stream task. Used in synchronous part of the checkpoint. */ - private CheckpointExceptionHandler checkpointExceptionHandler; - private final List>>> recordWriters; protected final MailboxProcessor mailboxProcessor; @@ -279,14 +277,13 @@ public abstract class StreamTask> * @throws Exception on any problems in the action. */ protected void processInput(DefaultActionContext context) throws Exception { - if (!inputProcessor.processInput()) { - if (inputProcessor.isFinished()) { - context.allActionsCompleted(); - } - else { - SuspendedMailboxDefaultAction suspendedDefaultAction = context.suspendDefaultAction(); - inputProcessor.isAvailable().thenRun(suspendedDefaultAction::resume); - } + InputStatus status = inputProcessor.processInput(); + if (status == InputStatus.END_OF_INPUT) { + context.allActionsCompleted(); + } + else if (status == InputStatus.NOTHING_AVAILABLE){ + SuspendedMailboxDefaultAction suspendedDefaultAction = context.suspendDefaultAction(); + inputProcessor.isAvailable().thenRun(suspendedDefaultAction::resume); } } @@ -375,11 +372,6 @@ public abstract class StreamTask> asyncOperationsThreadPool = Executors.newCachedThreadPool(new ExecutorThreadFactory("AsyncOperations", uncaughtExceptionHandler)); - CheckpointExceptionHandlerFactory cpExceptionHandlerFactory = createCheckpointExceptionHandlerFactory(); - - checkpointExceptionHandler = cpExceptionHandlerFactory - .createCheckpointExceptionHandler(getEnvironment()); - stateBackend = createStateBackend(); checkpointStorage = stateBackend.createCheckpointStorage(getEnvironment().getJobID()); @@ -522,6 +514,11 @@ public abstract class StreamTask> synchronized (lock) { operatorChain.releaseOutputs(); } + } else { + // failed to allocate operatorChain, clean up record writers + for (RecordWriter>> writer: recordWriters) { + writer.close(); + } } mailboxProcessor.close(); @@ -974,10 +971,6 @@ public abstract class StreamTask> LOG); } - protected CheckpointExceptionHandlerFactory createCheckpointExceptionHandlerFactory() { - return new CheckpointExceptionHandlerFactory(); - } - /** * Returns the {@link ProcessingTimeService} responsible for telling the current * processing time and registering timers. @@ -1186,7 +1179,7 @@ public abstract class StreamTask> // We only report the exception for the original cause of fail and cleanup. // Otherwise this followup exception could race the original exception in failing the task. try { - owner.checkpointExceptionHandler.tryHandleCheckpointException(checkpointMetaData, checkpointException); + owner.getEnvironment().declineCheckpoint(checkpointMetaData.getCheckpointId(), checkpointException); } catch (Exception unhandled) { AsynchronousException asyncException = new AsynchronousException(unhandled); owner.handleAsyncException("Failure in asynchronous checkpoint materialization", asyncException); @@ -1348,12 +1341,12 @@ public abstract class StreamTask> if (checkpointOptions.getCheckpointType().isSynchronous()) { // in the case of a synchronous checkpoint, we always rethrow the exception, - // so that the task fails (as if we had the FailingCheckpointExceptionHandler). + // so that the task fails. // this is because the intention is always to stop the job after this checkpointing // operation, and without the failure, the task would go back to normal execution. throw ex; } else { - owner.checkpointExceptionHandler.tryHandleCheckpointException(checkpointMetaData, ex); + owner.getEnvironment().declineCheckpoint(checkpointMetaData.getCheckpointId(), ex); } } } @@ -1420,7 +1413,7 @@ public abstract class StreamTask> } } - RecordWriter>> output = new RecordWriterBuilder() + RecordWriter>> output = new RecordWriterBuilder>>() .setChannelSelector(outputPartitioner) .setTimeout(bufferTimeout) .setTaskName(taskName) diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java index 41fea759482c57d0181584216159116828feece8..f8c96facbf51fa6f50df8fdf70ce48b4ada94902 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java +++ b/flink-streaming-java/src/main/java/org/apache/flink/streaming/runtime/tasks/TwoInputStreamTask.java @@ -23,12 +23,17 @@ import org.apache.flink.runtime.execution.Environment; import org.apache.flink.runtime.io.network.partition.consumer.InputGate; import org.apache.flink.streaming.api.operators.InputSelectable; import org.apache.flink.streaming.api.operators.TwoInputStreamOperator; +import org.apache.flink.streaming.runtime.io.CheckpointedInputGate; +import org.apache.flink.streaming.runtime.io.InputGateUtil; +import org.apache.flink.streaming.runtime.io.InputProcessorUtil; import org.apache.flink.streaming.runtime.io.StreamTwoInputProcessor; import org.apache.flink.streaming.runtime.io.TwoInputSelectionHandler; import java.io.IOException; import java.util.Collection; +import static org.apache.flink.util.Preconditions.checkState; + /** * A {@link StreamTask} for executing a {@link TwoInputStreamOperator} and supporting * the {@link TwoInputStreamOperator} to select input for reading. @@ -50,20 +55,31 @@ public class TwoInputStreamTask extends AbstractTwoInputStreamTas TwoInputSelectionHandler twoInputSelectionHandler = new TwoInputSelectionHandler( headOperator instanceof InputSelectable ? (InputSelectable) headOperator : null); - this.inputProcessor = new StreamTwoInputProcessor<>( - inputGates1, inputGates2, - inputDeserializer1, inputDeserializer2, + InputGate unionedInputGate1 = InputGateUtil.createInputGate(inputGates1.toArray(new InputGate[0])); + InputGate unionedInputGate2 = InputGateUtil.createInputGate(inputGates2.toArray(new InputGate[0])); + + // create a Input instance for each input + CheckpointedInputGate[] checkpointedInputGates = InputProcessorUtil.createCheckpointedInputGatePair( this, getConfiguration().getCheckpointMode(), - getCheckpointLock(), getEnvironment().getIOManager(), + unionedInputGate1, + unionedInputGate2, getEnvironment().getTaskManagerInfo().getConfiguration(), + getTaskNameWithSubtaskAndId()); + checkState(checkpointedInputGates.length == 2); + + inputProcessor = new StreamTwoInputProcessor<>( + checkpointedInputGates, + inputDeserializer1, + inputDeserializer2, + getCheckpointLock(), + getEnvironment().getIOManager(), getStreamStatusMaintainer(), - this.headOperator, + headOperator, twoInputSelectionHandler, input1WatermarkGauge, input2WatermarkGauge, - getTaskNameWithSubtaskAndId(), operatorChain, setupNumRecordsInCounter(headOperator)); } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/StreamTestSingleInputGate.java b/flink-streaming-java/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/StreamTestSingleInputGate.java index 291e15ccd6771c063f3e2e72b5aa7d124857c104..d3754fde19af1992cc5a12837ad1ca81ede6db05 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/StreamTestSingleInputGate.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/runtime/io/network/partition/consumer/StreamTestSingleInputGate.java @@ -27,6 +27,7 @@ import org.apache.flink.runtime.io.network.api.serialization.EventSerializer; import org.apache.flink.runtime.io.network.api.serialization.RecordSerializer; import org.apache.flink.runtime.io.network.api.serialization.SpanningRecordSerializer; import org.apache.flink.runtime.io.network.buffer.BufferBuilder; +import org.apache.flink.runtime.io.network.buffer.BufferConsumer; import org.apache.flink.runtime.io.network.partition.consumer.InputChannel.BufferAndAvailability; import org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel.BufferAndAvailabilityProvider; import org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID; @@ -38,7 +39,6 @@ import java.io.IOException; import java.util.Optional; import java.util.concurrent.ConcurrentLinkedQueue; -import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.buildSingleBuffer; import static org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createBufferBuilder; /** @@ -106,11 +106,12 @@ public class StreamTestSingleInputGate extends TestSingleInputGate { delegate.setInstance(inputElement); recordSerializer.serializeRecord(delegate); BufferBuilder bufferBuilder = createBufferBuilder(bufferSize); + BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer(); recordSerializer.copyToBufferBuilder(bufferBuilder); bufferBuilder.finish(); // Call getCurrentBuffer to ensure size is set - return Optional.of(new BufferAndAvailability(buildSingleBuffer(bufferBuilder), moreAvailable, 0)); + return Optional.of(new BufferAndAvailability(bufferConsumer.build(), moreAvailable, 0)); } else if (input != null && input.isEvent()) { AbstractEvent event = input.getEvent(); if (event instanceof EndOfPartitionEvent) { diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/environment/RemoteStreamExecutionEnvironmentTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/environment/RemoteStreamExecutionEnvironmentTest.java index b30c33844a39dcc513a85b653a416df0e12dd223..acfa6050be4c41bca66b4e9a43f818aa5769af7f 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/environment/RemoteStreamExecutionEnvironmentTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/environment/RemoteStreamExecutionEnvironmentTest.java @@ -19,6 +19,7 @@ package org.apache.flink.streaming.api.environment; import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.client.RemoteExecutor; import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.RestOptions; @@ -39,7 +40,8 @@ import static org.mockito.Mockito.when; * Tests for the {@link RemoteStreamEnvironment}. */ @RunWith(PowerMockRunner.class) -@PrepareForTest({RemoteStreamEnvironment.class}) +// TODO: I don't like that I have to do this +@PrepareForTest({RemoteStreamEnvironment.class, RemoteExecutor.class}) public class RemoteStreamExecutionEnvironmentTest extends TestLogger { /** @@ -53,8 +55,7 @@ public class RemoteStreamExecutionEnvironmentTest extends TestLogger { JobExecutionResult expectedResult = new JobExecutionResult(null, 0, null); RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class); - when(mockedClient.run(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) - .thenReturn(expectedResult); + when(mockedClient.submitJob(Mockito.any(), Mockito.any())).thenReturn(expectedResult); PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenAnswer((invocation) -> { Object[] args = invocation.getArguments(); @@ -85,8 +86,7 @@ public class RemoteStreamExecutionEnvironmentTest extends TestLogger { JobExecutionResult expectedResult = new JobExecutionResult(null, 0, null); PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenReturn(mockedClient); - when(mockedClient.run(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.eq(restoreSettings))) - .thenReturn(expectedResult); + when(mockedClient.submitJob(Mockito.any(), Mockito.any())).thenReturn(expectedResult); JobExecutionResult actualResult = env.execute("fakeJobName"); Assert.assertEquals(expectedResult, actualResult); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunctionTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunctionTest.java index 2970b87789d3b2c3cba6abf88c0fb4f3acc94d05..84c01043c6e8aac730f1a7829e894d24613fbc61 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunctionTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/TwoPhaseCommitSinkFunctionTest.java @@ -47,6 +47,7 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import static org.apache.flink.util.ExceptionUtils.findSerializedThrowable; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.hasItem; import static org.junit.Assert.assertEquals; @@ -167,7 +168,7 @@ public class TwoPhaseCommitSinkFunctionTest { harness.snapshot(2, 5); fail("something should fail"); } catch (Exception ex) { - if (!(ex.getCause() instanceof ContentDump.NotWritableException)) { + if (!findSerializedThrowable(ex, ContentDump.NotWritableException.class, ClassLoader.getSystemClassLoader()).isPresent()) { throw ex; } // ignore diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkWriterTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkWriterTest.java index 3c1b9badc8ad8dda5d157cdf0d4e937fff6821b1..bab18aac3b0536b35dbf4db3fc4eec213710064a 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkWriterTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/BulkWriterTest.java @@ -61,7 +61,28 @@ public class BulkWriterTest extends TestLogger { new TestBulkWriterFactory(), new DefaultBucketFactoryImpl<>()) ) { - testPartFiles(testHarness, outDir, ".part-0-0.inprogress", ".part-0-1.inprogress"); + testPartFilesWithStringBucketer(testHarness, outDir, ".part-0-0.inprogress", ".part-0-1.inprogress"); + } + } + + @Test + public void testCustomBulkWriterWithBucketAssigner() throws Exception { + final File outDir = TEMP_FOLDER.newFolder(); + + // we set the max bucket size to small so that we can know when it rolls + try ( + OneInputStreamOperatorTestHarness, Object> testHarness = + TestUtils.createTestSinkWithCustomizedBulkEncoder( + outDir, + 1, + 0, + 10L, + // use a customized bucketer with Integer bucket ID + new TestUtils.TupleToIntegerBucketer(), + new TestBulkWriterFactory(), + new DefaultBucketFactoryImpl<>()) + ) { + testPartFilesWithIntegerBucketer(testHarness, outDir, ".part-0-0.inprogress", ".part-0-1.inprogress", ".part-0-2.inprogress"); } } @@ -83,11 +104,11 @@ public class BulkWriterTest extends TestLogger { "prefix", ".ext") ) { - testPartFiles(testHarness, outDir, ".prefix-0-0.ext.inprogress", ".prefix-0-1.ext.inprogress"); + testPartFilesWithStringBucketer(testHarness, outDir, ".prefix-0-0.ext.inprogress", ".prefix-0-1.ext.inprogress"); } } - private void testPartFiles( + private void testPartFilesWithStringBucketer( OneInputStreamOperatorTestHarness, Object> testHarness, File outDir, String partFileName1, @@ -122,6 +143,8 @@ public class BulkWriterTest extends TestLogger { fileCounter++; Assert.assertEquals("test1@2\ntest1@3\n", fileContents.getValue()); } + // check bucket name + Assert.assertEquals("test1", fileContents.getKey().getParentFile().getName()); } Assert.assertEquals(2L, fileCounter); @@ -131,6 +154,57 @@ public class BulkWriterTest extends TestLogger { TestUtils.checkLocalFs(outDir, 0, 2); } + private void testPartFilesWithIntegerBucketer( + OneInputStreamOperatorTestHarness, Object> testHarness, + File outDir, + String partFileName1, + String partFileName2, + String partFileName3) throws Exception { + + testHarness.setup(); + testHarness.open(); + + // this creates a new bucket "test1" and part-0-0 + testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 1), 1L)); + TestUtils.checkLocalFs(outDir, 1, 0); + + // we take a checkpoint so we roll. + testHarness.snapshot(1L, 1L); + + // these will close part-0-0 and open part-0-1 and part-0-2 + testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 2), 2L)); + testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 3), 3L)); + + // we take a checkpoint so we roll again. + testHarness.snapshot(2L, 2L); + + TestUtils.checkLocalFs(outDir, 3, 0); + + Map contents = TestUtils.getFileContentByPath(outDir); + int fileCounter = 0; + for (Map.Entry fileContents : contents.entrySet()) { + if (fileContents.getKey().getName().contains(partFileName1)) { + fileCounter++; + Assert.assertEquals("test1@1\n", fileContents.getValue()); + Assert.assertEquals("1", fileContents.getKey().getParentFile().getName()); + } else if (fileContents.getKey().getName().contains(partFileName2)) { + fileCounter++; + Assert.assertEquals("test1@2\n", fileContents.getValue()); + Assert.assertEquals("2", fileContents.getKey().getParentFile().getName()); + } else if (fileContents.getKey().getName().contains(partFileName3)) { + fileCounter++; + Assert.assertEquals("test1@3\n", fileContents.getValue()); + Assert.assertEquals("3", fileContents.getKey().getParentFile().getName()); + } + } + Assert.assertEquals(3L, fileCounter); + + // we acknowledge the latest checkpoint, so everything should be published. + testHarness.notifyOfCompletedCheckpoint(2L); + + TestUtils.checkLocalFs(outDir, 0, 3); + } + /** * A {@link BulkWriter} used for the tests. */ diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/LocalStreamingFileSinkTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/LocalStreamingFileSinkTest.java index 5b5aff35db3fa8a4a170a9439420595af44a46e3..82033756791d4922d06613417bfffeeb783e831b 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/LocalStreamingFileSinkTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/LocalStreamingFileSinkTest.java @@ -20,6 +20,9 @@ package org.apache.flink.streaming.api.functions.sink.filesystem; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.runtime.checkpoint.OperatorSubtaskState; +import org.apache.flink.streaming.api.functions.sink.filesystem.TestUtils.Tuple2Encoder; +import org.apache.flink.streaming.api.functions.sink.filesystem.TestUtils.TupleToIntegerBucketer; +import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness; @@ -403,6 +406,65 @@ public class LocalStreamingFileSinkTest extends TestLogger { TestUtils.checkLocalFs(outDir, 1, 3); } + @Test + public void testClosingWithCustomizedBucketer() throws Exception { + final File outDir = TEMP_FOLDER.newFolder(); + final long partMaxSize = 2L; + final long inactivityInterval = 100L; + final RollingPolicy, Integer> rollingPolicy = + DefaultRollingPolicy + .builder() + .withMaxPartSize(partMaxSize) + .withRolloverInterval(inactivityInterval) + .withInactivityInterval(inactivityInterval) + .build(); + + try ( + OneInputStreamOperatorTestHarness, Object> testHarness = + TestUtils.createCustomizedRescalingTestSink(outDir, 1, 0, 100L, new TupleToIntegerBucketer(), new Tuple2Encoder(), rollingPolicy, new DefaultBucketFactoryImpl<>()); + ) { + testHarness.setup(); + testHarness.open(); + + testHarness.setProcessingTime(0L); + + testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 1), 1L)); + testHarness.processElement(new StreamRecord<>(Tuple2.of("test2", 2), 1L)); + TestUtils.checkLocalFs(outDir, 2, 0); + + // this is to check the inactivity threshold + testHarness.setProcessingTime(101L); + TestUtils.checkLocalFs(outDir, 2, 0); + + testHarness.processElement(new StreamRecord<>(Tuple2.of("test3", 3), 1L)); + TestUtils.checkLocalFs(outDir, 3, 0); + + testHarness.snapshot(0L, 1L); + TestUtils.checkLocalFs(outDir, 3, 0); + + testHarness.notifyOfCompletedCheckpoint(0L); + TestUtils.checkLocalFs(outDir, 0, 3); + + testHarness.processElement(new StreamRecord<>(Tuple2.of("test4", 4), 10L)); + TestUtils.checkLocalFs(outDir, 1, 3); + + testHarness.snapshot(1L, 0L); + testHarness.notifyOfCompletedCheckpoint(1L); + } + + // at close all files moved to final. + TestUtils.checkLocalFs(outDir, 0, 4); + + // check file content and bucket ID. + Map contents = TestUtils.getFileContentByPath(outDir); + for (Map.Entry fileContents : contents.entrySet()) { + Integer bucketId = Integer.parseInt(fileContents.getKey().getParentFile().getName()); + + Assert.assertTrue(bucketId >= 1 && bucketId <= 4); + Assert.assertEquals(String.format("test%d@%d\n", bucketId, bucketId), fileContents.getValue()); + } + } + @Test public void testScalingDownAndMergingOfStates() throws Exception { final File outDir = TEMP_FOLDER.newFolder(); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestUtils.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestUtils.java index 55ba6d33e183748c193954247d49672c3f2c0b4c..c17a2d6a24ab305a749e2852c80148d9a6fcb96a 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestUtils.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/functions/sink/filesystem/TestUtils.java @@ -38,6 +38,9 @@ import javax.annotation.Nullable; import java.io.File; import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; @@ -68,25 +71,18 @@ public class TestUtils { .withInactivityInterval(inactivityInterval) .build(); - final BucketAssigner, String> bucketer = new TupleToStringBucketer(); - - final Encoder> encoder = (element, stream) -> { - stream.write((element.f0 + '@' + element.f1).getBytes(StandardCharsets.UTF_8)); - stream.write('\n'); - }; - - return createCustomRescalingTestSink( + return createRescalingTestSink( outDir, totalParallelism, taskIdx, 10L, - bucketer, - encoder, + new TupleToStringBucketer(), + new Tuple2Encoder(), rollingPolicy, new DefaultBucketFactoryImpl<>()); } - static OneInputStreamOperatorTestHarness, Object> createCustomRescalingTestSink( + static OneInputStreamOperatorTestHarness, Object> createRescalingTestSink( final File outDir, final int totalParallelism, final int taskIdx, @@ -107,6 +103,26 @@ public class TestUtils { return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), MAX_PARALLELISM, totalParallelism, taskIdx); } + static OneInputStreamOperatorTestHarness, Object> createCustomizedRescalingTestSink( + final File outDir, + final int totalParallelism, + final int taskIdx, + final long bucketCheckInterval, + final BucketAssigner, ID> bucketer, + final Encoder> writer, + final RollingPolicy, ID> rollingPolicy, + final BucketFactory, ID> bucketFactory) throws Exception { + + StreamingFileSink> sink = StreamingFileSink + .forRowFormat(new Path(outDir.toURI()), writer) + .withNewBucketAssignerAndPolicy(bucketer, rollingPolicy) + .withBucketCheckInterval(bucketCheckInterval) + .withBucketFactory(bucketFactory) + .build(); + + return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), MAX_PARALLELISM, totalParallelism, taskIdx); + } + static OneInputStreamOperatorTestHarness, Object> createTestSinkWithBulkEncoder( final File outDir, final int totalParallelism, @@ -151,6 +167,50 @@ public class TestUtils { return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), MAX_PARALLELISM, totalParallelism, taskIdx); } + static OneInputStreamOperatorTestHarness, Object> createTestSinkWithCustomizedBulkEncoder( + final File outDir, + final int totalParallelism, + final int taskIdx, + final long bucketCheckInterval, + final BucketAssigner, ID> bucketer, + final BulkWriter.Factory> writer, + final BucketFactory, ID> bucketFactory) throws Exception { + + return createTestSinkWithCustomizedBulkEncoder( + outDir, + totalParallelism, + taskIdx, + bucketCheckInterval, + bucketer, + writer, + bucketFactory, + PartFileConfig.DEFAULT_PART_PREFIX, + PartFileConfig.DEFAULT_PART_SUFFIX); + } + + static OneInputStreamOperatorTestHarness, Object> createTestSinkWithCustomizedBulkEncoder( + final File outDir, + final int totalParallelism, + final int taskIdx, + final long bucketCheckInterval, + final BucketAssigner, ID> bucketer, + final BulkWriter.Factory> writer, + final BucketFactory, ID> bucketFactory, + final String partFilePrefix, + final String partFileSuffix) throws Exception { + + StreamingFileSink> sink = StreamingFileSink + .forBulkFormat(new Path(outDir.toURI()), writer) + .withNewBucketAssigner(bucketer) + .withBucketCheckInterval(bucketCheckInterval) + .withBucketFactory(bucketFactory) + .withPartFilePrefix(partFilePrefix) + .withPartFileSuffix(partFileSuffix) + .build(); + + return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), MAX_PARALLELISM, totalParallelism, taskIdx); + } + static void checkLocalFs(File outDir, int expectedInProgress, int expectedCompleted) { int inProgress = 0; int finished = 0; @@ -181,6 +241,21 @@ public class TestUtils { return contents; } + /** + * A simple {@link Encoder} that encodes {@code Tuple2} object. + */ + static class Tuple2Encoder implements Encoder> { + @Override + public void encode(Tuple2 element, OutputStream stream) throws IOException { + stream.write((element.f0 + '@' + element.f1).getBytes(StandardCharsets.UTF_8)); + stream.write('\n'); + } + } + + /** + * A simple {@link BucketAssigner} that returns the first (String) element of a {@code Tuple2} + * object as the bucket id. + */ static class TupleToStringBucketer implements BucketAssigner, String> { private static final long serialVersionUID = 1L; @@ -196,6 +271,48 @@ public class TestUtils { } } + /** + * A simple {@link BucketAssigner} that returns the second (Integer) element of a {@code Tuple2} + * object as the bucket id. + */ + static class TupleToIntegerBucketer implements BucketAssigner, Integer> { + + private static final long serialVersionUID = 1L; + + @Override + public Integer getBucketId(Tuple2 element, Context context) { + return element.f1; + } + + @Override + public SimpleVersionedSerializer getSerializer() { + return new SimpleVersionedIntegerSerializer(); + } + } + + private static final class SimpleVersionedIntegerSerializer implements SimpleVersionedSerializer { + static final int VERSION = 1; + + private SimpleVersionedIntegerSerializer() { + } + + public int getVersion() { + return 1; + } + + public byte[] serialize(Integer value) { + byte[] bytes = new byte[4]; + ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN).putInt(value); + return bytes; + } + + public Integer deserialize(int version, byte[] serialized) throws IOException { + Assert.assertEquals(1L, (long) version); + Assert.assertEquals(4L, serialized.length); + return ByteBuffer.wrap(serialized).order(ByteOrder.LITTLE_ENDIAN).getInt(); + } + } + /** * A simple {@link BucketAssigner} that accepts {@code String}'s * and returns the element itself as the bucket id. diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java index e09334aa8cf8447d07541929e9d034abd35fa326..fc057e1027a431f91ec570b805bdc5517a05a2f7 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamGraphGeneratorTest.java @@ -23,10 +23,12 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.api.dag.Transformation; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.streaming.api.datastream.ConnectedStreams; import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.IterativeStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.co.CoMapFunction; import org.apache.flink.streaming.api.functions.sink.DiscardingSink; @@ -53,11 +55,14 @@ import org.junit.Test; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -68,6 +73,19 @@ import static org.junit.Assert.assertTrue; @SuppressWarnings("serial") public class StreamGraphGeneratorTest { + @Test + public void generatorForwardsSavepointRestoreSettings() { + StreamGraphGenerator streamGraphGenerator = + new StreamGraphGenerator(Collections.emptyList(), + new ExecutionConfig(), + new CheckpointConfig()); + + streamGraphGenerator.setSavepointRestoreSettings(SavepointRestoreSettings.forPath("hello")); + + StreamGraph streamGraph = streamGraphGenerator.generate(); + assertThat(streamGraph.getSavepointRestoreSettings().getRestorePath(), is("hello")); + } + @Test public void testBufferTimeout() { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java index 1334d2830da375a3ee8bd39a5b66ab844ed6ac42..18b25f7936941de9ac875874346440e36e7b0bc3 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/graph/StreamingJobGraphGeneratorTest.java @@ -17,6 +17,7 @@ package org.apache.flink.streaming.api.graph; +import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.functions.FilterFunction; import org.apache.flink.api.common.functions.FlatMapFunction; import org.apache.flink.api.common.functions.MapFunction; @@ -37,6 +38,7 @@ import org.apache.flink.runtime.jobgraph.InputOutputFormatVertex; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobVertex; import org.apache.flink.runtime.jobgraph.OperatorID; +import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; import org.apache.flink.runtime.jobgraph.ScheduleMode; import org.apache.flink.runtime.jobgraph.tasks.JobCheckpointingSettings; import org.apache.flink.runtime.jobmanager.scheduler.CoLocationGroup; @@ -47,6 +49,7 @@ import org.apache.flink.streaming.api.datastream.DataStreamSink; import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.datastream.IterativeStream; import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; +import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.functions.sink.SinkFunction; import org.apache.flink.streaming.api.functions.source.InputFormatSourceFunction; @@ -68,10 +71,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -137,7 +142,7 @@ public class StreamingJobGraphGeneratorTest extends TestLogger { @Test public void testDisabledCheckpointing() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - StreamGraph streamGraph = new StreamGraph(env.getConfig(), env.getCheckpointConfig()); + StreamGraph streamGraph = new StreamGraph(env.getConfig(), env.getCheckpointConfig(), SavepointRestoreSettings.none()); assertFalse("Checkpointing enabled", streamGraph.getCheckpointConfig().isCheckpointingEnabled()); JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph); @@ -146,6 +151,19 @@ public class StreamingJobGraphGeneratorTest extends TestLogger { assertEquals(Long.MAX_VALUE, snapshottingSettings.getCheckpointCoordinatorConfiguration().getCheckpointInterval()); } + @Test + public void generatorForwardsSavepointRestoreSettings() { + StreamGraph streamGraph = new StreamGraph( + new ExecutionConfig(), + new CheckpointConfig(), + SavepointRestoreSettings.forPath("hello")); + + JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph); + + SavepointRestoreSettings savepointRestoreSettings = jobGraph.getSavepointRestoreSettings(); + assertThat(savepointRestoreSettings.getRestorePath(), is("hello")); + } + /** * Verifies that the chain start/end is correctly set. */ diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java index 36fb86787267686aa36103695e0dd4ad000ed210..f9d9aa5e503ea4b5030678b8cb52a9ad7f351231 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/AbstractStreamOperatorTest.java @@ -558,7 +558,7 @@ public class AbstractStreamOperatorTest { new MemCheckpointStreamFactory(Integer.MAX_VALUE)); fail("Exception expected."); } catch (Exception e) { - assertEquals(failingException, e.getCause()); + assertEquals(failingException.getMessage(), e.getCause().getMessage()); } } @@ -636,7 +636,7 @@ public class AbstractStreamOperatorTest { new MemCheckpointStreamFactory(Integer.MAX_VALUE)); fail("Exception expected."); } catch (Exception e) { - assertEquals(failingException, e.getCause()); + assertEquals(failingException.getMessage(), e.getCause().getMessage()); } // verify that the context has been closed, the operator snapshot result has been cancelled diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperatorTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperatorTest.java index bb5e694433b8726b8a2243c0c98f102bc6b28a61..cfa6ce66f69a0862a950c75c4961f120ebc2d6db 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperatorTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/AsyncWaitOperatorTest.java @@ -26,7 +26,6 @@ import org.apache.flink.api.common.typeutils.base.IntSerializer; import org.apache.flink.api.java.Utils; import org.apache.flink.api.java.typeutils.TypeExtractor; import org.apache.flink.configuration.Configuration; -import org.apache.flink.core.testutils.CheckedThread; import org.apache.flink.core.testutils.OneShotLatch; import org.apache.flink.runtime.checkpoint.CheckpointMetaData; import org.apache.flink.runtime.checkpoint.CheckpointOptions; @@ -49,9 +48,8 @@ import org.apache.flink.streaming.api.functions.sink.DiscardingSink; import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue; -import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueueEntry; -import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamElement; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.tasks.OneInputStreamTask; import org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTestHarness; @@ -63,13 +61,16 @@ import org.apache.flink.util.TestLogger; import org.hamcrest.Matchers; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.Timeout; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.Queue; @@ -100,6 +101,9 @@ import static org.junit.Assert.assertTrue; public class AsyncWaitOperatorTest extends TestLogger { private static final long TIMEOUT = 1000L; + @Rule + public Timeout timeoutRule = new Timeout(10, TimeUnit.SECONDS); + private static class MyAsyncFunction extends RichAsyncFunction { private static final long serialVersionUID = 8522411971886428444L; @@ -165,7 +169,7 @@ public class AsyncWaitOperatorTest extends TestLogger { * {@link ResultFuture#complete} until the latch counts to zero. * {@link ResultFuture#complete} until the latch counts to zero. * This function is used in the testStateSnapshotAndRestore, ensuring - * that {@link StreamElementQueueEntry} can stay + * that {@link StreamElement} can stay * in the {@link StreamElementQueue} to be * snapshotted while checkpointing. */ @@ -469,7 +473,7 @@ public class AsyncWaitOperatorTest extends TestLogger { testHarness.endInput(); testHarness.waitForTaskCompletion(); - ConcurrentLinkedQueue expectedOutput = new ConcurrentLinkedQueue<>(); + List expectedOutput = new LinkedList<>(); expectedOutput.add(new StreamRecord<>(22, initialTimestamp)); expectedOutput.add(new StreamRecord<>(26, initialTimestamp + 1L)); expectedOutput.add(new StreamRecord<>(30, initialTimestamp + 2L)); @@ -734,7 +738,7 @@ public class AsyncWaitOperatorTest extends TestLogger { *

    Note that this test does not enforce the exact strict ordering because with the fix it is no * longer possible. However, it provokes the described situation without the fix. */ - @Test(timeout = 10000L) + @Test public void testClosingWithBlockedEmitter() throws Exception { JobVertex chainedVertex = createChainedVertex(new MyAsyncFunction(), new EmitterBlockingFunction()); @@ -770,7 +774,7 @@ public class AsyncWaitOperatorTest extends TestLogger { /** * FLINK-5652 * Tests that registered timers are properly canceled upon completion of a - * {@link StreamRecordQueueEntry} in order to avoid resource leaks because TriggerTasks hold + * {@link StreamElement} in order to avoid resource leaks because TriggerTasks hold * a reference on the StreamRecordQueueEntry. */ @Test @@ -801,7 +805,7 @@ public class AsyncWaitOperatorTest extends TestLogger { *

    Tests that a user exception triggers the completion of a StreamElementQueueEntry and does not wait to until * another StreamElementQueueEntry is properly completed before it is collected. */ - @Test(timeout = 2000) + @Test public void testOrderedWaitUserExceptionHandling() throws Exception { testUserExceptionHandling(AsyncDataStream.OutputMode.ORDERED); } @@ -812,7 +816,7 @@ public class AsyncWaitOperatorTest extends TestLogger { *

    Tests that a user exception triggers the completion of a StreamElementQueueEntry and does not wait to until * another StreamElementQueueEntry is properly completed before it is collected. */ - @Test(timeout = 2000) + @Test public void testUnorderedWaitUserExceptionHandling() throws Exception { testUserExceptionHandling(AsyncDataStream.OutputMode.UNORDERED); } @@ -910,7 +914,7 @@ public class AsyncWaitOperatorTest extends TestLogger { final OperatorSubtaskState snapshot; - final ArrayList expectedOutput = new ArrayList<>(capacity + 1); + final ArrayList expectedOutput = new ArrayList<>(capacity); try { synchronized (snapshotHarness.getCheckpointLock()) { @@ -920,24 +924,6 @@ public class AsyncWaitOperatorTest extends TestLogger { } } - expectedOutput.add(capacity); - - final OneShotLatch lastElement = new OneShotLatch(); - - final CheckedThread lastElementWriter = new CheckedThread() { - @Override - public void go() throws Exception { - synchronized (snapshotHarness.getCheckpointLock()) { - lastElement.trigger(); - snapshotHarness.processElement(capacity, 0L); - } - } - }; - - lastElementWriter.start(); - - lastElement.await(); - synchronized (snapshotHarness.getCheckpointLock()) { // execute the snapshot within the checkpoint lock, because then it is guaranteed // that the lastElementWriter has written the exceeding element diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/EmitterTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/EmitterTest.java deleted file mode 100644 index 8b18cc3651dea4b382b4e473d4dd3f18683406cc..0000000000000000000000000000000000000000 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/EmitterTest.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.streaming.api.operators.async; - -import org.apache.flink.streaming.api.functions.async.ResultFuture; -import org.apache.flink.streaming.api.operators.Output; -import org.apache.flink.streaming.api.operators.async.queue.OrderedStreamElementQueue; -import org.apache.flink.streaming.api.operators.async.queue.StreamElementQueue; -import org.apache.flink.streaming.api.operators.async.queue.StreamRecordQueueEntry; -import org.apache.flink.streaming.api.operators.async.queue.WatermarkQueueEntry; -import org.apache.flink.streaming.api.watermark.Watermark; -import org.apache.flink.streaming.runtime.streamrecord.StreamElement; -import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; -import org.apache.flink.streaming.runtime.tasks.mailbox.execution.MailboxExecutor; -import org.apache.flink.streaming.util.CollectorOutput; -import org.apache.flink.util.TestLogger; - -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.mockito.ArgumentCaptor; - -import javax.annotation.Nonnull; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -/** - * Tests for {@link Emitter}. - */ -public class EmitterTest extends TestLogger { - - private static final long timeout = 10000L; - private static ExecutorService executor; - - @BeforeClass - public static void setup() { - executor = Executors.newFixedThreadPool(3); - } - - @AfterClass - public static void shutdown() { - executor.shutdown(); - - try { - if (!executor.awaitTermination(timeout, TimeUnit.MILLISECONDS)) { - executor.shutdownNow(); - } - } catch (InterruptedException interrupted) { - executor.shutdownNow(); - - Thread.currentThread().interrupt(); - } - } - - /** - * Tests that the emitter outputs completed stream element queue entries. - */ - @Test - public void testEmitterWithOrderedQueue() throws Exception { - List list = new ArrayList<>(); - Output> output = new CollectorOutput<>(list); - - List expected = Arrays.asList( - new StreamRecord<>(1, 0L), - new StreamRecord<>(2, 0L), - new StreamRecord<>(3, 1L), - new StreamRecord<>(4, 1L), - new Watermark(3L), - new StreamRecord<>(5, 4L), - new StreamRecord<>(6, 4L)); - - OperatorActions operatorActions = mock(OperatorActions.class); - - final int capacity = 5; - - StreamElementQueue queue = new OrderedStreamElementQueue(capacity, executor, operatorActions); - - final Emitter emitter = new Emitter<>( - new Object(), - new MockExecutor(), - output, - queue, - operatorActions); - - final Thread emitterThread = new Thread(emitter); - emitterThread.start(); - - try { - StreamRecordQueueEntry record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L)); - StreamRecordQueueEntry record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L)); - WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L)); - StreamRecordQueueEntry record3 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 4L)); - - queue.put(record1); - queue.put(record2); - queue.put(watermark1); - queue.put(record3); - - record2.complete(Arrays.asList(3, 4)); - record1.complete(Arrays.asList(1, 2)); - record3.complete(Arrays.asList(5, 6)); - - while (!queue.isEmpty()) { - Thread.sleep(10); - } - - Assert.assertEquals(expected, list); - } finally { - emitter.stop(); - emitterThread.interrupt(); - } - } - - /** - * Tests that the emitter handles exceptions occurring in the {@link ResultFuture} correctly. - */ - @Test - public void testEmitterWithExceptions() throws Exception { - Object lock = new Object(); - List list = new ArrayList<>(); - Output> output = new CollectorOutput<>(list); - - List expected = Arrays.asList( - new StreamRecord<>(1, 0L), - new Watermark(3L)); - - OperatorActions operatorActions = mock(OperatorActions.class); - - final int capacity = 3; - - StreamElementQueue queue = new OrderedStreamElementQueue(capacity, executor, operatorActions); - - final Emitter emitter = new Emitter<>( - lock, - new MockExecutor(), - output, - queue, - operatorActions); - - final Thread emitterThread = new Thread(emitter); - emitterThread.start(); - - final Exception testException = new Exception("Test exception"); - - try { - StreamRecordQueueEntry record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L)); - StreamRecordQueueEntry record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L)); - WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(3L)); - - queue.put(record1); - queue.put(record2); - queue.put(watermark1); - - record2.completeExceptionally(testException); - record1.complete(Arrays.asList(1)); - - synchronized (lock) { - while (!queue.isEmpty()) { - lock.wait(1); - } - } - - Assert.assertEquals(expected, list); - - ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(Throwable.class); - - verify(operatorActions).failOperator(argumentCaptor.capture()); - - Throwable failureCause = argumentCaptor.getValue(); - - Assert.assertNotNull(failureCause.getCause()); - Assert.assertTrue(failureCause.getCause() instanceof ExecutionException); - - Assert.assertNotNull(failureCause.getCause().getCause()); - Assert.assertEquals(testException, failureCause.getCause().getCause()); - } finally { - emitter.stop(); - emitterThread.interrupt(); - } - } - - private static class MockExecutor implements MailboxExecutor { - @Override - public void execute(@Nonnull Runnable command) throws RejectedExecutionException { - executor.execute(command); - } - - @Override - public void yield() throws InterruptedException, IllegalStateException { - throw new UnsupportedOperationException(); - } - - @Override - public boolean tryYield() throws IllegalStateException { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isMailboxThread() { - return true; - } - } -} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueueTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueueTest.java index 1bc3c20335cc5e5880d3d2c82df5ea55c22cb0bb..f637c40e7da438118712c08a7993869de6306c5d 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueueTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/OrderedStreamElementQueueTest.java @@ -18,110 +18,59 @@ package org.apache.flink.streaming.api.operators.async.queue; -import org.apache.flink.streaming.api.operators.async.OperatorActions; +import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.streamrecord.StreamElement; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.TestLogger; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; +import static org.apache.flink.streaming.api.operators.async.queue.QueueUtil.popCompleted; +import static org.apache.flink.streaming.api.operators.async.queue.QueueUtil.putSuccessfully; /** * {@link OrderedStreamElementQueue} specific tests. */ public class OrderedStreamElementQueueTest extends TestLogger { - - private static final long timeout = 10000L; - private static ExecutorService executor; - - @BeforeClass - public static void setup() { - executor = Executors.newFixedThreadPool(3); - } - - @AfterClass - public static void shutdown() { - executor.shutdown(); - - try { - if (!executor.awaitTermination(timeout, TimeUnit.MILLISECONDS)) { - executor.shutdownNow(); - } - } catch (InterruptedException interrupted) { - executor.shutdownNow(); - - Thread.currentThread().interrupt(); - } - } - /** * Tests that only the head element is pulled from the ordered queue if it has been * completed. */ @Test - public void testCompletionOrder() throws Exception { - OperatorActions operatorActions = mock(OperatorActions.class); - final OrderedStreamElementQueue queue = new OrderedStreamElementQueue(4, executor, operatorActions); - - StreamRecordQueueEntry entry1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L)); - StreamRecordQueueEntry entry2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L)); - WatermarkQueueEntry entry3 = new WatermarkQueueEntry(new Watermark(2L)); - StreamRecordQueueEntry entry4 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 3L)); - - List> expected = Arrays.asList(entry1, entry2, entry3, entry4); + public void testCompletionOrder() { + final OrderedStreamElementQueue queue = new OrderedStreamElementQueue<>(4); - for (StreamElementQueueEntry entry : expected) { - queue.put(entry); - } - - CompletableFuture> pollOperation = CompletableFuture.supplyAsync( - () -> { - List result = new ArrayList<>(4); - while (!queue.isEmpty()) { - try { - result.add(queue.poll()); - } catch (InterruptedException e) { - throw new CompletionException(e); - } - } - - return result; - }, - executor); - - Thread.sleep(10L); - - Assert.assertFalse(pollOperation.isDone()); - - entry2.complete(Collections.emptyList()); - - entry4.complete(Collections.emptyList()); - - Thread.sleep(10L); + ResultFuture entry1 = putSuccessfully(queue, new StreamRecord<>(1, 0L)); + ResultFuture entry2 = putSuccessfully(queue, new StreamRecord<>(2, 1L)); + putSuccessfully(queue, new Watermark(2L)); + ResultFuture entry4 = putSuccessfully(queue, new StreamRecord<>(3, 3L)); + Assert.assertEquals(Collections.emptyList(), popCompleted(queue)); Assert.assertEquals(4, queue.size()); + Assert.assertFalse(queue.isEmpty()); - entry1.complete(Collections.emptyList()); + entry2.complete(Collections.singleton(11)); + entry4.complete(Collections.singleton(13)); - Assert.assertEquals(expected, pollOperation.get()); - - verify(operatorActions, never()).failOperator(any(Exception.class)); + Assert.assertEquals(Collections.emptyList(), popCompleted(queue)); + Assert.assertEquals(4, queue.size()); + Assert.assertFalse(queue.isEmpty()); + + entry1.complete(Collections.singleton(10)); + + List expected = Arrays.asList( + new StreamRecord<>(10, 0L), + new StreamRecord<>(11, 1L), + new Watermark(2L), + new StreamRecord<>(13, 3L)); + Assert.assertEquals(expected, popCompleted(queue)); + Assert.assertEquals(0, queue.size()); + Assert.assertTrue(queue.isEmpty()); } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/QueueUtil.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/QueueUtil.java new file mode 100644 index 0000000000000000000000000000000000000000..ef919a386ce12e4bcaf775bdd4ca210586ab23e2 --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/QueueUtil.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.api.operators.async.queue; + +import org.apache.flink.streaming.api.functions.async.ResultFuture; +import org.apache.flink.streaming.api.operators.TimestampedCollector; +import org.apache.flink.streaming.runtime.streamrecord.StreamElement; +import org.apache.flink.streaming.util.CollectorOutput; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Utility for putting elements inside a {@link StreamElementQueue}. + */ +class QueueUtil { + static ResultFuture putSuccessfully(StreamElementQueue queue, StreamElement streamElement) { + Optional> resultFuture = queue.tryPut(streamElement); + assertTrue(resultFuture.isPresent()); + return resultFuture.get(); + } + + static void putUnsuccessfully(StreamElementQueue queue, StreamElement streamElement) { + Optional> resultFuture = queue.tryPut(streamElement); + assertFalse(resultFuture.isPresent()); + } + + /** + * Pops all completed elements from the head of this queue. + * + * @return Completed elements or empty list if none exists. + */ + static List popCompleted(StreamElementQueue queue) { + final List completed = new ArrayList<>(); + TimestampedCollector collector = new TimestampedCollector<>(new CollectorOutput<>(completed)); + while (queue.hasCompletedElements()) { + queue.emitCompletedElement(collector); + } + collector.close(); + return completed; + } + +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueTest.java index 1c286fa20afd1b0a39ee4166cdf903938b8cf9a5..1b58c3e667e00fe53bae7dac3bdabba0be2721b6 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/StreamElementQueueTest.java @@ -18,15 +18,14 @@ package org.apache.flink.streaming.api.operators.async.queue; -import org.apache.flink.streaming.api.operators.async.OperatorActions; +import org.apache.flink.streaming.api.datastream.AsyncDataStream; +import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.Preconditions; import org.apache.flink.util.TestLogger; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -34,19 +33,13 @@ import org.junit.runners.Parameterized; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import static org.apache.flink.streaming.api.operators.async.queue.StreamElementQueueTest.StreamElementQueueType.OrderedStreamElementQueueType; -import static org.apache.flink.streaming.api.operators.async.queue.StreamElementQueueTest.StreamElementQueueType.UnorderedStreamElementQueueType; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; +import static org.apache.flink.streaming.api.operators.async.queue.QueueUtil.popCompleted; +import static org.apache.flink.streaming.api.operators.async.queue.QueueUtil.putSuccessfully; +import static org.apache.flink.streaming.api.operators.async.queue.QueueUtil.putUnsuccessfully; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * Tests for the basic functionality of {@link StreamElementQueue}. The basic operations consist @@ -54,217 +47,111 @@ import static org.mockito.Mockito.verify; */ @RunWith(Parameterized.class) public class StreamElementQueueTest extends TestLogger { - - private static final long timeout = 10000L; - private static ExecutorService executor; - - @BeforeClass - public static void setup() { - executor = Executors.newFixedThreadPool(3); - } - - @AfterClass - public static void shutdown() { - executor.shutdown(); - - try { - if (!executor.awaitTermination(timeout, TimeUnit.MILLISECONDS)) { - executor.shutdownNow(); - } - } catch (InterruptedException interrupted) { - executor.shutdownNow(); - - Thread.currentThread().interrupt(); - } - } - - enum StreamElementQueueType { - OrderedStreamElementQueueType, - UnorderedStreamElementQueueType - } - @Parameterized.Parameters - public static Collection streamElementQueueTypes() { - return Arrays.asList(OrderedStreamElementQueueType, UnorderedStreamElementQueueType); + public static Collection outputModes() { + return Arrays.asList(AsyncDataStream.OutputMode.ORDERED, AsyncDataStream.OutputMode.UNORDERED); } - private final StreamElementQueueType streamElementQueueType; + private final AsyncDataStream.OutputMode outputMode; - public StreamElementQueueTest(StreamElementQueueType streamElementQueueType) { - this.streamElementQueueType = Preconditions.checkNotNull(streamElementQueueType); + public StreamElementQueueTest(AsyncDataStream.OutputMode outputMode) { + this.outputMode = Preconditions.checkNotNull(outputMode); } - public StreamElementQueue createStreamElementQueue(int capacity, OperatorActions operatorActions) { - switch (streamElementQueueType) { - case OrderedStreamElementQueueType: - return new OrderedStreamElementQueue(capacity, executor, operatorActions); - case UnorderedStreamElementQueueType: - return new UnorderedStreamElementQueue(capacity, executor, operatorActions); + private StreamElementQueue createStreamElementQueue(int capacity) { + switch (outputMode) { + case ORDERED: + return new OrderedStreamElementQueue<>(capacity); + case UNORDERED: + return new UnorderedStreamElementQueue<>(capacity); default: - throw new IllegalStateException("Unknown stream element queue type: " + streamElementQueueType); + throw new IllegalStateException("Unknown output mode: " + outputMode); } } @Test - public void testPut() throws InterruptedException { - OperatorActions operatorActions = mock(OperatorActions.class); - StreamElementQueue queue = createStreamElementQueue(2, operatorActions); - - final Watermark watermark = new Watermark(0L); - final StreamRecord streamRecord = new StreamRecord<>(42, 1L); - final Watermark nextWatermark = new Watermark(2L); + public void testPut() { + StreamElementQueue queue = createStreamElementQueue(2); - final WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(watermark); - final StreamRecordQueueEntry streamRecordQueueEntry = new StreamRecordQueueEntry<>(streamRecord); + Watermark watermark = new Watermark(0L); + StreamRecord streamRecord = new StreamRecord<>(42, 1L); - queue.put(watermarkQueueEntry); - queue.put(streamRecordQueueEntry); + // add two elements to reach capacity + assertTrue(queue.tryPut(watermark).isPresent()); + assertTrue(queue.tryPut(streamRecord).isPresent()); - Assert.assertEquals(2, queue.size()); - - Assert.assertFalse(queue.tryPut(new WatermarkQueueEntry(nextWatermark))); - - Collection> actualValues = queue.values(); + assertEquals(2, queue.size()); - List> expectedValues = Arrays.asList(watermarkQueueEntry, streamRecordQueueEntry); + // queue full, cannot add new element + assertFalse(queue.tryPut(new Watermark(2L)).isPresent()); - Assert.assertEquals(expectedValues, actualValues); - - verify(operatorActions, never()).failOperator(any(Exception.class)); + // check if expected values are returned (for checkpointing) + assertEquals(Arrays.asList(watermark, streamRecord), queue.values()); } @Test - public void testPoll() throws InterruptedException { - OperatorActions operatorActions = mock(OperatorActions.class); - StreamElementQueue queue = createStreamElementQueue(2, operatorActions); - - WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(new Watermark(0L)); - StreamRecordQueueEntry streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(42, 1L)); - - queue.put(watermarkQueueEntry); - queue.put(streamRecordQueueEntry); + public void testPop() { + StreamElementQueue queue = createStreamElementQueue(2); - Assert.assertEquals(watermarkQueueEntry, queue.peekBlockingly()); - Assert.assertEquals(2, queue.size()); - - Assert.assertEquals(watermarkQueueEntry, queue.poll()); - Assert.assertEquals(1, queue.size()); + // add two elements to reach capacity + putSuccessfully(queue, new Watermark(0L)); + ResultFuture recordResult = putSuccessfully(queue, new StreamRecord<>(42, 1L)); - streamRecordQueueEntry.complete(Collections.emptyList()); + assertEquals(2, queue.size()); - Assert.assertEquals(streamRecordQueueEntry, queue.poll()); + // remove completed elements (watermarks are always completed) + assertEquals(Arrays.asList(new Watermark(0L)), popCompleted(queue)); + assertEquals(1, queue.size()); - Assert.assertEquals(0, queue.size()); - Assert.assertTrue(queue.isEmpty()); + // now complete the stream record + recordResult.complete(Collections.singleton(43)); - verify(operatorActions, never()).failOperator(any(Exception.class)); + assertEquals(Arrays.asList(new StreamRecord<>(43, 1L)), popCompleted(queue)); + assertEquals(0, queue.size()); + assertTrue(queue.isEmpty()); } /** - * Tests that a put operation blocks if the queue is full. + * Tests that a put operation fails if the queue is full. */ @Test - public void testBlockingPut() throws Exception { - OperatorActions operatorActions = mock(OperatorActions.class); - final StreamElementQueue queue = createStreamElementQueue(1, operatorActions); - - StreamRecordQueueEntry streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(42, 0L)); - final StreamRecordQueueEntry streamRecordQueueEntry2 = new StreamRecordQueueEntry<>(new StreamRecord<>(43, 1L)); - - queue.put(streamRecordQueueEntry); + public void testPutOnFull() throws Exception { + final StreamElementQueue queue = createStreamElementQueue(1); - Assert.assertEquals(1, queue.size()); + // fill up queue + ResultFuture resultFuture = putSuccessfully(queue, new StreamRecord<>(42, 0L)); + assertEquals(1, queue.size()); - CompletableFuture putOperation = CompletableFuture.runAsync( - () -> { - try { - queue.put(streamRecordQueueEntry2); - } catch (InterruptedException e) { - throw new CompletionException(e); - } - }, - executor); + // cannot add more + putUnsuccessfully(queue, new StreamRecord<>(43, 1L)); - // give the future a chance to complete - Thread.sleep(10L); - - // but it shouldn't ;-) - Assert.assertFalse(putOperation.isDone()); - - streamRecordQueueEntry.complete(Collections.emptyList()); - - // polling the completed head element frees the queue again - Assert.assertEquals(streamRecordQueueEntry, queue.poll()); + // popping the completed element frees the queue again + resultFuture.complete(Collections.singleton(42 * 42)); + assertEquals(Arrays.asList(new StreamRecord(42 * 42, 0L)), popCompleted(queue)); // now the put operation should complete - putOperation.get(); - - verify(operatorActions, never()).failOperator(any(Exception.class)); + putSuccessfully(queue, new StreamRecord<>(43, 1L)); } /** - * Test that a poll operation on an empty queue blocks. + * Tests two adjacent watermarks can be processed successfully. */ @Test - public void testBlockingPoll() throws Exception { - OperatorActions operatorActions = mock(OperatorActions.class); - final StreamElementQueue queue = createStreamElementQueue(1, operatorActions); - - WatermarkQueueEntry watermarkQueueEntry = new WatermarkQueueEntry(new Watermark(1L)); - StreamRecordQueueEntry streamRecordQueueEntry = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 2L)); - - Assert.assertTrue(queue.isEmpty()); - - CompletableFuture peekOperation = CompletableFuture.supplyAsync( - () -> { - try { - return queue.peekBlockingly(); - } catch (InterruptedException e) { - throw new CompletionException(e); - } - }, - executor); - - Thread.sleep(10L); - - Assert.assertFalse(peekOperation.isDone()); - - queue.put(watermarkQueueEntry); - - AsyncResult watermarkResult = peekOperation.get(); + public void testWatermarkOnly() { + final StreamElementQueue queue = createStreamElementQueue(2); - Assert.assertEquals(watermarkQueueEntry, watermarkResult); - Assert.assertEquals(1, queue.size()); + putSuccessfully(queue, new Watermark(2L)); + putSuccessfully(queue, new Watermark(5L)); - Assert.assertEquals(watermarkQueueEntry, queue.poll()); - Assert.assertTrue(queue.isEmpty()); - - CompletableFuture pollOperation = CompletableFuture.supplyAsync( - () -> { - try { - return queue.poll(); - } catch (InterruptedException e) { - throw new CompletionException(e); - } - }, - executor); - - Thread.sleep(10L); - - Assert.assertFalse(pollOperation.isDone()); - - queue.put(streamRecordQueueEntry); - - Thread.sleep(10L); - - Assert.assertFalse(pollOperation.isDone()); - - streamRecordQueueEntry.complete(Collections.emptyList()); - - Assert.assertEquals(streamRecordQueueEntry, pollOperation.get()); + Assert.assertEquals(2, queue.size()); + Assert.assertFalse(queue.isEmpty()); + Assert.assertEquals(Arrays.asList( + new Watermark(2L), + new Watermark(5L)), + popCompleted(queue)); + Assert.assertEquals(0, queue.size()); Assert.assertTrue(queue.isEmpty()); - - verify(operatorActions, never()).failOperator(any(Exception.class)); + Assert.assertEquals(Collections.emptyList(), popCompleted(queue)); } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueueTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueueTest.java index 73bdcc5b156199f57214b99cdd0dbfa84766c2e0..0cbded622a4405ee283c9ce385de799b2397d554 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueueTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/api/operators/async/queue/UnorderedStreamElementQueueTest.java @@ -18,174 +18,85 @@ package org.apache.flink.streaming.api.operators.async.queue; -import org.apache.flink.streaming.api.operators.async.OperatorActions; +import org.apache.flink.streaming.api.functions.async.ResultFuture; import org.apache.flink.streaming.api.watermark.Watermark; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.util.TestLogger; -import org.junit.AfterClass; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; import java.util.Arrays; import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; + +import static org.apache.flink.streaming.api.operators.async.queue.QueueUtil.popCompleted; +import static org.apache.flink.streaming.api.operators.async.queue.QueueUtil.putSuccessfully; /** * {@link UnorderedStreamElementQueue} specific tests. */ public class UnorderedStreamElementQueueTest extends TestLogger { - private static final long timeout = 10000L; - private static ExecutorService executor; - - @BeforeClass - public static void setup() { - executor = Executors.newFixedThreadPool(3); - } - - @AfterClass - public static void shutdown() { - executor.shutdown(); - - try { - if (!executor.awaitTermination(timeout, TimeUnit.MILLISECONDS)) { - executor.shutdownNow(); - } - } catch (InterruptedException interrupted) { - executor.shutdownNow(); - - Thread.currentThread().interrupt(); - } - } - /** * Tests that only elements before the oldest watermark are returned if they are completed. */ @Test - public void testCompletionOrder() throws Exception { - OperatorActions operatorActions = mock(OperatorActions.class); - - final UnorderedStreamElementQueue queue = new UnorderedStreamElementQueue(8, executor, operatorActions); - - StreamRecordQueueEntry record1 = new StreamRecordQueueEntry<>(new StreamRecord<>(1, 0L)); - StreamRecordQueueEntry record2 = new StreamRecordQueueEntry<>(new StreamRecord<>(2, 1L)); - WatermarkQueueEntry watermark1 = new WatermarkQueueEntry(new Watermark(2L)); - StreamRecordQueueEntry record3 = new StreamRecordQueueEntry<>(new StreamRecord<>(3, 3L)); - StreamRecordQueueEntry record4 = new StreamRecordQueueEntry<>(new StreamRecord<>(4, 4L)); - WatermarkQueueEntry watermark2 = new WatermarkQueueEntry(new Watermark(5L)); - StreamRecordQueueEntry record5 = new StreamRecordQueueEntry<>(new StreamRecord<>(5, 6L)); - StreamRecordQueueEntry record6 = new StreamRecordQueueEntry<>(new StreamRecord<>(6, 7L)); - - List> entries = Arrays.asList(record1, record2, watermark1, record3, - record4, watermark2, record5, record6); - - // The queue should look like R1, R2, W1, R3, R4, W2, R5, R6 - for (StreamElementQueueEntry entry : entries) { - queue.put(entry); - } - - Assert.assertTrue(8 == queue.size()); - - CompletableFuture firstPoll = CompletableFuture.supplyAsync( - () -> { - try { - return queue.poll(); - } catch (InterruptedException e) { - throw new CompletionException(e); - } - }, - executor); - - // this should not fulfill the poll, because R3 is behind W1 - record3.complete(Collections.emptyList()); + public void testCompletionOrder() { + final UnorderedStreamElementQueue queue = new UnorderedStreamElementQueue<>(8); - Thread.sleep(10L); + ResultFuture record1 = putSuccessfully(queue, new StreamRecord<>(1, 0L)); + ResultFuture record2 = putSuccessfully(queue, new StreamRecord<>(2, 1L)); + putSuccessfully(queue, new Watermark(2L)); + ResultFuture record3 = putSuccessfully(queue, new StreamRecord<>(3, 3L)); + ResultFuture record4 = putSuccessfully(queue, new StreamRecord<>(4, 4L)); + putSuccessfully(queue, new Watermark(5L)); + ResultFuture record5 = putSuccessfully(queue, new StreamRecord<>(5, 6L)); + ResultFuture record6 = putSuccessfully(queue, new StreamRecord<>(6, 7L)); - Assert.assertFalse(firstPoll.isDone()); + Assert.assertEquals(Collections.emptyList(), popCompleted(queue)); + Assert.assertEquals(8, queue.size()); + Assert.assertFalse(queue.isEmpty()); - record2.complete(Collections.emptyList()); + // this should not make any item completed, because R3 is behind W1 + record3.complete(Arrays.asList(13)); - Assert.assertEquals(record2, firstPoll.get()); + Assert.assertEquals(Collections.emptyList(), popCompleted(queue)); + Assert.assertEquals(8, queue.size()); + Assert.assertFalse(queue.isEmpty()); - CompletableFuture secondPoll = CompletableFuture.supplyAsync( - () -> { - try { - return queue.poll(); - } catch (InterruptedException e) { - throw new CompletionException(e); - } - }, - executor); + record2.complete(Arrays.asList(12)); - record6.complete(Collections.emptyList()); - record4.complete(Collections.emptyList()); + Assert.assertEquals(Arrays.asList(new StreamRecord<>(12, 1L)), popCompleted(queue)); + Assert.assertEquals(7, queue.size()); + Assert.assertFalse(queue.isEmpty()); - Thread.sleep(10L); + // Should not be completed because R1 has not been completed yet + record6.complete(Arrays.asList(16)); + record4.complete(Arrays.asList(14)); - // The future should not be completed because R1 has not been completed yet - Assert.assertFalse(secondPoll.isDone()); - - record1.complete(Collections.emptyList()); - - Assert.assertEquals(record1, secondPoll.get()); + Assert.assertEquals(Collections.emptyList(), popCompleted(queue)); + Assert.assertEquals(7, queue.size()); + Assert.assertFalse(queue.isEmpty()); // Now W1, R3, R4 and W2 are completed and should be pollable - Assert.assertEquals(watermark1, queue.poll()); - - // The order of R3 and R4 is not specified - Set expected = new HashSet<>(2); - expected.add(record3); - expected.add(record4); - - Set actual = new HashSet<>(2); - - actual.add(queue.poll()); - actual.add(queue.poll()); - - Assert.assertEquals(expected, actual); - - Assert.assertEquals(watermark2, queue.poll()); - - // since R6 has been completed before and W2 has been consumed, we should be able to poll - // this record as well - Assert.assertEquals(record6, queue.poll()); + record1.complete(Arrays.asList(11)); + + Assert.assertEquals(Arrays.asList( + new StreamRecord<>(11, 0L), + new Watermark(2L), + new StreamRecord<>(13, 3L), + new StreamRecord<>(14, 4L), + new Watermark(5L), + new StreamRecord<>(16, 7L)), + popCompleted(queue)); + Assert.assertEquals(1, queue.size()); + Assert.assertFalse(queue.isEmpty()); // only R5 left in the queue - Assert.assertTrue(1 == queue.size()); - - CompletableFuture thirdPoll = CompletableFuture.supplyAsync( - () -> { - try { - return queue.poll(); - } catch (InterruptedException e) { - throw new CompletionException(e); - } - }, - executor); - - Thread.sleep(10L); - - Assert.assertFalse(thirdPoll.isDone()); - - record5.complete(Collections.emptyList()); - - Assert.assertEquals(record5, thirdPoll.get()); + record5.complete(Arrays.asList(15)); + Assert.assertEquals(Arrays.asList(new StreamRecord<>(15, 6L)), popCompleted(queue)); + Assert.assertEquals(0, queue.size()); Assert.assertTrue(queue.isEmpty()); - - verify(operatorActions, never()).failOperator(any(Exception.class)); + Assert.assertEquals(Collections.emptyList(), popCompleted(queue)); } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInputTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInputTest.java index c5301b2f56c36e11b87d2ec40b7ae3debd0a6023..d4ba08ff93da7a1614034edf59743e662c491cf7 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInputTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/StreamTaskNetworkInputTest.java @@ -25,16 +25,21 @@ import org.apache.flink.runtime.io.network.api.EndOfPartitionEvent; import org.apache.flink.runtime.io.network.api.serialization.RecordSerializer; import org.apache.flink.runtime.io.network.api.serialization.SpanningRecordSerializer; import org.apache.flink.runtime.io.network.api.serialization.SpillingAdaptiveSpanningRecordDeserializer; -import org.apache.flink.runtime.io.network.buffer.Buffer; import org.apache.flink.runtime.io.network.buffer.BufferBuilder; import org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils; +import org.apache.flink.runtime.io.network.buffer.BufferConsumer; import org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent; import org.apache.flink.runtime.io.network.partition.consumer.StreamTestSingleInputGate; import org.apache.flink.runtime.plugable.DeserializationDelegate; import org.apache.flink.runtime.plugable.SerializationDelegate; +import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput.DataOutput; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; import org.apache.flink.streaming.runtime.streamrecord.StreamElement; import org.apache.flink.streaming.runtime.streamrecord.StreamElementSerializer; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.runtime.streamstatus.StatusWatermarkValve; +import org.apache.flink.streaming.runtime.streamstatus.StreamStatus; import org.junit.After; import org.junit.Test; @@ -44,9 +49,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; /** @@ -66,25 +74,27 @@ public class StreamTaskNetworkInputTest { @Test public void testIsAvailableWithBufferedDataInDeserializer() throws Exception { BufferBuilder bufferBuilder = BufferBuilderTestUtils.createEmptyBufferBuilder(PAGE_SIZE); + BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer(); serializeRecord(42L, bufferBuilder); serializeRecord(44L, bufferBuilder); - Buffer buffer = bufferBuilder.createBufferConsumer().build(); + List buffers = Collections.singletonList(new BufferOrEvent(bufferConsumer.build(), 0, false)); - List buffers = Collections.singletonList(new BufferOrEvent(buffer, 0, false)); - - StreamTaskNetworkInput input = new StreamTaskNetworkInput( + VerifyRecordsDataOutput output = new VerifyRecordsDataOutput<>(); + StreamTaskNetworkInput input = new StreamTaskNetworkInput<>( new CheckpointedInputGate( new MockInputGate(1, buffers, false), new EmptyBufferStorage(), new CheckpointBarrierTracker(1)), LongSerializer.INSTANCE, ioManager, + new StatusWatermarkValve(1, output), 0); - assertHasNextElement(input); - assertHasNextElement(input); + assertHasNextElement(input, output); + assertHasNextElement(input, output); + assertEquals(2, output.getNumberOfEmittedRecords()); } @Test @@ -101,20 +111,21 @@ public class StreamTaskNetworkInputTest { } TestRecordDeserializer[] copiedDeserializers = Arrays.copyOf(deserializers, deserializers.length); - StreamTaskNetworkInput input = new StreamTaskNetworkInput( + DataOutput output = new NoOpDataOutput<>(); + StreamTaskNetworkInput input = new StreamTaskNetworkInput<>( new CheckpointedInputGate( inputGate.getInputGate(), new EmptyBufferStorage(), new CheckpointBarrierTracker(1)), inSerializer, - ioManager, + new StatusWatermarkValve(1, output), 0, deserializers); for (int i = 0; i < numInputChannels; i++) { assertNotNull(deserializers[i]); inputGate.sendEvent(EndOfPartitionEvent.INSTANCE, i); - input.pollNextNullable(); + input.emitNext(output); assertNull(deserializers[i]); assertTrue(copiedDeserializers[i].isCleared()); } @@ -131,11 +142,10 @@ public class StreamTaskNetworkInputTest { assertFalse(serializer.copyToBufferBuilder(bufferBuilder).isFullBuffer()); } - private static void assertHasNextElement(StreamTaskNetworkInput input) throws Exception { + private static void assertHasNextElement(StreamTaskNetworkInput input, DataOutput output) throws Exception { assertTrue(input.isAvailable().isDone()); - StreamElement element = input.pollNextNullable(); - assertNotNull(element); - assertTrue(element.isRecord()); + InputStatus status = input.emitNext(output); + assertThat(status, is(InputStatus.MORE_AVAILABLE)); } private static class TestRecordDeserializer @@ -156,4 +166,37 @@ public class StreamTaskNetworkInputTest { return cleared; } } + + private static class NoOpDataOutput implements DataOutput { + + @Override + public void emitRecord(StreamRecord record) { + } + + @Override + public void emitWatermark(Watermark watermark) { + } + + @Override + public void emitStreamStatus(StreamStatus streamStatus) { + } + + @Override + public void emitLatencyMarker(LatencyMarker latencyMarker) { + } + } + + private static class VerifyRecordsDataOutput extends NoOpDataOutput { + + private int numberOfEmittedRecords; + + @Override + public void emitRecord(StreamRecord record) { + numberOfEmittedRecords++; + } + + int getNumberOfEmittedRecords() { + return numberOfEmittedRecords; + } + } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/DataSkewStreamNetworkThroughputBenchmark.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/DataSkewStreamNetworkThroughputBenchmark.java new file mode 100644 index 0000000000000000000000000000000000000000..d97d3f4cec511b4f106011cea7d391aef222c22c --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/DataSkewStreamNetworkThroughputBenchmark.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.runtime.io.benchmark; + +import org.apache.flink.core.io.IOReadableWritable; +import org.apache.flink.runtime.io.network.api.writer.ChannelSelector; +import org.apache.flink.runtime.io.network.api.writer.RecordWriterBuilder; + +import static org.apache.flink.util.Preconditions.checkArgument; + +/** + * Network throughput benchmarks for data skew scenario executed by the external + * flink-benchmarks project. + */ +public class DataSkewStreamNetworkThroughputBenchmark extends StreamNetworkThroughputBenchmark { + + @Override + protected void setChannelSelector(RecordWriterBuilder recordWriterBuilder, boolean broadcastMode) { + checkArgument(!broadcastMode, "Combining broadcasting with data skew doesn't make sense"); + recordWriterBuilder.setChannelSelector(new DataSkewChannelSelector()); + } + + /** + * A {@link ChannelSelector} which selects channel 0 for nearly all records. And all other channels + * except for channel 0 will be only selected at most once. + */ + private static class DataSkewChannelSelector implements ChannelSelector { + private int numberOfChannels; + private int channelIndex = 0; + + @Override + public void setup(int numberOfChannels) { + this.numberOfChannels = numberOfChannels; + } + + @Override + public int selectChannel(IOReadableWritable record) { + if (channelIndex >= numberOfChannels) { + return 0; + } + return channelIndex++; + } + + @Override + public boolean isBroadcast() { + return false; + } + } +} diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/OperatorActions.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/DataSkewStreamNetworkThroughputBenchmarkTest.java similarity index 65% rename from flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/OperatorActions.java rename to flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/DataSkewStreamNetworkThroughputBenchmarkTest.java index 916b4128356b0b84324c8ab948eabd97ec7753f8..14c3d8f1bfe3fa540cb2b45c300e951c137abef3 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/OperatorActions.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/DataSkewStreamNetworkThroughputBenchmarkTest.java @@ -16,21 +16,14 @@ * limitations under the License. */ -package org.apache.flink.streaming.api.operators.async; - -import org.apache.flink.annotation.Internal; -import org.apache.flink.streaming.api.operators.StreamOperator; +package org.apache.flink.streaming.runtime.io.benchmark; /** - * Interface for {@link StreamOperator} actions. + * Tests for various network benchmarks based on {@link DataSkewStreamNetworkThroughputBenchmark}. */ -@Internal -public interface OperatorActions { - - /** - * Fail the respective stream operator with the given throwable. - * - * @param throwable to fail the stream operator with - */ - void failOperator(Throwable throwable); +public class DataSkewStreamNetworkThroughputBenchmarkTest extends StreamNetworkThroughputBenchmarkTest { + @Override + protected StreamNetworkThroughputBenchmark createBenchmark() { + return new DataSkewStreamNetworkThroughputBenchmark(); + } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkBenchmarkEnvironment.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkBenchmarkEnvironment.java index 8351b3ce7a9bb56194a5bd4eec8c93ed643a2370..aaeb1b0ef6abec483bddade6c1c901a7383af3a6 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkBenchmarkEnvironment.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkBenchmarkEnvironment.java @@ -27,8 +27,6 @@ import org.apache.flink.runtime.deployment.InputGateDeploymentDescriptor; import org.apache.flink.runtime.io.network.NettyShuffleEnvironment; import org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder; import org.apache.flink.runtime.io.network.TaskEventDispatcher; -import org.apache.flink.runtime.io.network.api.writer.RecordWriter; -import org.apache.flink.runtime.io.network.api.writer.RecordWriterBuilder; import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter; import org.apache.flink.runtime.io.network.netty.NettyConfig; import org.apache.flink.runtime.io.network.partition.InputChannelTestUtils; @@ -80,7 +78,6 @@ public class StreamNetworkBenchmarkEnvironment { protected NettyShuffleEnvironment receiverEnv; protected int channels; - protected boolean broadcastMode = false; protected boolean localMode = false; protected ResultPartitionID[] partitionIds; @@ -98,7 +95,6 @@ public class StreamNetworkBenchmarkEnvironment { setUp( writers, channels, - false, localMode, senderBufferPoolSize, receiverBufferPoolSize, @@ -122,12 +118,10 @@ public class StreamNetworkBenchmarkEnvironment { public void setUp( int writers, int channels, - boolean broadcastMode, boolean localMode, int senderBufferPoolSize, int receiverBufferPoolSize, Configuration config) throws Exception { - this.broadcastMode = broadcastMode; this.localMode = localMode; this.channels = channels; this.partitionIds = new ResultPartitionID[writers]; @@ -178,9 +172,25 @@ public class StreamNetworkBenchmarkEnvironment { return receiver; } - public RecordWriter createRecordWriter(int partitionIndex, long flushTimeout) throws Exception { - ResultPartitionWriter sender = createResultPartition(jobId, partitionIds[partitionIndex], senderEnv, channels); - return new RecordWriterBuilder().setTimeout(flushTimeout).build(sender); + public ResultPartitionWriter createResultPartitionWriter(int partitionIndex) throws Exception { + + ResultPartitionWriter resultPartitionWriter = new ResultPartitionBuilder() + .setResultPartitionId(partitionIds[partitionIndex]) + .setResultPartitionType(ResultPartitionType.PIPELINED_BOUNDED) + .setNumberOfSubpartitions(channels) + .setResultPartitionManager(senderEnv.getResultPartitionManager()) + .setupBufferPoolFactoryFromNettyShuffleEnvironment(senderEnv) + .build(); + + ResultPartitionWriter consumableNotifyingPartitionWriter = new ConsumableNotifyingResultPartitionWriterDecorator( + new NoOpTaskActions(), + jobId, + resultPartitionWriter, + new NoOpResultPartitionConsumableNotifier()); + + consumableNotifyingPartitionWriter.setup(); + + return consumableNotifyingPartitionWriter; } private void generatePartitionIds() throws Exception { @@ -205,31 +215,6 @@ public class StreamNetworkBenchmarkEnvironment { .build(); } - protected ResultPartitionWriter createResultPartition( - JobID jobId, - ResultPartitionID partitionId, - NettyShuffleEnvironment environment, - int channels) throws Exception { - - ResultPartitionWriter resultPartitionWriter = new ResultPartitionBuilder() - .setResultPartitionId(partitionId) - .setResultPartitionType(ResultPartitionType.PIPELINED_BOUNDED) - .setNumberOfSubpartitions(channels) - .setResultPartitionManager(environment.getResultPartitionManager()) - .setupBufferPoolFactoryFromNettyShuffleEnvironment(environment) - .build(); - - ResultPartitionWriter consumableNotifyingPartitionWriter = new ConsumableNotifyingResultPartitionWriterDecorator( - new NoOpTaskActions(), - jobId, - resultPartitionWriter, - new NoOpResultPartitionConsumableNotifier()); - - consumableNotifyingPartitionWriter.setup(); - - return consumableNotifyingPartitionWriter; - } - private InputGate createInputGate(TaskManagerLocation senderLocation) throws Exception { InputGate[] gates = new InputGate[channels]; for (int channel = 0; channel < channels; ++channel) { diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkPointToPointBenchmark.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkPointToPointBenchmark.java index bb6b9e2f16960f60ae9a889b30f254d07f322157..3277d50958b53b8e399e4ad7d863c5189ca69a36 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkPointToPointBenchmark.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkPointToPointBenchmark.java @@ -20,6 +20,8 @@ package org.apache.flink.streaming.runtime.io.benchmark; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.io.network.api.writer.RecordWriter; +import org.apache.flink.runtime.io.network.api.writer.RecordWriterBuilder; +import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter; import org.apache.flink.types.LongValue; import java.util.concurrent.CompletableFuture; @@ -75,9 +77,11 @@ public class StreamNetworkPointToPointBenchmark { */ public void setUp(long flushTimeout, Configuration config) throws Exception { environment = new StreamNetworkBenchmarkEnvironment<>(); - environment.setUp(1, 1, false, false, -1, -1, config); + environment.setUp(1, 1, false, -1, -1, config); - recordWriter = environment.createRecordWriter(0, flushTimeout); + ResultPartitionWriter resultPartitionWriter = environment.createResultPartitionWriter(0); + + recordWriter = new RecordWriterBuilder().setTimeout(flushTimeout).build(resultPartitionWriter); receiver = environment.createReceiver(); } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkThroughputBenchmark.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkThroughputBenchmark.java index b5d1c0729c12dcf3aab7accd2b8b9a44339923ed..a1afc8bb141a16ee829bfbb3e7f788b024213dfb 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkThroughputBenchmark.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/io/benchmark/StreamNetworkThroughputBenchmark.java @@ -19,6 +19,9 @@ package org.apache.flink.streaming.runtime.io.benchmark; import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.io.network.api.writer.RecordWriterBuilder; +import org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter; +import org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner; import org.apache.flink.types.LongValue; import java.util.concurrent.CompletableFuture; @@ -105,21 +108,29 @@ public class StreamNetworkThroughputBenchmark { environment.setUp( recordWriters, channels, - broadcastMode, localMode, senderBufferPoolSize, receiverBufferPoolSize, config); writerThreads = new LongRecordWriterThread[recordWriters]; for (int writer = 0; writer < recordWriters; writer++) { + ResultPartitionWriter resultPartitionWriter = environment.createResultPartitionWriter(writer); + RecordWriterBuilder recordWriterBuilder = new RecordWriterBuilder().setTimeout(flushTimeout); + setChannelSelector(recordWriterBuilder, broadcastMode); writerThreads[writer] = new LongRecordWriterThread( - environment.createRecordWriter(writer, flushTimeout), + recordWriterBuilder.build(resultPartitionWriter), broadcastMode); writerThreads[writer].start(); } receiver = environment.createReceiver(); } + protected void setChannelSelector(RecordWriterBuilder recordWriterBuilder, boolean broadcastMode) { + if (broadcastMode) { + recordWriterBuilder.setChannelSelector(new BroadcastPartitioner()); + } + } + /** * Shuts down a benchmark previously set up via {@link #setUp}. * diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValveTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValveTest.java index 4d0666b4c2eb2983c95cc217dd0369cb4276d517..ff20a16c9b9ec4137f97d95e6cc968c1531203e4 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValveTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/streamstatus/StatusWatermarkValveTest.java @@ -19,7 +19,10 @@ package org.apache.flink.streaming.runtime.streamstatus; import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.streaming.runtime.io.PushingAsyncDataInput; +import org.apache.flink.streaming.runtime.streamrecord.LatencyMarker; import org.apache.flink.streaming.runtime.streamrecord.StreamElement; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.junit.Test; @@ -49,7 +52,7 @@ public class StatusWatermarkValveTest { */ @Test public void testSingleInputIncreasingWatermarks() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(1, valveOutput); valve.inputWatermark(new Watermark(0), 0); @@ -66,7 +69,7 @@ public class StatusWatermarkValveTest { */ @Test public void testSingleInputDecreasingWatermarksYieldsNoOutput() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(1, valveOutput); valve.inputWatermark(new Watermark(25), 0); @@ -86,7 +89,7 @@ public class StatusWatermarkValveTest { */ @Test public void testSingleInputStreamStatusToggling() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(1, valveOutput); valve.inputStreamStatus(StreamStatus.ACTIVE, 0); @@ -109,7 +112,7 @@ public class StatusWatermarkValveTest { */ @Test public void testSingleInputWatermarksIntactDuringIdleness() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(1, valveOutput); valve.inputWatermark(new Watermark(25), 0); @@ -137,7 +140,7 @@ public class StatusWatermarkValveTest { */ @Test public void testMultipleInputYieldsWatermarkOnlyWhenAllChannelsReceivesWatermarks() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput); valve.inputWatermark(new Watermark(0), 0); @@ -156,7 +159,7 @@ public class StatusWatermarkValveTest { */ @Test public void testMultipleInputIncreasingWatermarks() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput); valve.inputWatermark(new Watermark(0), 0); @@ -190,7 +193,7 @@ public class StatusWatermarkValveTest { */ @Test public void testMultipleInputDecreasingWatermarksYieldsNoOutput() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput); valve.inputWatermark(new Watermark(25), 0); @@ -210,7 +213,7 @@ public class StatusWatermarkValveTest { */ @Test public void testMultipleInputStreamStatusToggling() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(2, valveOutput); // this also implicitly verifies that all input channels start as active @@ -244,7 +247,7 @@ public class StatusWatermarkValveTest { */ @Test public void testMultipleInputWatermarkAdvancingWithPartiallyIdleChannels() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput); valve.inputWatermark(new Watermark(15), 0); @@ -273,7 +276,7 @@ public class StatusWatermarkValveTest { */ @Test public void testMultipleInputWatermarkAdvancingAsChannelsIndividuallyBecomeIdle() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput); valve.inputWatermark(new Watermark(25), 0); @@ -301,7 +304,7 @@ public class StatusWatermarkValveTest { */ @Test public void testMultipleInputFlushMaxWatermarkAndStreamStatusOnceAllInputsBecomeIdle() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput); // ------------------------------------------------------------------------------------------- @@ -339,7 +342,7 @@ public class StatusWatermarkValveTest { */ @Test public void testMultipleInputWatermarkRealignmentAfterResumeActive() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput); valve.inputWatermark(new Watermark(10), 0); @@ -380,7 +383,7 @@ public class StatusWatermarkValveTest { */ @Test public void testNoOutputWhenAllActiveChannelsAreUnaligned() throws Exception { - BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler(); + StatusWatermarkOutput valveOutput = new StatusWatermarkOutput(); StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput); valve.inputWatermark(new Watermark(10), 0); @@ -403,22 +406,32 @@ public class StatusWatermarkValveTest { assertEquals(null, valveOutput.popLastSeenOutput()); } - private class BufferedValveOutputHandler implements StatusWatermarkValve.ValveOutputHandler { + private static class StatusWatermarkOutput implements PushingAsyncDataInput.DataOutput { + private BlockingQueue allOutputs = new LinkedBlockingQueue<>(); @Override - public void handleWatermark(Watermark watermark) { + public void emitWatermark(Watermark watermark) { allOutputs.add(watermark); } @Override - public void handleStreamStatus(StreamStatus streamStatus) { + public void emitStreamStatus(StreamStatus streamStatus) { allOutputs.add(streamStatus); } + @Override + public void emitRecord(StreamRecord record) { + throw new UnsupportedOperationException(); + } + + @Override + public void emitLatencyMarker(LatencyMarker latencyMarker) { + throw new UnsupportedOperationException(); + } + public StreamElement popLastSeenOutput() { return allOutputs.poll(); } } - } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceReaderStreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceReaderStreamTaskTest.java new file mode 100644 index 0000000000000000000000000000000000000000..61934c6dcd578029a22901e67cd60aff57472936 --- /dev/null +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SourceReaderStreamTaskTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.streaming.runtime.tasks; + +import org.apache.flink.api.common.typeinfo.BasicTypeInfo; +import org.apache.flink.runtime.jobgraph.OperatorID; +import org.apache.flink.streaming.api.graph.StreamConfig; +import org.apache.flink.streaming.api.operators.SourceReaderOperator; +import org.apache.flink.streaming.runtime.io.InputStatus; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.TestHarnessUtil; + +import org.junit.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * Tests for verifying that the {@link SourceReaderOperator} as a task input can be integrated + * well with {@link org.apache.flink.streaming.runtime.io.StreamOneInputProcessor}. + */ +public class SourceReaderStreamTaskTest { + + @Test + public void testSourceOutputCorrectness() throws Exception { + final int numRecords = 10; + final StreamTaskTestHarness testHarness = new StreamTaskTestHarness<>( + SourceReaderStreamTask::new, + BasicTypeInfo.INT_TYPE_INFO); + final StreamConfig streamConfig = testHarness.getStreamConfig(); + + testHarness.setupOutputForSingletonOperatorChain(); + streamConfig.setStreamOperator(new TestingFiniteSourceReaderOperator(numRecords)); + streamConfig.setOperatorID(new OperatorID()); + + testHarness.invoke(); + testHarness.waitForTaskCompletion(); + + final LinkedBlockingQueue expectedOutput = new LinkedBlockingQueue<>(); + for (int i = 1; i <= numRecords; i++) { + expectedOutput.add(new StreamRecord<>(i)); + } + + TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); + } + + /** + * A simple {@link SourceReaderOperator} implementation for emitting limited int type records. + */ + private static class TestingFiniteSourceReaderOperator extends SourceReaderOperator { + private static final long serialVersionUID = 1L; + + private final int numRecords; + private int counter; + + TestingFiniteSourceReaderOperator(int numRecords) { + this.numRecords = numRecords; + } + + @Override + public InputStatus emitNext(DataOutput output) throws Exception { + output.emitRecord(new StreamRecord<>(++counter)); + + return counter < numRecords ? InputStatus.MORE_AVAILABLE : InputStatus.END_OF_INPUT; + } + + @Override + public CompletableFuture isAvailable() { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + } + } +} diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamConfigChainer.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamConfigChainer.java index 06e1b99ead1fe36a91a686b17e36dfb48a93b806..0efb278946e9c643c36aada9f788715e001a3165 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamConfigChainer.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamConfigChainer.java @@ -25,7 +25,9 @@ import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.graph.StreamEdge; import org.apache.flink.streaming.api.graph.StreamNode; import org.apache.flink.streaming.api.operators.OneInputStreamOperator; +import org.apache.flink.streaming.api.operators.SimpleOperatorFactory; import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; import org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner; import java.util.Collections; @@ -42,29 +44,36 @@ import static org.apache.flink.util.Preconditions.checkNotNull; public class StreamConfigChainer { private final StreamConfig headConfig; private final Map chainedConfigs = new HashMap<>(); + private final long bufferTimeout; private StreamConfig tailConfig; private int chainIndex = 0; public StreamConfigChainer(OperatorID headOperatorID, StreamOperator headOperator, StreamConfig headConfig) { + this(headOperatorID, SimpleOperatorFactory.of(headOperator), headConfig); + } + + public StreamConfigChainer(OperatorID headOperatorID, StreamOperatorFactory headOperatorFactory, StreamConfig headConfig) { this.headConfig = checkNotNull(headConfig); this.tailConfig = checkNotNull(headConfig); + this.bufferTimeout = headConfig.getBufferTimeout(); - head(headOperator, headOperatorID); + head(headOperatorID, headOperatorFactory); } - private void head(StreamOperator headOperator, OperatorID headOperatorID) { - headConfig.setStreamOperator(headOperator); + private void head(OperatorID headOperatorID, StreamOperatorFactory headOperatorFactory) { + headConfig.setStreamOperatorFactory(headOperatorFactory); headConfig.setOperatorID(headOperatorID); headConfig.setChainStart(); headConfig.setChainIndex(chainIndex); + headConfig.setBufferTimeout(bufferTimeout); } public StreamConfigChainer chain( - OperatorID operatorID, - OneInputStreamOperator operator, - TypeSerializer typeSerializer, - boolean createKeyedStateBackend) { + OperatorID operatorID, + OneInputStreamOperator operator, + TypeSerializer typeSerializer, + boolean createKeyedStateBackend) { return chain(operatorID, operator, typeSerializer, typeSerializer, createKeyedStateBackend); } @@ -76,10 +85,10 @@ public class StreamConfigChainer { } public StreamConfigChainer chain( - OperatorID operatorID, - OneInputStreamOperator operator, - TypeSerializer inputSerializer, - TypeSerializer outputSerializer) { + OperatorID operatorID, + OneInputStreamOperator operator, + TypeSerializer inputSerializer, + TypeSerializer outputSerializer) { return chain(operatorID, operator, inputSerializer, outputSerializer, false); } @@ -89,6 +98,20 @@ public class StreamConfigChainer { TypeSerializer inputSerializer, TypeSerializer outputSerializer, boolean createKeyedStateBackend) { + return chain( + operatorID, + SimpleOperatorFactory.of(operator), + inputSerializer, + outputSerializer, + createKeyedStateBackend); + } + + public StreamConfigChainer chain( + OperatorID operatorID, + StreamOperatorFactory operatorFactory, + TypeSerializer inputSerializer, + TypeSerializer outputSerializer, + boolean createKeyedStateBackend) { chainIndex++; tailConfig.setChainedOutputs(Collections.singletonList( @@ -100,7 +123,7 @@ public class StreamConfigChainer { null, null))); tailConfig = new StreamConfig(new Configuration()); - tailConfig.setStreamOperator(checkNotNull(operator)); + tailConfig.setStreamOperatorFactory(checkNotNull(operatorFactory)); tailConfig.setOperatorID(checkNotNull(operatorID)); tailConfig.setTypeSerializerIn1(inputSerializer); tailConfig.setTypeSerializerOut(outputSerializer); @@ -109,6 +132,7 @@ public class StreamConfigChainer { tailConfig.setStateKeySerializer(inputSerializer); } tailConfig.setChainIndex(chainIndex); + tailConfig.setBufferTimeout(bufferTimeout); chainedConfigs.put(chainIndex, tailConfig); @@ -116,7 +140,6 @@ public class StreamConfigChainer { } public void finish() { - List outEdgesInOrder = new LinkedList(); outEdgesInOrder.add( new StreamEdge( @@ -127,7 +150,6 @@ public class StreamConfigChainer { new BroadcastPartitioner(), null)); - tailConfig.setBufferTimeout(0); tailConfig.setChainEnd(); tailConfig.setOutputSelectors(Collections.emptyList()); tailConfig.setNumberOfOutputs(1); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java index e9cbc2837895f729899166c6358e86dcc4f9296a..222e45b70fef6b52d8404d403d95c2fcea633e64 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTest.java @@ -21,6 +21,7 @@ package org.apache.flink.streaming.runtime.tasks; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.JobID; import org.apache.flink.api.common.typeutils.TypeSerializer; +import org.apache.flink.api.common.typeutils.base.StringSerializer; import org.apache.flink.configuration.CheckpointingOptions; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.NettyShuffleEnvironmentOptions; @@ -53,6 +54,7 @@ import org.apache.flink.runtime.filecache.FileCache; import org.apache.flink.runtime.io.disk.iomanager.IOManager; import org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder; import org.apache.flink.runtime.io.network.TaskEventDispatcher; +import org.apache.flink.runtime.io.network.api.writer.RecordWriter; import org.apache.flink.runtime.io.network.partition.NoOpResultPartitionConsumableNotifier; import org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier; import org.apache.flink.runtime.jobgraph.JobVertexID; @@ -102,13 +104,16 @@ import org.apache.flink.streaming.api.TimeCharacteristic; import org.apache.flink.streaming.api.functions.source.SourceFunction; import org.apache.flink.streaming.api.graph.StreamConfig; import org.apache.flink.streaming.api.operators.AbstractStreamOperator; +import org.apache.flink.streaming.api.operators.ChainingStrategy; import org.apache.flink.streaming.api.operators.InternalTimeServiceManager; import org.apache.flink.streaming.api.operators.OperatorSnapshotFutures; import org.apache.flink.streaming.api.operators.Output; import org.apache.flink.streaming.api.operators.StreamOperator; +import org.apache.flink.streaming.api.operators.StreamOperatorFactory; import org.apache.flink.streaming.api.operators.StreamOperatorStateContext; import org.apache.flink.streaming.api.operators.StreamSource; import org.apache.flink.streaming.api.operators.StreamTaskStateInitializer; +import org.apache.flink.streaming.runtime.io.InputStatus; import org.apache.flink.streaming.runtime.io.StreamInputProcessor; import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; @@ -134,6 +139,7 @@ import javax.annotation.Nullable; import java.io.Closeable; import java.io.IOException; import java.io.ObjectInputStream; +import java.io.StreamCorruptedException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -347,7 +353,7 @@ public class StreamTaskTest extends TestLogger { @Test public void testDecliningCheckpointStreamOperator() throws Exception { - CheckpointExceptionHandlerTest.DeclineDummyEnvironment declineDummyEnvironment = new CheckpointExceptionHandlerTest.DeclineDummyEnvironment(); + DeclineDummyEnvironment declineDummyEnvironment = new DeclineDummyEnvironment(); // mock the returned snapshots OperatorSnapshotFutures operatorSnapshotResult1 = mock(OperatorSnapshotFutures.class); @@ -835,6 +841,53 @@ public class StreamTaskTest extends TestLogger { } } + /** + * This test ensures that {@link RecordWriter} is correctly closed even if we fail to construct + * {@link OperatorChain}, for example because of user class deserialization error. + */ + @Test + public void testRecordWriterClosedOnStreamOperatorFactoryDeserializationError() throws Exception { + Configuration taskConfiguration = new Configuration(); + StreamConfig streamConfig = new StreamConfig(taskConfiguration); + + // Make sure that there is some output edge in the config so that some RecordWriter is created + StreamConfigChainer cfg = new StreamConfigChainer(new OperatorID(42, 42), new UnusedOperatorFactory(), streamConfig); + cfg.chain( + new OperatorID(44, 44), + new UnusedOperatorFactory(), + StringSerializer.INSTANCE, + StringSerializer.INSTANCE, + false); + cfg.finish(); + + // Overwrite the serialized bytes to some garbage to induce deserialization exception + taskConfiguration.setBytes(StreamConfig.SERIALIZEDUDF, new byte[42]); + + try (MockEnvironment mockEnvironment = + new MockEnvironmentBuilder() + .setTaskConfiguration(taskConfiguration) + .build()) { + + mockEnvironment.addOutput(new ArrayList<>()); + StreamTask streamTask = + new NoOpStreamTask<>(mockEnvironment); + + try { + streamTask.invoke(); + fail("Should have failed with an exception!"); + } catch (Exception ex) { + if (!ExceptionUtils.findThrowable(ex, StreamCorruptedException.class).isPresent()) { + throw ex; + } + } + } + + assertTrue( + RecordWriter.DEFAULT_OUTPUT_FLUSH_THREAD_NAME + " thread is still running", + Thread.getAllStackTraces().keySet().stream() + .noneMatch(thread -> thread.getName().startsWith(RecordWriter.DEFAULT_OUTPUT_FLUSH_THREAD_NAME))); + } + // ------------------------------------------------------------------------ // Test Utilities // ------------------------------------------------------------------------ @@ -1150,19 +1203,14 @@ public class StreamTaskTest extends TestLogger { } @Override - public boolean processInput() throws Exception { - return false; + public InputStatus processInput() throws Exception { + return isFinished ? InputStatus.END_OF_INPUT : InputStatus.NOTHING_AVAILABLE; } @Override public void close() throws IOException { } - @Override - public boolean isFinished() { - return isFinished; - } - @Override public CompletableFuture isAvailable() { return AVAILABLE; @@ -1673,4 +1721,54 @@ public class StreamTaskTest extends TestLogger { throw failingCause; } } + + static final class DeclineDummyEnvironment extends DummyEnvironment { + + private long lastDeclinedCheckpointId; + private Throwable lastDeclinedCheckpointCause; + + DeclineDummyEnvironment() { + super("test", 1, 0); + this.lastDeclinedCheckpointId = Long.MIN_VALUE; + this.lastDeclinedCheckpointCause = null; + } + + @Override + public void declineCheckpoint(long checkpointId, Throwable cause) { + this.lastDeclinedCheckpointId = checkpointId; + this.lastDeclinedCheckpointCause = cause; + } + + long getLastDeclinedCheckpointId() { + return lastDeclinedCheckpointId; + } + + Throwable getLastDeclinedCheckpointCause() { + return lastDeclinedCheckpointCause; + } + } + + private static class UnusedOperatorFactory implements StreamOperatorFactory { + @Override + public > T createStreamOperator( + StreamTask containingTask, + StreamConfig config, + Output> output) { + throw new UnsupportedOperationException("This shouldn't be called"); + } + + @Override + public void setChainingStrategy(ChainingStrategy strategy) { + } + + @Override + public ChainingStrategy getChainingStrategy() { + return ChainingStrategy.ALWAYS; + } + + @Override + public Class getStreamOperatorClass(ClassLoader classLoader) { + throw new UnsupportedOperationException(); + } + } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java index aa5f25e0a4c8a6ad6fe2851f726111b665093271..c5db3ffff77bc37ba449ffd4d596fb93f614012d 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/StreamTaskTestHarness.java @@ -136,6 +136,7 @@ public class StreamTaskTestHarness { this.executionConfig = new ExecutionConfig(); streamConfig = new StreamConfig(taskConfig); + streamConfig.setBufferTimeout(0); outputSerializer = outputType.createSerializer(executionConfig); outputStreamRecordSerializer = new StreamElementSerializer(outputSerializer); @@ -184,7 +185,6 @@ public class StreamTaskTestHarness { Preconditions.checkState(!setupCalled, "This harness was already setup."); setupCalled = true; streamConfig.setChainStart(); - streamConfig.setBufferTimeout(0); streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime); streamConfig.setOutputSelectors(Collections.>emptyList()); streamConfig.setNumberOfOutputs(1); diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java index a01a9c8fc82210f04fcd6e89cfeabc4bd65e1c3a..493dbf3636f1dc7d3f5f407fb34b6500422cdb2f 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/runtime/tasks/SystemProcessingTimeServiceTest.java @@ -26,6 +26,7 @@ import org.junit.Assert; import org.junit.Test; import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -34,10 +35,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; -import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.nullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -140,40 +138,26 @@ public class SystemProcessingTimeServiceTest extends TestLogger { @Test public void testImmediateShutdown() throws Exception { + final CompletableFuture errorFuture = new CompletableFuture<>(); - final Object lock = new Object(); - final AtomicReference errorRef = new AtomicReference<>(); - - final SystemProcessingTimeService timer = createSystemProcessingTimeService(errorRef); + final SystemProcessingTimeService timer = createSystemProcessingTimeService(errorFuture); try { assertFalse(timer.isTerminated()); final OneShotLatch latch = new OneShotLatch(); - // the task should trigger immediately and should block until terminated with interruption + // the task should trigger immediately and sleep until terminated with interruption timer.registerTimer(System.currentTimeMillis(), timestamp -> { - synchronized (lock) { - latch.trigger(); - Thread.sleep(100000000); - } + latch.trigger(); + Thread.sleep(100000000); }); latch.await(); timer.shutdownService(); - synchronized (lock) { - assertTrue(timer.isTerminated()); - - // The shutdownService() may not necessary wait for active tasks to finish properly. - // From the ScheduledThreadPoolExecutor Java docs: - // There are no guarantees beyond best-effort attempts to stop processing actively executing tasks. - // This implementation cancels tasks via {@link Thread#interrupt}, so any task that - // fails to respond to interrupts may never terminate. - assertThat(errorRef.get(), is(anyOf(nullValue(), instanceOf(InterruptedException.class)))); - assertEquals(0, timer.getNumTasksScheduled()); - } - errorRef.set(null); + assertTrue(timer.isTerminated()); + assertEquals(0, timer.getNumTasksScheduled()); try { timer.registerTimer(System.currentTimeMillis() + 1000, timestamp -> fail("should not be called")); @@ -183,7 +167,6 @@ public class SystemProcessingTimeServiceTest extends TestLogger { catch (IllegalStateException e) { // expected } - assertThat(errorRef.get(), is(nullValue())); try { timer.scheduleAtFixedRate(timestamp -> fail("should not be called"), 0L, 100L); @@ -193,7 +176,9 @@ public class SystemProcessingTimeServiceTest extends TestLogger { catch (IllegalStateException e) { // expected } - assertThat(errorRef.get(), is(nullValue())); + + // check that the task eventually responded to interruption + assertThat(errorFuture.get(30L, TimeUnit.SECONDS), instanceOf(InterruptedException.class)); } finally { timer.shutdownService(); @@ -375,6 +360,12 @@ public class SystemProcessingTimeServiceTest extends TestLogger { Assert.assertTrue(timerFinished.get()); } + private static SystemProcessingTimeService createSystemProcessingTimeService(CompletableFuture errorFuture) { + Preconditions.checkArgument(!errorFuture.isDone()); + + return new SystemProcessingTimeService(new TestOnTimerCompletablyCallbackContext(errorFuture)); + } + private static SystemProcessingTimeService createSystemProcessingTimeService(AtomicReference errorRef) { Preconditions.checkArgument(errorRef.get() == null); @@ -439,4 +430,21 @@ public class SystemProcessingTimeServiceTest extends TestLogger { } } } + + private static class TestOnTimerCompletablyCallbackContext implements SystemProcessingTimeService.ScheduledCallbackExecutionContext { + private final CompletableFuture completableExceptionHandler; + + TestOnTimerCompletablyCallbackContext(CompletableFuture completableExceptionHandler) { + this.completableExceptionHandler = completableExceptionHandler; + } + + @Override + public void invoke(ProcessingTimeCallback callback, long timestamp) { + try { + callback.onProcessingTime(timestamp); + } catch (Throwable t) { + completableExceptionHandler.complete(t); + } + } + } } diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java index c0339510f1cc1995ccd3cc52290714412b77aaca..8c7713a913873ac6dba8453677b6b1b6f606b827 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/AbstractStreamOperatorTestHarness.java @@ -544,6 +544,13 @@ public class AbstractStreamOperatorTestHarness implements AutoCloseable { operator.open(); } + /** + * Calls {@link StreamOperator#prepareSnapshotPreBarrier(long)}. + */ + public void prepareSnapshotPreBarrier(long checkpointId) throws Exception { + operator.prepareSnapshotPreBarrier(checkpointId); + } + /** * Calls {@link StreamOperator#snapshotState(long, long, CheckpointOptions, org.apache.flink.runtime.state.CheckpointStreamFactory)}. */ diff --git a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/ContentDump.java b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/ContentDump.java index 903b2377908ac0fe927e052b2e54f50732ba68cb..5c1568eeb201f9ad3c9fbf25acae0aaaf9f92ae0 100644 --- a/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/ContentDump.java +++ b/flink-streaming-java/src/test/java/org/apache/flink/streaming/util/ContentDump.java @@ -124,7 +124,7 @@ public class ContentDump { /** * Exception thrown for an attempt to write into read-only {@link ContentDump}. */ - public class NotWritableException extends RuntimeException { + public static class NotWritableException extends RuntimeException { public NotWritableException(String name) { super(String.format("File [%s] is not writable", name)); } diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AsyncDataStreamITCase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AsyncDataStreamITCase.scala index a4a9772e43a395b1ad7fb395f7012dbc46ea30fa..497b61cffef265957e4163dfb23f1faa6fd6a29c 100644 --- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AsyncDataStreamITCase.scala +++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/AsyncDataStreamITCase.scala @@ -18,8 +18,6 @@ package org.apache.flink.streaming.api.scala -import java.util.concurrent.{CountDownLatch, TimeUnit} - import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.sink.SinkFunction import org.apache.flink.streaming.api.scala.AsyncDataStreamITCase._ @@ -27,27 +25,28 @@ import org.apache.flink.streaming.api.scala.async.{ResultFuture, RichAsyncFuncti import org.apache.flink.test.util.AbstractTestBase import org.junit.Assert._ import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized +import org.junit.runners.Parameterized.Parameters + +import java.{util => ju} +import java.util.concurrent.{CountDownLatch, TimeUnit} import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future} object AsyncDataStreamITCase { private var testResult: mutable.ArrayBuffer[Int] = _ + + @Parameters(name = "ordered = {0}") + def parameters: ju.Collection[Boolean] = ju.Arrays.asList(true, false) } -class AsyncDataStreamITCase extends AbstractTestBase { +@RunWith(value = classOf[Parameterized]) +class AsyncDataStreamITCase(ordered: Boolean) extends AbstractTestBase { @Test - def testOrderedWait(): Unit = { - testAsyncWait(true) - } - - @Test - def testUnorderedWait(): Unit = { - testAsyncWait(false) - } - - private def testAsyncWait(ordered: Boolean): Unit = { + def testAsyncWithTimeout(): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment env.setParallelism(1) @@ -65,6 +64,25 @@ class AsyncDataStreamITCase extends AbstractTestBase { executeAndValidate(ordered, env, asyncMapped, mutable.ArrayBuffer[Int](3)) } + @Test + def testAsyncWithoutTimeout(): Unit = { + val env = StreamExecutionEnvironment.getExecutionEnvironment + env.setParallelism(1) + + val source = env.fromElements(1) + + val timeout = 1L + val asyncMapped = if (ordered) { + AsyncDataStream.orderedWait( + source, new AsyncFunctionWithoutTimeoutExpired(), timeout, TimeUnit.MILLISECONDS) + } else { + AsyncDataStream.unorderedWait( + source, new AsyncFunctionWithoutTimeoutExpired(), timeout, TimeUnit.MILLISECONDS) + } + + executeAndValidate(ordered, env, asyncMapped, mutable.ArrayBuffer[Int](2)) + } + private def executeAndValidate(ordered: Boolean, env: StreamExecutionEnvironment, dataStream: DataStream[Int], @@ -86,16 +104,6 @@ class AsyncDataStreamITCase extends AbstractTestBase { } } - @Test - def testOrderedWaitUsingAnonymousFunction(): Unit = { - testAsyncWaitUsingAnonymousFunction(true) - } - - @Test - def testUnorderedWaitUsingAnonymousFunction(): Unit = { - testAsyncWaitUsingAnonymousFunction(false) - } - @Test def testRichAsyncFunctionRuntimeContext(): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment @@ -105,13 +113,19 @@ class AsyncDataStreamITCase extends AbstractTestBase { val timeout = 10000L val richAsyncFunction = new MyRichAsyncFunction - val asyncMapped = AsyncDataStream - .unorderedWait(source, richAsyncFunction, timeout, TimeUnit.MILLISECONDS) + val asyncMapped = if (ordered) { + AsyncDataStream + .orderedWait(source, richAsyncFunction, timeout, TimeUnit.MILLISECONDS) + } else { + AsyncDataStream + .unorderedWait(source, richAsyncFunction, timeout, TimeUnit.MILLISECONDS) + } executeAndValidate(false, env, asyncMapped, mutable.ArrayBuffer[Int](2)) } - private def testAsyncWaitUsingAnonymousFunction(ordered: Boolean): Unit = { + @Test + def testAsyncWaitUsingAnonymousFunction(): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment env.setParallelism(1) @@ -157,6 +171,33 @@ class AsyncFunctionWithTimeoutExpired extends RichAsyncFunction[Int, Int] { } } +/** + * The asyncInvoke and timeout might be invoked at the same time. + * The target is checking whether there is a race condition or not between asyncInvoke, + * timeout and timer cancellation. + * See https://issues.apache.org/jira/browse/FLINK-13605 for more details. + */ +class AsyncFunctionWithoutTimeoutExpired extends RichAsyncFunction[Int, Int] { + @transient var timeoutLatch: CountDownLatch = _ + + override def open(parameters: Configuration): Unit = { + timeoutLatch = new CountDownLatch(1) + } + + override def asyncInvoke(input: Int, resultFuture: ResultFuture[Int]): Unit = { + Future { + resultFuture.complete(Seq(input * 2)) + timeoutLatch.countDown() + } (ExecutionContext.global) + } + override def timeout(input: Int, resultFuture: ResultFuture[Int]): Unit = { + // this sleeping helps reproducing race condition with cancellation + Thread.sleep(10) + timeoutLatch.await() + resultFuture.complete(Seq(input * 3)) + } +} + class MyRichAsyncFunction extends RichAsyncFunction[Int, Int] { override def open(parameters: Configuration): Unit = { diff --git a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliStrings.java b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliStrings.java index 930ebbcb67e77fa2e3f02afa94453d7c3d9a55b8..1c2fe921a3ca1430c990627d61430ce73dcfd1ee 100644 --- a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliStrings.java +++ b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/cli/CliStrings.java @@ -52,7 +52,7 @@ public final class CliStrings { .append(formatCommand(SqlCommand.RESET, "Resets all session configuration properties.")) .append(formatCommand(SqlCommand.SELECT, "Executes a SQL SELECT query on the Flink cluster.")) .append(formatCommand(SqlCommand.SET, "Sets a session configuration property. Syntax: 'SET =;'. Use 'SET;' for listing all properties.")) - .append(formatCommand(SqlCommand.SHOW_FUNCTIONS, "Shows all registered user-defined functions.")) + .append(formatCommand(SqlCommand.SHOW_FUNCTIONS, "Shows all user-defined and built-in functions.")) .append(formatCommand(SqlCommand.SHOW_TABLES, "Shows all registered tables.")) .append(formatCommand(SqlCommand.SOURCE, "Reads a SQL SELECT query from a file and executes it on the Flink cluster.")) .append(formatCommand(SqlCommand.USE_CATALOG, "Sets the current catalog. The current database is set to the catalog's default one. Experimental! Syntax: 'USE CATALOG ;'")) diff --git a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java index 27c3ee261a7e1db7a224e0e4f90f3379b15dced3..73c9ccebd9f42a01c067290046fed4005c47938b 100644 --- a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java +++ b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ExecutionContext.java @@ -21,19 +21,17 @@ package org.apache.flink.table.client.gateway.local; import org.apache.flink.api.common.ExecutionConfig; import org.apache.flink.api.common.Plan; import org.apache.flink.api.common.time.Time; +import org.apache.flink.api.dag.Pipeline; import org.apache.flink.api.java.ExecutionEnvironment; +import org.apache.flink.client.ClientUtils; +import org.apache.flink.client.FlinkPipelineTranslationUtil; import org.apache.flink.client.cli.CliArgsException; import org.apache.flink.client.cli.CustomCommandLine; import org.apache.flink.client.cli.RunOptions; import org.apache.flink.client.deployment.ClusterDescriptor; import org.apache.flink.client.deployment.ClusterSpecification; -import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.plugin.TemporaryClassLoaderContext; -import org.apache.flink.optimizer.DataStatistics; -import org.apache.flink.optimizer.Optimizer; -import org.apache.flink.optimizer.costs.DefaultCostEstimator; -import org.apache.flink.optimizer.plan.FlinkPlan; import org.apache.flink.runtime.execution.librarycache.FlinkUserCodeClassLoaders; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.streaming.api.TimeCharacteristic; @@ -440,16 +438,29 @@ public class ExecutionContext { } public JobGraph createJobGraph(String name) { - final FlinkPlan plan = createPlan(name, flinkConfig); - return ClusterClient.getJobGraph( - flinkConfig, - plan, - dependencies, - runOptions.getClasspaths(), - runOptions.getSavepointRestoreSettings()); + final Pipeline pipeline = createPipeline(name, flinkConfig); + + int parallelism; + if (execEnv != null) { + parallelism = execEnv.getParallelism(); + } else if (streamExecEnv != null) { + parallelism = streamExecEnv.getParallelism(); + } else { + throw new RuntimeException("No execution environment defined."); + } + JobGraph jobGraph = FlinkPipelineTranslationUtil.getJobGraph( + pipeline, + flinkConfig, + parallelism); + + ClientUtils.addJarFiles(jobGraph, dependencies); + jobGraph.setClasspaths(runOptions.getClasspaths()); + jobGraph.setSavepointRestoreSettings(runOptions.getSavepointRestoreSettings()); + + return jobGraph; } - private FlinkPlan createPlan(String name, Configuration flinkConfig) { + private Pipeline createPipeline(String name, Configuration flinkConfig) { if (streamExecEnv != null) { // special case for Blink planner to apply batch optimizations // note: it also modifies the ExecutionConfig! @@ -461,8 +472,7 @@ public class ExecutionContext { final int parallelism = execEnv.getParallelism(); final Plan unoptimizedPlan = execEnv.createProgramPlan(); unoptimizedPlan.setJobName(name); - final Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig); - return ClusterClient.getOptimizedPlan(compiler, unoptimizedPlan, parallelism); + return unoptimizedPlan; } } diff --git a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java index 16f1161c6ed82ba077073a7dba3870aff914256d..aff7e7daff00b040d90514330fa4cbc4dee3a3b1 100644 --- a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java +++ b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/LocalExecutor.java @@ -20,12 +20,12 @@ package org.apache.flink.table.client.gateway.local; import org.apache.flink.api.common.JobID; import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.client.ClientUtils; import org.apache.flink.client.cli.CliFrontend; import org.apache.flink.client.cli.CliFrontendParser; import org.apache.flink.client.cli.CustomCommandLine; import org.apache.flink.client.deployment.ClusterDescriptor; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.JobWithJars; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.GlobalConfiguration; import org.apache.flink.core.fs.FileSystem; @@ -581,7 +581,7 @@ public class LocalExecutor implements Executor { try { // find jar files for (URL url : jars) { - JobWithJars.checkJarFile(url); + ClientUtils.checkJarFile(url); dependencies.add(url); } @@ -601,7 +601,7 @@ public class LocalExecutor implements Executor { // only consider jars if (f.isFile() && f.getAbsolutePath().toLowerCase().endsWith(".jar")) { final URL url = f.toURI().toURL(); - JobWithJars.checkJarFile(url); + ClientUtils.checkJarFile(url); dependencies.add(url); } } diff --git a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ProgramDeployer.java b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ProgramDeployer.java index b484521eddeaaf1759a10e064585c66486bce792..61de25c54d7cf0d33600f327caeea73900ffa645 100644 --- a/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ProgramDeployer.java +++ b/flink-table/flink-sql-client/src/main/java/org/apache/flink/table/client/gateway/local/ProgramDeployer.java @@ -21,7 +21,6 @@ package org.apache.flink.table.client.gateway.local; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.client.deployment.ClusterDescriptor; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.table.client.gateway.SqlExecutionException; import org.apache.flink.table.client.gateway.local.result.Result; @@ -122,7 +121,7 @@ public class ProgramDeployer implements Runnable { // get result if (awaitJobResult) { // we need to hard cast for now - final JobExecutionResult jobResult = ((RestClusterClient) clusterClient) + final JobExecutionResult jobResult = clusterClient .requestJobResult(jobGraph.getJobID()) .get() .toJobExecutionResult(context.getClassLoader()); // throws exception if job fails diff --git a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java index c102acf8a483c70a4912e6f13b11485872340c4a..24799e047838cd142544c1966226ecf7c3460f9c 100644 --- a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java +++ b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/gateway/local/LocalExecutorITCase.java @@ -119,7 +119,7 @@ public class LocalExecutorITCase extends TestLogger { private static Configuration getConfig() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "4m"); config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM); config.setBoolean(WebOptions.SUBMIT_ENABLE, false); diff --git a/flink-table/flink-sql-parser/pom.xml b/flink-table/flink-sql-parser/pom.xml index a4c9b5dfa2d830210c7e48a06046246cfcdd8ac4..67dcf3bbdc9a55a59b24bc9e146a2f0dd30a5729 100644 --- a/flink-table/flink-sql-parser/pom.xml +++ b/flink-table/flink-sql-parser/pom.xml @@ -35,7 +35,6 @@ under the License. - 1.20.0 @@ -46,9 +45,9 @@ under the License. ${calcite.version} -/** -* Parse a nullable option, default to be nullable. -*/ -boolean NullableOpt() : -{ -} -{ - { return true; } -| - { return false; } -| - { return true; } -} - void TableColumn(TableCreationContext context) : { } @@ -69,8 +55,7 @@ void TableColumn2(List list) : } { name = SimpleIdentifier() - <#-- #FlinkDataType already takes care of the nullable attribute. --> - type = FlinkDataType() + type = ExtendedDataType() [ { String p = SqlParserUtil.parseString(token.image); comment = SqlLiteral.createCharString(p, getPos()); @@ -81,6 +66,35 @@ void TableColumn2(List list) : } } +/** +* Different with {@link #DataType()}, we support a [ NULL | NOT NULL ] suffix syntax for both the +* collection element data type and the data type itself. +* +*

    See {@link #SqlDataTypeSpec} for the syntax details of {@link #DataType()}. +*/ +SqlDataTypeSpec ExtendedDataType() : +{ + SqlTypeNameSpec typeName; + final Span s; + boolean elementNullable = true; + boolean nullable = true; +} +{ + <#-- #DataType does not take care of the nullable attribute. --> + typeName = TypeName() { + s = span(); + } + ( + LOOKAHEAD(3) + elementNullable = NullableOptDefaultTrue() + typeName = ExtendedCollectionsTypeName(typeName, elementNullable) + )* + nullable = NullableOptDefaultTrue() + { + return new SqlDataTypeSpec(typeName, s.end(this)).withNullable(nullable); + } +} + SqlNodeList PrimaryKey() : { List pkList = new ArrayList(); @@ -159,8 +173,8 @@ SqlCreate SqlCreateTable(Span s, boolean replace) : { final SqlParserPos startPos = s.pos(); SqlIdentifier tableName; - SqlNodeList primaryKeyList = null; - List uniqueKeysList = null; + SqlNodeList primaryKeyList = SqlNodeList.EMPTY; + List uniqueKeysList = new ArrayList(); SqlNodeList columnList = SqlNodeList.EMPTY; SqlCharStringLiteral comment = null; @@ -375,210 +389,100 @@ SqlDrop SqlDropView(Span s, boolean replace) : } } -SqlIdentifier FlinkCollectionsTypeName() : -{ -} -{ - LOOKAHEAD(2) - { - return new SqlIdentifier(SqlTypeName.MULTISET.name(), getPos()); - } -| - { - return new SqlIdentifier(SqlTypeName.ARRAY.name(), getPos()); - } -} - -SqlIdentifier FlinkTypeName() : +/** +* A sql type name extended basic data type, it has a counterpart basic +* sql type name but always represents as a special alias compared with the standard name. +* +*

    For example, STRING is synonym of VARCHAR(INT_MAX) +* and BYTES is synonym of VARBINARY(INT_MAX). +*/ +SqlTypeNameSpec ExtendedSqlBasicTypeName() : { - final SqlTypeName sqlTypeName; - final SqlIdentifier typeName; - final Span s = Span.of(); + final SqlTypeName typeName; + final String typeAlias; + int precision = -1; } { ( -<#-- additional types are included here --> -<#-- make custom data types in front of Calcite core data types --> -<#list parser.flinkDataTypeParserMethods as method> - <#if (method?index > 0)> - | - - LOOKAHEAD(2) - typeName = ${method} - - | - LOOKAHEAD(2) - sqlTypeName = SqlTypeName(s) { - typeName = new SqlIdentifier(sqlTypeName.name(), s.end(this)); + { + typeName = SqlTypeName.VARCHAR; + typeAlias = token.image; + precision = Integer.MAX_VALUE; } | - LOOKAHEAD(2) - typeName = FlinkCollectionsTypeName() - | - typeName = CompoundIdentifier() { - throw new ParseException("UDT in DDL is not supported yet."); + { + typeName = SqlTypeName.VARBINARY; + typeAlias = token.image; + precision = Integer.MAX_VALUE; } ) { - return typeName; + return new ExtendedSqlBasicTypeNameSpec(typeAlias, typeName, precision, getPos()); } } -/** -* Parse a Flink data type with nullable options, NULL -> nullable, NOT NULL -> not nullable. -* Default to be nullable. +/* +* Parses collection type name that does not belong to standard SQL, i.e. ARRAY<INT NOT NULL>. */ -SqlDataTypeSpec FlinkDataType() : +SqlTypeNameSpec CustomizedCollectionsTypeName() : { - final SqlIdentifier typeName; - SqlIdentifier collectionTypeName = null; - int scale = -1; - int precision = -1; - String charSetName = null; - final Span s; - boolean nullable = true; + final SqlTypeName collectionTypeName; + final SqlTypeNameSpec elementTypeName; boolean elementNullable = true; } { - typeName = FlinkTypeName() { - s = span(); - } - [ - - precision = UnsignedIntLiteral() - [ - - scale = UnsignedIntLiteral() - ] - - ] - elementNullable = NullableOpt() - [ - collectionTypeName = FlinkCollectionsTypeName() - nullable = NullableOpt() - ] - { - if (null != collectionTypeName) { - return new FlinkSqlDataTypeSpec( - collectionTypeName, - typeName, - precision, - scale, - charSetName, - nullable, - elementNullable, - s.end(collectionTypeName)); - } - nullable = elementNullable; - return new FlinkSqlDataTypeSpec(typeName, - precision, - scale, - charSetName, - null, - nullable, - elementNullable, - s.end(this)); - } -} - -SqlIdentifier SqlStringType() : -{ -} -{ - { return new SqlStringType(getPos()); } -} - -SqlIdentifier SqlBytesType() : -{ -} -{ - { return new SqlBytesType(getPos()); } -} - -boolean WithLocalTimeZone() : -{ -} -{ -

    The difference with {@link #SqlRowTypeName()}: +*

      +*
    • Support comment syntax for every field
      • +*
    • Field data type default is nullable
      • +*
    • Support ROW type with empty fields, e.g. ROW()
      • +*
    */ -SqlIdentifier SqlRowType() : +SqlTypeNameSpec ExtendedSqlRowTypeName() : { List fieldNames = new ArrayList(); List fieldTypes = new ArrayList(); List comments = new ArrayList(); + final boolean unparseAsStandard; } { ( - + { unparseAsStandard = false; } | - FieldNameTypeCommaList(fieldNames, fieldTypes, comments) + ExtendedFieldNameTypeCommaList(fieldNames, fieldTypes, comments) + { unparseAsStandard = false; } | - FieldNameTypeCommaList(fieldNames, fieldTypes, comments) + ExtendedFieldNameTypeCommaList(fieldNames, fieldTypes, comments) + { unparseAsStandard = true; } ) { - return new SqlRowType(getPos(), fieldNames, fieldTypes, comments); + return new ExtendedSqlRowTypeNameSpec( + getPos(), + fieldNames, + fieldTypes, + comments, + unparseAsStandard); } } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlRowType.java b/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/ExtendedSqlRowTypeNameSpec.java similarity index 41% rename from flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlRowType.java rename to flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/ExtendedSqlRowTypeNameSpec.java index 886125cd07a70313dd1655acf22f16f16b2a7d23..42828d8c8c1fc1b79aa255fa882766bdb55ac8fa 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlRowType.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/ExtendedSqlRowTypeNameSpec.java @@ -16,35 +16,56 @@ * limitations under the License. */ -package org.apache.flink.sql.parser.type; +package org.apache.calcite.sql; -import org.apache.calcite.sql.SqlCharStringLiteral; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.util.Litmus; import org.apache.calcite.util.Pair; import java.util.List; +import java.util.stream.Collectors; /** - * Parse column of Row type. + * A sql type name specification of ROW type. + * + *

    The difference with {@link SqlRowTypeNameSpec}: + *

      + *
    • Support comment syntax for every field
      • + *
    • Field data type default is nullable
      • + *
    • Support ROW type with empty fields, e.g. ROW()
      • + *
    */ -public class SqlRowType extends SqlIdentifier implements ExtendedSqlType { +public class ExtendedSqlRowTypeNameSpec extends SqlTypeNameSpec { private final List fieldNames; private final List fieldTypes; private final List comments; - public SqlRowType(SqlParserPos pos, + private final boolean unparseAsStandard; + + /** + * Creates a ROW type specification. + * + * @param pos parser position + * @param fieldNames field names + * @param fieldTypes field data types + * @param comments field comments + * @param unparseAsStandard whether to unparse as standard SQL style + */ + public ExtendedSqlRowTypeNameSpec(SqlParserPos pos, List fieldNames, List fieldTypes, - List comments) { - super(SqlTypeName.ROW.getName(), pos); + List comments, + boolean unparseAsStandard) { + super(new SqlIdentifier(SqlTypeName.ROW.getName(), pos), pos); this.fieldNames = fieldNames; this.fieldTypes = fieldTypes; this.comments = comments; + this.unparseAsStandard = unparseAsStandard; } public List getFieldNames() { @@ -59,30 +80,30 @@ public class SqlRowType extends SqlIdentifier implements ExtendedSqlType { return comments; } - public int getArity() { - return fieldNames.size(); - } - - public SqlIdentifier getFieldName(int i) { - return fieldNames.get(i); - } - - public SqlDataTypeSpec getFieldType(int i) { - return fieldTypes.get(i); - } - @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { writer.print("ROW"); if (getFieldNames().size() == 0) { - writer.print("<>"); + if (unparseAsStandard) { + writer.print("()"); + } else { + writer.print("<>"); + } } else { - SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "<", ">"); + SqlWriter.Frame frame; + if (unparseAsStandard) { + frame = writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "(", ")"); + } else { + frame = writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "<", ">"); + } int i = 0; for (Pair p : Pair.zip(this.fieldNames, this.fieldTypes)) { writer.sep(",", false); p.left.unparse(writer, 0, 0); - ExtendedSqlType.unparseType(p.right, writer, leftPrec, rightPrec); + p.right.unparse(writer, leftPrec, rightPrec); + if (!p.right.getNullable()) { + writer.keyword("NOT NULL"); + } if (comments.get(i) != null) { comments.get(i).unparse(writer, leftPrec, rightPrec); } @@ -91,4 +112,39 @@ public class SqlRowType extends SqlIdentifier implements ExtendedSqlType { writer.endList(frame); } } + + @Override public boolean equalsDeep(SqlTypeNameSpec node, Litmus litmus) { + if (!(node instanceof SqlRowTypeNameSpec)) { + return litmus.fail("{} != {}", this, node); + } + ExtendedSqlRowTypeNameSpec that = (ExtendedSqlRowTypeNameSpec) node; + if (this.fieldNames.size() != that.fieldNames.size()) { + return litmus.fail("{} != {}", this, node); + } + for (int i = 0; i < fieldNames.size(); i++) { + if (!this.fieldNames.get(i).equalsDeep(that.fieldNames.get(i), litmus)) { + return litmus.fail("{} != {}", this, node); + } + } + if (this.fieldTypes.size() != that.fieldTypes.size()) { + return litmus.fail("{} != {}", this, node); + } + for (int i = 0; i < fieldTypes.size(); i++) { + if (!this.fieldTypes.get(i).equals(that.fieldTypes.get(i))) { + return litmus.fail("{} != {}", this, node); + } + } + return litmus.succeed(); + } + + @Override public RelDataType deriveType(SqlValidator sqlValidator) { + final RelDataTypeFactory typeFactory = sqlValidator.getTypeFactory(); + return typeFactory.createStructType( + fieldTypes.stream() + .map(dt -> dt.deriveType(sqlValidator)) + .collect(Collectors.toList()), + fieldNames.stream() + .map(SqlIdentifier::toString) + .collect(Collectors.toList())); + } } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlMapType.java b/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/SqlMapTypeNameSpec.java similarity index 45% rename from flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlMapType.java rename to flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/SqlMapTypeNameSpec.java index fd071c09c807985f658198372e3d0eb50fdc7bd6..0e22dbfd6ddca407f22df8e0a33242e5f9f4f6ac 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlMapType.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/SqlMapTypeNameSpec.java @@ -16,24 +16,37 @@ * limitations under the License. */ -package org.apache.flink.sql.parser.type; +package org.apache.calcite.sql; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.util.Litmus; /** - * Parse column of Map type. + * Parse SQL MAP type, i.e. MAP<INT NOT NULL, TIMESTAMP NULL>, the key and value can specify + * a suffix to indicate if the type is nullable, default is nullable. + * + *

    MAP type does not belong to standard SQL. */ -public class SqlMapType extends SqlIdentifier implements ExtendedSqlType { +public class SqlMapTypeNameSpec extends SqlTypeNameSpec { private final SqlDataTypeSpec keyType; private final SqlDataTypeSpec valType; - public SqlMapType(SqlParserPos pos, SqlDataTypeSpec keyType, SqlDataTypeSpec valType) { - super(SqlTypeName.MAP.getName(), pos); + /** + * Creates a {@code SqlMapTypeNameSpec}. + * + * @param keyType key type + * @param valType value type + * @param pos the parser position + */ + public SqlMapTypeNameSpec( + SqlDataTypeSpec keyType, + SqlDataTypeSpec valType, + SqlParserPos pos) { + super(new SqlIdentifier(SqlTypeName.MAP.getName(), pos), pos); this.keyType = keyType; this.valType = valType; } @@ -46,14 +59,45 @@ public class SqlMapType extends SqlIdentifier implements ExtendedSqlType { return valType; } + @Override + public RelDataType deriveType(SqlValidator validator) { + return validator.getTypeFactory() + .createMapType( + keyType.deriveType(validator), + valType.deriveType(validator)); + } + @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { writer.keyword("MAP"); SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "<", ">"); writer.sep(","); - ExtendedSqlType.unparseType(keyType, writer, leftPrec, rightPrec); + keyType.unparse(writer, leftPrec, rightPrec); + // Default is nullable. + if (!keyType.getNullable()) { + writer.keyword("NOT NULL"); + } writer.sep(","); - ExtendedSqlType.unparseType(valType, writer, leftPrec, rightPrec); + valType.unparse(writer, leftPrec, rightPrec); + // Default is nullable. + if (!valType.getNullable()) { + writer.keyword("NOT NULL"); + } writer.endList(frame); } + + @Override + public boolean equalsDeep(SqlTypeNameSpec spec, Litmus litmus) { + if (!(spec instanceof SqlMapTypeNameSpec)) { + return litmus.fail("{} != {}", this, spec); + } + SqlMapTypeNameSpec that = (SqlMapTypeNameSpec) spec; + if (!this.keyType.equalsDeep(that.keyType, litmus)) { + return litmus.fail("{} != {}", this, spec); + } + if (!this.valType.equalsDeep(that.valType, litmus)) { + return litmus.fail("{} != {}", this, spec); + } + return litmus.succeed(); + } } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/package-info.java b/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/package-info.java new file mode 100644 index 0000000000000000000000000000000000000000..439912b52b4d827952ae30866b715f5489e9748d --- /dev/null +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/calcite/sql/package-info.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package is needed because the constructor of SqlTypeNameSpec is package scope, + * we should merge this package into org.apache.flink.sql.parser.type when CALCITE-3360 + * is resolved. + */ +@PackageMarker +package org.apache.calcite.sql; + +import org.apache.calcite.avatica.util.PackageMarker; diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ExtendedSqlNode.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ExtendedSqlNode.java index df541922b062dbfd017f47ae9ee1a2665336668f..5c5c69be47e9fc694ec6ddb84046a22966ef201a 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ExtendedSqlNode.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ExtendedSqlNode.java @@ -18,7 +18,7 @@ package org.apache.flink.sql.parser; -import org.apache.flink.sql.parser.error.SqlParseException; +import org.apache.flink.sql.parser.error.SqlValidateException; /** An remark interface which should be inherited by extended sql nodes which are not supported * by Calcite core parser. @@ -27,5 +27,5 @@ import org.apache.flink.sql.parser.error.SqlParseException; * {@link org.apache.calcite.sql.validate.SqlValidatorImpl}. */ public interface ExtendedSqlNode { - void validate() throws SqlParseException; + void validate() throws SqlValidateException; } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/FlinkSqlDataTypeSpec.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/FlinkSqlDataTypeSpec.java deleted file mode 100644 index a3797d7a5d4e756dcb1bd3ed7e205fa68683cf6b..0000000000000000000000000000000000000000 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/FlinkSqlDataTypeSpec.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.sql.parser; - -import org.apache.flink.sql.parser.type.ExtendedSqlType; -import org.apache.flink.sql.parser.type.SqlArrayType; -import org.apache.flink.sql.parser.type.SqlBytesType; -import org.apache.flink.sql.parser.type.SqlMapType; -import org.apache.flink.sql.parser.type.SqlMultisetType; -import org.apache.flink.sql.parser.type.SqlRowType; -import org.apache.flink.sql.parser.type.SqlStringType; -import org.apache.flink.sql.parser.type.SqlTimeType; -import org.apache.flink.sql.parser.type.SqlTimestampType; - -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rel.type.RelDataTypeFactory; -import org.apache.calcite.sql.SqlCollation; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlIdentifier; -import org.apache.calcite.sql.SqlNode; -import org.apache.calcite.sql.SqlUtil; -import org.apache.calcite.sql.SqlWriter; -import org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.calcite.sql.type.SqlTypeName; -import org.apache.calcite.sql.type.SqlTypeUtil; -import org.apache.calcite.util.Util; - -import java.nio.charset.Charset; -import java.util.Objects; -import java.util.TimeZone; -import java.util.stream.Collectors; - -/** - * Represents a SQL data type specification in a parse tree. - * - *

    A SqlDataTypeSpec is immutable; once created, you cannot - * change any of the fields.

    - * - *

    This class is an extension to {@link SqlDataTypeSpec}, we support - * complex type expressions like:

    - * - *
    ROW(
    - * foo NUMBER(5, 2) NOT NULL,
    - * rec ROW(b BOOLEAN, i MyUDT NOT NULL))
    - * - *

    Until CALCITE-3213 - * is resolved, we can remove this class. - */ -public class FlinkSqlDataTypeSpec extends SqlDataTypeSpec { - // Flag saying if the element type is nullable if this type is a collection type. - // For collection type, we mean ARRAY and MULTISET type now. - private Boolean elementNullable; - - public FlinkSqlDataTypeSpec( - SqlIdentifier collectionsTypeName, - SqlIdentifier typeName, - int precision, - int scale, - String charSetName, - Boolean nullable, - Boolean elementNullable, - SqlParserPos pos) { - super(collectionsTypeName, typeName, precision, scale, - charSetName, null, nullable, pos); - this.elementNullable = elementNullable; - } - - public FlinkSqlDataTypeSpec( - SqlIdentifier collectionsTypeName, - SqlIdentifier typeName, - int precision, - int scale, - String charSetName, - TimeZone timeZone, - Boolean nullable, - Boolean elementNullable, - SqlParserPos pos) { - super(collectionsTypeName, typeName, precision, scale, - charSetName, timeZone, nullable, pos); - this.elementNullable = elementNullable; - } - - public FlinkSqlDataTypeSpec( - SqlIdentifier typeName, - int precision, - int scale, - String charSetName, - TimeZone timeZone, - Boolean nullable, - Boolean elementNullable, - SqlParserPos pos) { - super(null, typeName, precision, scale, - charSetName, timeZone, nullable, pos); - this.elementNullable = elementNullable; - } - - @Override - public SqlNode clone(SqlParserPos pos) { - return (getCollectionsTypeName() != null) - ? new FlinkSqlDataTypeSpec(getCollectionsTypeName(), getTypeName(), getPrecision(), - getScale(), getCharSetName(), getNullable(), this.elementNullable, pos) - : new FlinkSqlDataTypeSpec(getTypeName(), getPrecision(), getScale(), - getCharSetName(), getTimeZone(), getNullable(), this.elementNullable, pos); - } - - /** Returns a copy of this data type specification with a given - * nullability. */ - @Override - public SqlDataTypeSpec withNullable(Boolean nullable) { - if (Objects.equals(nullable, this.getNullable())) { - return this; - } - return new FlinkSqlDataTypeSpec(getCollectionsTypeName(), getTypeName(), - getPrecision(), getScale(), getCharSetName(), getTimeZone(), nullable, - this.elementNullable, getParserPosition()); - } - - @Override - public RelDataType deriveType(RelDataTypeFactory typeFactory) { - // Default to be nullable. - return this.deriveType(typeFactory, true); - } - - @Override - public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - final SqlIdentifier typeName = getTypeName(); - String name = typeName.getSimple(); - if (typeName instanceof ExtendedSqlType) { - typeName.unparse(writer, leftPrec, rightPrec); - } else if (SqlTypeName.get(name) != null) { - SqlTypeName sqlTypeName = SqlTypeName.get(name); - writer.keyword(name); - if (sqlTypeName.allowsPrec() && this.getPrecision() >= 0) { - SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "(", ")"); - writer.print(this.getPrecision()); - if (sqlTypeName.allowsScale() && this.getScale() >= 0) { - writer.sep(",", true); - writer.print(this.getScale()); - } - - writer.endList(frame); - } - - if (this.getCharSetName() != null) { - writer.keyword("CHARACTER SET"); - writer.identifier(this.getCharSetName(), false); - } - - if (this.getCollectionsTypeName() != null) { - // Fix up nullable attribute if this is a collection type. - if (elementNullable != null && !elementNullable) { - writer.keyword("NOT NULL"); - } - writer.keyword(this.getCollectionsTypeName().getSimple()); - } - } else if (name.startsWith("_")) { - writer.keyword(name.substring(1)); - } else { - this.getTypeName().unparse(writer, leftPrec, rightPrec); - } - if (getNullable() != null && !getNullable()) { - writer.keyword("NOT NULL"); - } - } - - @Override - public RelDataType deriveType(RelDataTypeFactory typeFactory, boolean nullable) { - final SqlIdentifier typeName = getTypeName(); - if (!typeName.isSimple()) { - return null; - } - final String name = typeName.getSimple(); - final SqlTypeName sqlTypeName = SqlTypeName.get(name); - // Try to get Flink custom data type first. - RelDataType type = getExtendedType(typeFactory, typeName); - if (type == null) { - if (sqlTypeName == null) { - return null; - } else { - // NOTE jvs 15-Jan-2009: earlier validation is supposed to - // have caught these, which is why it's OK for them - // to be assertions rather than user-level exceptions. - final int precision = getPrecision(); - final int scale = getScale(); - if ((precision >= 0) && (scale >= 0)) { - assert sqlTypeName.allowsPrecScale(true, true); - type = typeFactory.createSqlType(sqlTypeName, precision, scale); - } else if (precision >= 0) { - assert sqlTypeName.allowsPrecNoScale(); - type = typeFactory.createSqlType(sqlTypeName, precision); - } else { - assert sqlTypeName.allowsNoPrecNoScale(); - type = typeFactory.createSqlType(sqlTypeName); - } - } - } - - if (SqlTypeUtil.inCharFamily(type)) { - // Applying Syntax rule 10 from SQL:99 spec section 6.22 "If TD is a - // fixed-length, variable-length or large object character string, - // then the collating sequence of the result of the is the default collating sequence for the - // character repertoire of TD and the result of the has the Coercible coercibility characteristic." - SqlCollation collation = SqlCollation.COERCIBLE; - - Charset charset; - final String charSetName = getCharSetName(); - if (null == charSetName) { - charset = typeFactory.getDefaultCharset(); - } else { - String javaCharSetName = - Objects.requireNonNull( - SqlUtil.translateCharacterSetName(charSetName), charSetName); - charset = Charset.forName(javaCharSetName); - } - type = - typeFactory.createTypeWithCharsetAndCollation( - type, - charset, - collation); - } - - final SqlIdentifier collectionsTypeName = getCollectionsTypeName(); - if (null != collectionsTypeName) { - // Fix the nullability of the element type first. - boolean elementNullable = true; - if (this.elementNullable != null) { - elementNullable = this.elementNullable; - } - type = typeFactory.createTypeWithNullability(type, elementNullable); - - final String collectionName = collectionsTypeName.getSimple(); - final SqlTypeName collectionsSqlTypeName = - Objects.requireNonNull(SqlTypeName.get(collectionName), - collectionName); - - switch (collectionsSqlTypeName) { - case MULTISET: - type = typeFactory.createMultisetType(type, -1); - break; - case ARRAY: - type = typeFactory.createArrayType(type, -1); - break; - default: - throw Util.unexpected(collectionsSqlTypeName); - } - } - - // Fix the nullability of this type. - if (this.getNullable() != null) { - nullable = this.getNullable(); - } - type = typeFactory.createTypeWithNullability(type, nullable); - - return type; - } - - private RelDataType getExtendedType(RelDataTypeFactory typeFactory, SqlIdentifier typeName) { - // quick check. - if (!(typeName instanceof ExtendedSqlType)) { - return null; - } - if (typeName instanceof SqlBytesType) { - return typeFactory.createSqlType(SqlTypeName.VARBINARY, Integer.MAX_VALUE); - } else if (typeName instanceof SqlStringType) { - return typeFactory.createSqlType(SqlTypeName.VARCHAR, Integer.MAX_VALUE); - } else if (typeName instanceof SqlArrayType) { - final SqlArrayType arrayType = (SqlArrayType) typeName; - return typeFactory.createArrayType(arrayType.getElementType() - .deriveType(typeFactory), -1); - } else if (typeName instanceof SqlMultisetType) { - final SqlMultisetType multiSetType = (SqlMultisetType) typeName; - return typeFactory.createMultisetType(multiSetType.getElementType() - .deriveType(typeFactory), -1); - } else if (typeName instanceof SqlMapType) { - final SqlMapType mapType = (SqlMapType) typeName; - return typeFactory.createMapType( - mapType.getKeyType().deriveType(typeFactory), - mapType.getValType().deriveType(typeFactory)); - } else if (typeName instanceof SqlRowType) { - final SqlRowType rowType = (SqlRowType) typeName; - return typeFactory.createStructType( - rowType.getFieldTypes().stream().map(ft -> ft.deriveType(typeFactory)) - .collect(Collectors.toList()), - rowType.getFieldNames().stream().map(SqlIdentifier::getSimple) - .collect(Collectors.toList())); - } else if (typeName instanceof SqlTimeType) { - final SqlTimeType zonedTimeType = (SqlTimeType) typeName; - if (zonedTimeType.getPrecision() >= 0) { - return typeFactory.createSqlType(zonedTimeType.getSqlTypeName(), - zonedTimeType.getPrecision()); - } else { - // Use default precision. - return typeFactory.createSqlType(zonedTimeType.getSqlTypeName()); - } - } else if (typeName instanceof SqlTimestampType) { - final SqlTimestampType zonedTimestampType = (SqlTimestampType) typeName; - if (zonedTimestampType.getPrecision() >= 0) { - return typeFactory.createSqlType(zonedTimestampType.getSqlTypeName(), - zonedTimestampType.getPrecision()); - } else { - // Use default precision. - return typeFactory.createSqlType(zonedTimestampType.getSqlTypeName()); - } - } - return null; - } -} diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateTable.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateTable.java index 183e2d8e7e45979ca9a7f35d23dde20a01c4a1e5..ba0763938d62c198873cd12c2578f76c145a12c2 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateTable.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateTable.java @@ -19,7 +19,7 @@ package org.apache.flink.sql.parser.ddl; import org.apache.flink.sql.parser.ExtendedSqlNode; -import org.apache.flink.sql.parser.error.SqlParseException; +import org.apache.flink.sql.parser.error.SqlValidateException; import org.apache.calcite.sql.SqlBasicCall; import org.apache.calcite.sql.SqlCall; @@ -37,9 +37,12 @@ import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.pretty.SqlPrettyWriter; import org.apache.calcite.util.ImmutableNullableList; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Set; import static java.util.Objects.requireNonNull; @@ -63,6 +66,7 @@ public class SqlCreateTable extends SqlCreate implements ExtendedSqlNode { private final SqlNodeList partitionKeyList; + @Nullable private final SqlCharStringLiteral comment; public SqlCreateTable( @@ -75,12 +79,12 @@ public class SqlCreateTable extends SqlCreate implements ExtendedSqlNode { SqlNodeList partitionKeyList, SqlCharStringLiteral comment) { super(OPERATOR, pos, false, false); - this.tableName = requireNonNull(tableName, "Table name is missing"); - this.columnList = requireNonNull(columnList, "Column list should not be null"); - this.primaryKeyList = primaryKeyList; - this.uniqueKeysList = uniqueKeysList; - this.propertyList = propertyList; - this.partitionKeyList = partitionKeyList; + this.tableName = requireNonNull(tableName, "tableName should not be null"); + this.columnList = requireNonNull(columnList, "columnList should not be null"); + this.primaryKeyList = requireNonNull(primaryKeyList, "primayKeyList should not be null"); + this.uniqueKeysList = requireNonNull(uniqueKeysList, "uniqueKeysList should not be null"); + this.propertyList = requireNonNull(propertyList, "propertyList should not be null"); + this.partitionKeyList = requireNonNull(partitionKeyList, "partitionKeyList should not be null"); this.comment = comment; } @@ -119,70 +123,62 @@ public class SqlCreateTable extends SqlCreate implements ExtendedSqlNode { return uniqueKeysList; } - public SqlCharStringLiteral getComment() { - return comment; + public Optional getComment() { + return Optional.ofNullable(comment); } public boolean isIfNotExists() { return ifNotExists; } - public void validate() throws SqlParseException { + public void validate() throws SqlValidateException { Set columnNames = new HashSet<>(); - if (columnList != null) { - for (SqlNode column : columnList) { - String columnName = null; - if (column instanceof SqlTableColumn) { - SqlTableColumn tableColumn = (SqlTableColumn) column; - columnName = tableColumn.getName().getSimple(); - } else if (column instanceof SqlBasicCall) { - SqlBasicCall tableColumn = (SqlBasicCall) column; - columnName = tableColumn.getOperands()[1].toString(); - } + for (SqlNode column : columnList) { + String columnName = null; + if (column instanceof SqlTableColumn) { + SqlTableColumn tableColumn = (SqlTableColumn) column; + columnName = tableColumn.getName().getSimple(); + } else if (column instanceof SqlBasicCall) { + SqlBasicCall tableColumn = (SqlBasicCall) column; + columnName = tableColumn.getOperands()[1].toString(); + } - if (!columnNames.add(columnName)) { - throw new SqlParseException( - column.getParserPosition(), - "Duplicate column name [" + columnName + "], at " + - column.getParserPosition()); - } + if (!columnNames.add(columnName)) { + throw new SqlValidateException( + column.getParserPosition(), + "Duplicate column name [" + columnName + "], at " + + column.getParserPosition()); } } - if (this.primaryKeyList != null) { - for (SqlNode primaryKeyNode : this.primaryKeyList) { - String primaryKey = ((SqlIdentifier) primaryKeyNode).getSimple(); - if (!columnNames.contains(primaryKey)) { - throw new SqlParseException( - primaryKeyNode.getParserPosition(), - "Primary key [" + primaryKey + "] not defined in columns, at " + - primaryKeyNode.getParserPosition()); - } + for (SqlNode primaryKeyNode : this.primaryKeyList) { + String primaryKey = ((SqlIdentifier) primaryKeyNode).getSimple(); + if (!columnNames.contains(primaryKey)) { + throw new SqlValidateException( + primaryKeyNode.getParserPosition(), + "Primary key [" + primaryKey + "] not defined in columns, at " + + primaryKeyNode.getParserPosition()); } } - if (this.uniqueKeysList != null) { - for (SqlNodeList uniqueKeys: this.uniqueKeysList) { - for (SqlNode uniqueKeyNode : uniqueKeys) { - String uniqueKey = ((SqlIdentifier) uniqueKeyNode).getSimple(); - if (!columnNames.contains(uniqueKey)) { - throw new SqlParseException( - uniqueKeyNode.getParserPosition(), - "Unique key [" + uniqueKey + "] not defined in columns, at " + uniqueKeyNode.getParserPosition()); - } + for (SqlNodeList uniqueKeys: this.uniqueKeysList) { + for (SqlNode uniqueKeyNode : uniqueKeys) { + String uniqueKey = ((SqlIdentifier) uniqueKeyNode).getSimple(); + if (!columnNames.contains(uniqueKey)) { + throw new SqlValidateException( + uniqueKeyNode.getParserPosition(), + "Unique key [" + uniqueKey + "] not defined in columns, at " + uniqueKeyNode.getParserPosition()); } } } - if (this.partitionKeyList != null) { - for (SqlNode partitionKeyNode : this.partitionKeyList.getList()) { - String partitionKey = ((SqlIdentifier) partitionKeyNode).getSimple(); - if (!columnNames.contains(partitionKey)) { - throw new SqlParseException( - partitionKeyNode.getParserPosition(), - "Partition column [" + partitionKey + "] not defined in columns, at " - + partitionKeyNode.getParserPosition()); - } + for (SqlNode partitionKeyNode : this.partitionKeyList.getList()) { + String partitionKey = ((SqlIdentifier) partitionKeyNode).getSimple(); + if (!columnNames.contains(partitionKey)) { + throw new SqlValidateException( + partitionKeyNode.getParserPosition(), + "Partition column [" + partitionKey + "] not defined in columns, at " + + partitionKeyNode.getParserPosition()); } } @@ -254,14 +250,14 @@ public class SqlCreateTable extends SqlCreate implements ExtendedSqlNode { column.unparse(writer, leftPrec, rightPrec); } } - if (primaryKeyList != null && primaryKeyList.size() > 0) { + if (primaryKeyList.size() > 0) { printIndent(writer); writer.keyword("PRIMARY KEY"); SqlWriter.Frame keyFrame = writer.startList("(", ")"); primaryKeyList.unparse(writer, leftPrec, rightPrec); writer.endList(keyFrame); } - if (uniqueKeysList != null && uniqueKeysList.size() > 0) { + if (uniqueKeysList.size() > 0) { printIndent(writer); for (SqlNodeList uniqueKeyList : uniqueKeysList) { writer.keyword("UNIQUE"); @@ -280,7 +276,7 @@ public class SqlCreateTable extends SqlCreate implements ExtendedSqlNode { comment.unparse(writer, leftPrec, rightPrec); } - if (this.partitionKeyList != null && this.partitionKeyList.size() > 0) { + if (this.partitionKeyList.size() > 0) { writer.newlineAndIndent(); writer.keyword("PARTITIONED BY"); SqlWriter.Frame withFrame = writer.startList("(", ")"); @@ -289,7 +285,7 @@ public class SqlCreateTable extends SqlCreate implements ExtendedSqlNode { writer.newlineAndIndent(); } - if (propertyList != null) { + if (this.propertyList.size() > 0) { writer.keyword("WITH"); SqlWriter.Frame withFrame = writer.startList("(", ")"); for (SqlNode property : propertyList) { @@ -312,7 +308,7 @@ public class SqlCreateTable extends SqlCreate implements ExtendedSqlNode { */ public static class TableCreationContext { public List columnList = new ArrayList<>(); - public SqlNodeList primaryKeyList; + public SqlNodeList primaryKeyList = SqlNodeList.EMPTY; public List uniqueKeysList = new ArrayList<>(); } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java index b77af04832209313575d4dfb5db2a0ec8e0dd5bf..868592a7de6bf7cfc262fb4e0393ada471871732 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlCreateView.java @@ -19,7 +19,7 @@ package org.apache.flink.sql.parser.ddl; import org.apache.flink.sql.parser.ExtendedSqlNode; -import org.apache.flink.sql.parser.error.SqlParseException; +import org.apache.flink.sql.parser.error.SqlValidateException; import org.apache.calcite.sql.SqlCharStringLiteral; import org.apache.calcite.sql.SqlCreate; @@ -32,8 +32,13 @@ import org.apache.calcite.sql.SqlSpecialOperator; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.parser.SqlParserPos; +import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; /** * CREATE VIEW DDL sql call. @@ -44,6 +49,8 @@ public class SqlCreateView extends SqlCreate implements ExtendedSqlNode { private final SqlIdentifier viewName; private final SqlNodeList fieldList; private final SqlNode query; + + @Nullable private final SqlCharStringLiteral comment; public SqlCreateView( @@ -54,9 +61,9 @@ public class SqlCreateView extends SqlCreate implements ExtendedSqlNode { boolean replace, SqlCharStringLiteral comment) { super(OPERATOR, pos, replace, false); - this.viewName = viewName; - this.fieldList = fieldList; - this.query = query; + this.viewName = requireNonNull(viewName, "viewName should not be null"); + this.fieldList = requireNonNull(fieldList, "fieldList should not be null"); + this.query = requireNonNull(query, "query should not be null"); this.comment = comment; } @@ -82,8 +89,8 @@ public class SqlCreateView extends SqlCreate implements ExtendedSqlNode { return query; } - public SqlCharStringLiteral getComment() { - return comment; + public Optional getComment() { + return Optional.ofNullable(comment); } @Override @@ -115,7 +122,7 @@ public class SqlCreateView extends SqlCreate implements ExtendedSqlNode { } @Override - public void validate() throws SqlParseException { + public void validate() throws SqlValidateException { // no-op } } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlTableColumn.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlTableColumn.java index db9594be4044fed7ff184fadd3ce9e439f0c5aac..83e7376f2061d765286c6144cd09c0306fbddf7a 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlTableColumn.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/ddl/SqlTableColumn.java @@ -18,8 +18,6 @@ package org.apache.flink.sql.parser.ddl; -import org.apache.flink.sql.parser.type.ExtendedSqlType; - import org.apache.calcite.sql.SqlCall; import org.apache.calcite.sql.SqlCharStringLiteral; import org.apache.calcite.sql.SqlDataTypeSpec; @@ -32,7 +30,10 @@ import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.util.ImmutableNullableList; +import javax.annotation.Nullable; + import java.util.List; +import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -45,6 +46,8 @@ public class SqlTableColumn extends SqlCall { private SqlIdentifier name; private SqlDataTypeSpec type; + + @Nullable private SqlCharStringLiteral comment; public SqlTableColumn(SqlIdentifier name, @@ -71,7 +74,11 @@ public class SqlTableColumn extends SqlCall { public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { this.name.unparse(writer, leftPrec, rightPrec); writer.print(" "); - ExtendedSqlType.unparseType(type, writer, leftPrec, rightPrec); + this.type.unparse(writer, leftPrec, rightPrec); + if (!this.type.getNullable()) { + // Default is nullable. + writer.keyword("NOT NULL"); + } if (this.comment != null) { writer.print(" COMMENT "); this.comment.unparse(writer, leftPrec, rightPrec); @@ -94,8 +101,8 @@ public class SqlTableColumn extends SqlCall { this.type = type; } - public SqlCharStringLiteral getComment() { - return comment; + public Optional getComment() { + return Optional.ofNullable(comment); } public void setComment(SqlCharStringLiteral comment) { diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/dml/RichSqlInsert.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/dml/RichSqlInsert.java index 4b681b29554aa625dcdcd2b4a6429af7a44c1f65..5d7a16945769bf528044403929c3fafe6e6d4aac 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/dml/RichSqlInsert.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/dml/RichSqlInsert.java @@ -20,7 +20,7 @@ package org.apache.flink.sql.parser.dml; import org.apache.flink.sql.parser.ExtendedSqlNode; import org.apache.flink.sql.parser.SqlProperty; -import org.apache.flink.sql.parser.error.SqlParseException; +import org.apache.flink.sql.parser.error.SqlValidateException; import org.apache.calcite.sql.SqlInsert; import org.apache.calcite.sql.SqlInsertKeyword; @@ -142,7 +142,7 @@ public class RichSqlInsert extends SqlInsert implements ExtendedSqlNode { } @Override - public void validate() throws SqlParseException { + public void validate() throws SqlValidateException { // no-op } } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/error/SqlParseException.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/error/SqlValidateException.java similarity index 79% rename from flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/error/SqlParseException.java rename to flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/error/SqlValidateException.java index 365de20e725e291e78c309e417802c5307811bc6..9ee46d160bf480a69435a8cfef0d4fed5b74499f 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/error/SqlParseException.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/error/SqlValidateException.java @@ -21,22 +21,21 @@ package org.apache.flink.sql.parser.error; import org.apache.calcite.sql.parser.SqlParserPos; /** - * SQL parse Exception. This is a simpler version - * of Calcite {@link org.apache.calcite.sql.parser.SqlParseException} - * which is used for SqlNode validation. + * SQL parse Exception. This exception mainly throws during + * {@link org.apache.flink.sql.parser.ExtendedSqlNode} validation. */ -public class SqlParseException extends Exception { +public class SqlValidateException extends Exception { private SqlParserPos errorPosition; private String message; - public SqlParseException(SqlParserPos errorPosition, String message) { + public SqlValidateException(SqlParserPos errorPosition, String message) { this.errorPosition = errorPosition; this.message = message; } - public SqlParseException(SqlParserPos errorPosition, String message, Exception e) { + public SqlValidateException(SqlParserPos errorPosition, String message, Exception e) { super(e); this.errorPosition = errorPosition; this.message = message; diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlArrayType.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/ExtendedSqlBasicTypeNameSpec.java similarity index 53% rename from flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlArrayType.java rename to flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/ExtendedSqlBasicTypeNameSpec.java index 336f883732af8f3f6a759262c47a89f5d9a107cf..ec81a0dd6c16551d26356172ca42ff142674f5ff 100644 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/SqlArrayType.java +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/ExtendedSqlBasicTypeNameSpec.java @@ -7,7 +7,7 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -18,32 +18,40 @@ package org.apache.flink.sql.parser.type; -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlBasicTypeNameSpec; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.parser.SqlParserPos; import org.apache.calcite.sql.type.SqlTypeName; /** - * Parse column of ArrayType. + * A sql type name specification of extended basic data type, it has a counterpart basic + * sql type name but always represents as a special alias in Flink. + * + *

    For example, STRING is synonym of VARCHAR(INT_MAX) + * and BYTES is synonym of VARBINARY(INT_MAX). */ -public class SqlArrayType extends SqlIdentifier implements ExtendedSqlType { - - private final SqlDataTypeSpec elementType; - - public SqlArrayType(SqlParserPos pos, SqlDataTypeSpec elementType) { - super(SqlTypeName.ARRAY.getName(), pos); - this.elementType = elementType; - } +public class ExtendedSqlBasicTypeNameSpec extends SqlBasicTypeNameSpec { + // Type alias used for unparsing. + private final String typeAlias; - public SqlDataTypeSpec getElementType() { - return elementType; + /** + * Creates a {@code ExtendedSqlBuiltinTypeNameSpec} instance. + * + * @param typeName type name + * @param precision type precision + * @param pos parser position + */ + public ExtendedSqlBasicTypeNameSpec( + String typeAlias, + SqlTypeName typeName, + int precision, + SqlParserPos pos) { + super(typeName, precision, pos); + this.typeAlias = typeAlias; } @Override public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { - writer.keyword("ARRAY<"); - ExtendedSqlType.unparseType(this.elementType, writer, leftPrec, rightPrec); - writer.keyword(">"); + writer.keyword(typeAlias); } } diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/ExtendedSqlCollectionTypeNameSpec.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/ExtendedSqlCollectionTypeNameSpec.java new file mode 100644 index 0000000000000000000000000000000000000000..a69039231e4c49d203b9e95daa3b068703cebda2 --- /dev/null +++ b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/ExtendedSqlCollectionTypeNameSpec.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.sql.parser.type; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.sql.SqlCollectionTypeNameSpec; +import org.apache.calcite.sql.SqlTypeNameSpec; +import org.apache.calcite.sql.SqlWriter; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.util.Litmus; +import org.apache.calcite.util.Util; + +/** + * A extended sql type name specification of collection type, + * different with {@link SqlCollectionTypeNameSpec}, + * we support NULL or NOT NULL suffix for the element type name(this syntax + * does not belong to standard SQL). + */ +public class ExtendedSqlCollectionTypeNameSpec extends SqlCollectionTypeNameSpec { + private final boolean elementNullable; + private final SqlTypeName collectionTypeName; + private final boolean unparseAsStandard; + + /** + * Creates a {@code ExtendedSqlCollectionTypeNameSpec}. + * + * @param elementTypeName element type name specification + * @param elementNullable flag indicating if the element type is nullable + * @param collectionTypeName collection type name + * @param unparseAsStandard if we should unparse the collection type as standard SQL + * style + * @param pos the parser position + */ + public ExtendedSqlCollectionTypeNameSpec( + SqlTypeNameSpec elementTypeName, + boolean elementNullable, + SqlTypeName collectionTypeName, + boolean unparseAsStandard, + SqlParserPos pos) { + super(elementTypeName, collectionTypeName, pos); + this.elementNullable = elementNullable; + this.collectionTypeName = collectionTypeName; + this.unparseAsStandard = unparseAsStandard; + } + + @Override + public RelDataType deriveType(SqlValidator validator) { + RelDataType elementType = getElementTypeName().deriveType(validator); + elementType = validator.getTypeFactory() + .createTypeWithNullability(elementType, elementNullable); + return createCollectionType(elementType, validator.getTypeFactory()); + } + + @Override + public void unparse(SqlWriter writer, int leftPrec, int rightPrec) { + if (unparseAsStandard) { + this.getElementTypeName().unparse(writer, leftPrec, rightPrec); + // Default is nullable. + if (!elementNullable) { + writer.keyword("NOT NULL"); + } + writer.keyword(collectionTypeName.name()); + } else { + writer.keyword(collectionTypeName.name()); + SqlWriter.Frame frame = writer.startList(SqlWriter.FrameTypeEnum.FUN_CALL, "<", ">"); + + getElementTypeName().unparse(writer, leftPrec, rightPrec); + // Default is nullable. + if (!elementNullable) { + writer.keyword("NOT NULL"); + } + writer.endList(frame); + } + } + + @Override + public boolean equalsDeep(SqlTypeNameSpec spec, Litmus litmus) { + if (!(spec instanceof ExtendedSqlCollectionTypeNameSpec)) { + return litmus.fail("{} != {}", this, spec); + } + ExtendedSqlCollectionTypeNameSpec that = (ExtendedSqlCollectionTypeNameSpec) spec; + if (this.elementNullable != that.elementNullable) { + return litmus.fail("{} != {}", this, spec); + } + return super.equalsDeep(spec, litmus); + } + + //~ Tools ------------------------------------------------------------------ + + /** + * Create collection data type. + * + * @param elementType Type of the collection element + * @param typeFactory Type factory + * @return The collection data type, or throw exception if the collection + * type name does not belong to {@code SqlTypeName} enumerations + */ + private RelDataType createCollectionType(RelDataType elementType, + RelDataTypeFactory typeFactory) { + switch (collectionTypeName) { + case MULTISET: + return typeFactory.createMultisetType(elementType, -1); + case ARRAY: + return typeFactory.createArrayType(elementType, -1); + + default: + throw Util.unexpected(collectionTypeName); + } + } +} diff --git a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/ExtendedSqlType.java b/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/ExtendedSqlType.java deleted file mode 100644 index f27656db62aff271cd596079f9f5b4e6774a1d92..0000000000000000000000000000000000000000 --- a/flink-table/flink-sql-parser/src/main/java/org/apache/flink/sql/parser/type/ExtendedSqlType.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.sql.parser.type; - -import org.apache.calcite.sql.SqlDataTypeSpec; -import org.apache.calcite.sql.SqlWriter; - -/** An remark interface which should be inherited by supported sql types which are not supported - * by Calcite default parser. - * - *

    Caution that the subclass must override the method - * {@link org.apache.calcite.sql.SqlNode#unparse(SqlWriter, int, int)}. - */ -public interface ExtendedSqlType { - - static void unparseType(SqlDataTypeSpec type, - SqlWriter writer, - int leftPrec, - int rightPrec) { - if (type.getTypeName() instanceof ExtendedSqlType) { - type.getTypeName().unparse(writer, leftPrec, rightPrec); - } else { - type.unparse(writer, leftPrec, rightPrec); - } - } -} diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkDDLDataTypeTest.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkDDLDataTypeTest.java index 410e30de6ee0aa35b5a1956f947075378350d577..f5edc4f4fc963e92f3eeb8540600f5c844e6bc12 100644 --- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkDDLDataTypeTest.java +++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkDDLDataTypeTest.java @@ -21,10 +21,11 @@ package org.apache.flink.sql.parser; import org.apache.flink.sql.parser.ddl.SqlCreateTable; import org.apache.flink.sql.parser.ddl.SqlTableColumn; import org.apache.flink.sql.parser.impl.FlinkSqlParserImpl; -import org.apache.flink.sql.parser.validate.FlinkSqlConformance; import org.apache.calcite.avatica.util.Casing; import org.apache.calcite.avatica.util.Quoting; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.rel.type.DelegatingTypeSystem; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rel.type.RelDataTypeSystem; @@ -32,16 +33,27 @@ import org.apache.calcite.sql.SqlDataTypeSpec; import org.apache.calcite.sql.SqlDialect; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperatorTable; import org.apache.calcite.sql.SqlWriter; import org.apache.calcite.sql.dialect.CalciteSqlDialect; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.parser.SqlParseException; import org.apache.calcite.sql.parser.SqlParser; +import org.apache.calcite.sql.parser.SqlParserImplFactory; import org.apache.calcite.sql.parser.SqlParserTest; import org.apache.calcite.sql.parser.SqlParserUtil; import org.apache.calcite.sql.pretty.SqlPrettyWriter; -import org.apache.calcite.sql.type.SqlTypeFactoryImpl; +import org.apache.calcite.sql.test.SqlTestFactory; import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlConformance; +import org.apache.calcite.sql.validate.SqlConformanceEnum; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorCatalogReader; +import org.apache.calcite.sql.validate.SqlValidatorUtil; +import org.apache.calcite.test.MockSqlOperatorTable; import org.apache.calcite.test.SqlValidatorTestCase; +import org.apache.calcite.test.catalog.MockCatalogReaderSimple; +import org.apache.calcite.util.SourceStringReader; import org.apache.calcite.util.Util; import org.junit.Test; import org.junit.runner.RunWith; @@ -51,19 +63,19 @@ import javax.annotation.Nullable; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; /** - * Tests for all the sup[ported flink DDL data types. + * Tests for all the supported Flink DDL data types. */ @RunWith(Parameterized.class) public class FlinkDDLDataTypeTest { - private FlinkSqlConformance conformance = FlinkSqlConformance.DEFAULT; - private static final RelDataTypeFactory TYPE_FACTORY = - new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT); - private static final Fixture FIXTURE = new Fixture(TYPE_FACTORY); + private static final Fixture FIXTURE = + new Fixture(TestFactory.INSTANCE.getTypeFactory()); private static final String DDL_FORMAT = "create table t1 (\n" + " f0 %s\n" + ") with (\n" + @@ -171,7 +183,7 @@ public class FlinkDDLDataTypeTest { nullable(FIXTURE.createStructType( Arrays.asList(FIXTURE.intType, nullable(FIXTURE.booleanType)), Arrays.asList("f0", "f1"))), - "ROW< `f0` INTEGER NOT NULL, `f1` BOOLEAN >"), + "ROW(`f0` INTEGER NOT NULL, `f1` BOOLEAN)"), createTestItem("ROW<`f0` INT>", nullable(FIXTURE.createStructType( Collections.singletonList(nullable(FIXTURE.intType)), @@ -181,7 +193,7 @@ public class FlinkDDLDataTypeTest { nullable(FIXTURE.createStructType( Collections.singletonList(nullable(FIXTURE.intType)), Collections.singletonList("f0"))), - "ROW< `f0` INTEGER >"), + "ROW(`f0` INTEGER)"), createTestItem("ROW<>", nullable(FIXTURE.createStructType( Collections.emptyList(), @@ -191,7 +203,7 @@ public class FlinkDDLDataTypeTest { nullable(FIXTURE.createStructType( Collections.emptyList(), Collections.emptyList())), - "ROW<>"), + "ROW()"), createTestItem("ROW", nullable(FIXTURE.createStructType( @@ -200,25 +212,19 @@ public class FlinkDDLDataTypeTest { "ROW< `f0` INTEGER NOT NULL 'This is a comment.', " + "`f1` BOOLEAN 'This as well.' >"), - // test parse throws error. - createTestItem("TIMESTAMP WITH TIME ZONE", - "'WITH TIME ZONE' is not supported yet, options: " - + "'WITHOUT TIME ZONE', 'WITH LOCAL TIME ZONE'."), - createTestItem("TIMESTAMP(3) WITH TIME ZONE", - "'WITH TIME ZONE' is not supported yet, options: " - + "'WITHOUT TIME ZONE', 'WITH LOCAL TIME ZONE'."), + // Test parse throws error. + createTestItem("TIMESTAMP WITH ^TIME^ ZONE", + "(?s).*Encountered \"TIME\" at .*"), + createTestItem("TIMESTAMP(3) WITH ^TIME^ ZONE", + "(?s).*Encountered \"TIME\" at .*"), createTestItem("^NULL^", "(?s).*Encountered \"NULL\" at line 2, column 6..*"), - createTestItem("cat.db.MyType", - "(?s).*UDT in DDL is not supported yet..*"), - createTestItem("`db`.`MyType`", - "(?s).*UDT in DDL is not supported yet..*"), - createTestItem("MyType", - "(?s).*UDT in DDL is not supported yet..*"), - createTestItem("ARRAY", - "(?s).*UDT in DDL is not supported yet..*"), - createTestItem("ROW", - "(?s).*UDT in DDL is not supported yet..*"), + createTestItem("cat.db.MyType", null, "`cat`.`db`.`MyType`"), + createTestItem("`db`.`MyType`", null, "`db`.`MyType`"), + createTestItem("MyType", null, "`MyType`"), + createTestItem("ARRAY", null, "ARRAY< `MyType` >"), + createTestItem("ROW", null, + "ROW< `f0` `MyType`, `f1` `c`.`d`.`t` >"), createTestItem("^INTERVAL^ YEAR", "(?s).*Encountered \"INTERVAL\" at line 2, column 6..*"), createTestItem("ANY(^'unknown.class'^, '')", @@ -284,7 +290,7 @@ public class FlinkDDLDataTypeTest { } private Tester getTester() { - return new TesterImpl(); + return new TesterImpl(TestFactory.INSTANCE); } private Sql sql(String sql) { @@ -369,23 +375,10 @@ public class FlinkDDLDataTypeTest { * Default implementation of {@link SqlParserTest.Tester}. */ protected class TesterImpl implements Tester { - private SqlParser getSqlParser(String sql) { - return SqlParser.create(sql, - SqlParser.configBuilder() - .setParserFactory(FlinkSqlParserImpl.FACTORY) - .setQuoting(Quoting.BACK_TICK) - .setUnquotedCasing(Casing.UNCHANGED) - .setQuotedCasing(Casing.UNCHANGED) - .setConformance(conformance) - .build()); - } + private TestFactory factory; - private SqlDialect getSqlDialect() { - return new CalciteSqlDialect(SqlDialect.EMPTY_CONTEXT - .withQuotedCasing(Casing.UNCHANGED) - .withConformance(conformance) - .withUnquotedCasing(Casing.UNCHANGED) - .withIdentifierQuoteString("`")); + TesterImpl(TestFactory factory) { + this.factory = factory; } public void checkType(String sql, RelDataType type) { @@ -395,14 +388,14 @@ public class FlinkDDLDataTypeTest { SqlNodeList columns = sqlCreateTable.getColumnList(); assert columns.size() == 1; RelDataType columnType = ((SqlTableColumn) columns.get(0)).getType() - .deriveType(TYPE_FACTORY); + .deriveType(factory.getValidator()); assertEquals(type, columnType); } private SqlNode parseStmtAndHandleEx(String sql) { final SqlNode sqlNode; try { - sqlNode = getSqlParser(sql).parseStmt(); + sqlNode = factory.createParser(sql).parseStmt(); } catch (SqlParseException e) { throw new RuntimeException("Error while parsing SQL: " + sql, e); } @@ -416,7 +409,7 @@ public class FlinkDDLDataTypeTest { Throwable thrown = null; try { final SqlNode sqlNode; - sqlNode = getSqlParser(sap.sql).parseStmt(); + sqlNode = factory.createParser(sap.sql).parseStmt(); Util.discard(sqlNode); } catch (Throwable ex) { thrown = ex; @@ -432,8 +425,14 @@ public class FlinkDDLDataTypeTest { SqlNodeList columns = sqlCreateTable.getColumnList(); assert columns.size() == 1; SqlDataTypeSpec dataTypeSpec = ((SqlTableColumn) columns.get(0)).getType(); - SqlWriter sqlWriter = new SqlPrettyWriter(getSqlDialect(), false); + SqlWriter sqlWriter = new SqlPrettyWriter(factory.createSqlDialect(), false); dataTypeSpec.unparse(sqlWriter, 0, 0); + // SqlDataTypeSpec does not take care of the nullable attribute unparse, + // So we unparse nullable attribute specifically, this unparsing logic should + // keep sync with SqlTableColumn. + if (!dataTypeSpec.getNullable()) { + sqlWriter.keyword("NOT NULL"); + } assertEquals(expectedUnparsed, sqlWriter.toSqlString().getSql()); } @@ -443,4 +442,115 @@ public class FlinkDDLDataTypeTest { SqlValidatorTestCase.checkEx(thrown, expectedMsgPattern, sap); } } + + /** + * Factory to supply test components. + */ + private static class TestFactory { + static final Map DEFAULT_OPTIONS = buildDefaultOptions(); + + public static final TestFactory INSTANCE = new TestFactory(); + + private final Map options; + private final SqlTestFactory.ValidatorFactory validatorFactory; + + private final RelDataTypeFactory typeFactory; + private final SqlOperatorTable operatorTable; + private final SqlValidatorCatalogReader catalogReader; + private final SqlParser.Config parserConfig; + + TestFactory() { + this(DEFAULT_OPTIONS, MockCatalogReaderSimple::new, SqlValidatorUtil::newValidator); + } + + TestFactory( + Map options, + SqlTestFactory.MockCatalogReaderFactory catalogReaderFactory, + SqlTestFactory.ValidatorFactory validatorFactory) { + this.options = options; + this.validatorFactory = validatorFactory; + this.operatorTable = + createOperatorTable((SqlOperatorTable) options.get("operatorTable")); + this.typeFactory = createTypeFactory((SqlConformance) options.get("conformance")); + Boolean caseSensitive = (Boolean) options.get("caseSensitive"); + this.catalogReader = catalogReaderFactory.create(typeFactory, caseSensitive).init(); + this.parserConfig = createParserConfig(options); + } + + public SqlParser createParser(String sql) { + return SqlParser.create(new SourceStringReader(sql), parserConfig); + } + + public SqlDialect createSqlDialect() { + return new CalciteSqlDialect(SqlDialect.EMPTY_CONTEXT + .withQuotedCasing(parserConfig.unquotedCasing()) + .withConformance(parserConfig.conformance()) + .withUnquotedCasing(parserConfig.unquotedCasing()) + .withIdentifierQuoteString(parserConfig.quoting().string)); + } + + public RelDataTypeFactory getTypeFactory() { + return this.typeFactory; + } + + public SqlValidator getValidator() { + final SqlConformance conformance = + (SqlConformance) options.get("conformance"); + final boolean enableTypeCoercion = (boolean) options.get("enableTypeCoercion"); + return validatorFactory.create(operatorTable, + catalogReader, + typeFactory, + conformance) + .setEnableTypeCoercion(enableTypeCoercion); + } + + private static SqlOperatorTable createOperatorTable(SqlOperatorTable opTab0) { + MockSqlOperatorTable opTab = new MockSqlOperatorTable(opTab0); + MockSqlOperatorTable.addRamp(opTab); + return opTab; + } + + private static SqlParser.Config createParserConfig(Map options) { + return SqlParser.configBuilder() + .setQuoting((Quoting) options.get("quoting")) + .setUnquotedCasing((Casing) options.get("unquotedCasing")) + .setQuotedCasing((Casing) options.get("quotedCasing")) + .setConformance((SqlConformance) options.get("conformance")) + .setCaseSensitive((boolean) options.get("caseSensitive")) + .setParserFactory((SqlParserImplFactory) options.get("parserFactory")) + .build(); + } + + private static RelDataTypeFactory createTypeFactory(SqlConformance conformance) { + RelDataTypeSystem typeSystem = RelDataTypeSystem.DEFAULT; + if (conformance.shouldConvertRaggedUnionTypesToVarying()) { + typeSystem = new DelegatingTypeSystem(typeSystem) { + public boolean shouldConvertRaggedUnionTypesToVarying() { + return true; + } + }; + } + if (conformance.allowExtendedTrim()) { + typeSystem = new DelegatingTypeSystem(typeSystem) { + public boolean allowExtendedTrim() { + return true; + } + }; + } + return new JavaTypeFactoryImpl(typeSystem); + } + + private static Map buildDefaultOptions() { + final Map m = new HashMap<>(); + m.put("quoting", Quoting.BACK_TICK); + m.put("quotedCasing", Casing.UNCHANGED); + m.put("unquotedCasing", Casing.UNCHANGED); + m.put("caseSensitive", true); + m.put("enableTypeCoercion", false); + m.put("conformance", SqlConformanceEnum.DEFAULT); + m.put("operatorTable", SqlStdOperatorTable.instance()); + m.put("parserFactory", FlinkSqlParserImpl.FACTORY); + return Collections.unmodifiableMap(m); + } + } } diff --git a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java index 7a494b3e843cb89f912d08d5dfd1828471d70331..c4d1dc496078a4905e9b7659d48e179b7744acd4 100644 --- a/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java +++ b/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java @@ -416,8 +416,14 @@ public class FlinkSqlParserImplTest extends SqlParserTest { " 'k1' = 'v1',\n" + " 'k2' = 'v2'\n" + ")"; - final String errMsg = "UDT in DDL is not supported yet."; - checkFails(sql, errMsg); + final String expected = "CREATE TABLE `T` (\n" + + " `A` `CATALOG1`.`DB1`.`MYTYPE1`,\n" + + " `B` `DB2`.`MYTYPE2`\n" + + ") WITH (\n" + + " 'k1' = 'v1',\n" + + " 'k2' = 'v2'\n" + + ")"; + check(sql, expected); } @Test @@ -433,7 +439,7 @@ public class FlinkSqlParserImplTest extends SqlParserTest { ")\n", "(?s).*Encountered \"\\(\" at line 4, column 14.\n" + "Was expecting one of:\n" + " \"AS\" ...\n" + - " \"ARRAY\" ...\n" + + " \"STRING\" ...\n" + ".*"); } @@ -663,7 +669,37 @@ public class FlinkSqlParserImplTest extends SqlParserTest { check(sql, "DROP VIEW IF EXISTS `VIEW_NAME`"); } - /** Matcher that invokes the #validate() of the produced SqlNode. **/ + // Override the test because our ROW field type default is nullable, + // which is different with Calcite. + @Override + public void testCastAsRowType() { + checkExp("cast(a as row(f0 int, f1 varchar))", + "CAST(`A` AS ROW(`F0` INTEGER, `F1` VARCHAR))"); + checkExp("cast(a as row(f0 int not null, f1 varchar null))", + "CAST(`A` AS ROW(`F0` INTEGER NOT NULL, `F1` VARCHAR))"); + checkExp("cast(a as row(f0 row(ff0 int not null, ff1 varchar null) null," + + " f1 timestamp not null))", + "CAST(`A` AS ROW(`F0` ROW(`FF0` INTEGER NOT NULL, `FF1` VARCHAR)," + + " `F1` TIMESTAMP NOT NULL))"); + checkExp("cast(a as row(f0 bigint not null, f1 decimal null) array)", + "CAST(`A` AS ROW(`F0` BIGINT NOT NULL, `F1` DECIMAL) ARRAY)"); + checkExp("cast(a as row(f0 varchar not null, f1 timestamp null) multiset)", + "CAST(`A` AS ROW(`F0` VARCHAR NOT NULL, `F1` TIMESTAMP) MULTISET)"); + } + + @Test + public void testCreateTableWithNakedTableName() { + String sql = "CREATE TABLE tbl1"; + sql(sql).node(new ValidationMatcher()); + } + + @Test + public void testCreateViewWithEmptyFields() { + String sql = "CREATE VIEW v1 AS SELECT 1"; + sql(sql).node(new ValidationMatcher()); + } + + /** Matcher that invokes the #validate() of the {@link ExtendedSqlNode} instance. **/ private static class ValidationMatcher extends BaseMatcher { private String expectedColumnSql; private String failMsg; diff --git a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImpl.java b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImpl.java index cf2ace7c1686b3600f9ab5f2935f3c07568c79ba..06b20f529adc934cc5d6b0e2d2ef1b304c31275d 100644 --- a/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImpl.java +++ b/flink-table/flink-table-api-java-bridge/src/main/java/org/apache/flink/table/api/java/internal/StreamTableEnvironmentImpl.java @@ -147,7 +147,7 @@ public final class StreamTableEnvironmentImpl extends TableEnvironmentImpl imple public void registerFunction(String name, TableFunction tableFunction) { TypeInformation typeInfo = UserFunctionsTypeHelper.getReturnTypeOfTableFunction(tableFunction); - functionCatalog.registerTableFunction( + functionCatalog.registerTempSystemTableFunction( name, tableFunction, typeInfo @@ -160,7 +160,7 @@ public final class StreamTableEnvironmentImpl extends TableEnvironmentImpl imple TypeInformation accTypeInfo = UserFunctionsTypeHelper .getAccumulatorTypeOfAggregateFunction(aggregateFunction); - functionCatalog.registerAggregateFunction( + functionCatalog.registerTempSystemAggregateFunction( name, aggregateFunction, typeInfo, @@ -175,7 +175,7 @@ public final class StreamTableEnvironmentImpl extends TableEnvironmentImpl imple TypeInformation accTypeInfo = UserFunctionsTypeHelper .getAccumulatorTypeOfAggregateFunction(tableAggregateFunction); - functionCatalog.registerAggregateFunction( + functionCatalog.registerTempSystemAggregateFunction( name, tableAggregateFunction, typeInfo, diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java index 88aa167aa40db68dba266fb3e90ac2448b1dcac7..12e874ece3b8d057a4a5c0382fc51a088a6993a4 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/api/internal/TableEnvironmentImpl.java @@ -163,7 +163,7 @@ public class TableEnvironmentImpl implements TableEnvironment { @Override public void registerFunction(String name, ScalarFunction function) { - functionCatalog.registerScalarFunction( + functionCatalog.registerTempSystemScalarFunction( name, function); } diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/FunctionCatalog.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/FunctionCatalog.java index 423b5f33001d5c8b62b10ccd6dbf017a4ea0ae6f..43280b47029cb4b6eab35f5c550126bd05e2ea95 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/FunctionCatalog.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/catalog/FunctionCatalog.java @@ -58,9 +58,8 @@ public class FunctionCatalog implements FunctionLookup { private final CatalogManager catalogManager; - // For simplicity, currently hold registered Flink functions in memory here - // TODO: should move to catalog - private final Map userFunctions = new LinkedHashMap<>(); + private final Map tempSystemFunctions = new LinkedHashMap<>(); + private final Map tempCatalogFunctions = new LinkedHashMap<>(); /** * Temporary utility until the new type inference is fully functional. It needs to be set by the planner. @@ -75,15 +74,15 @@ public class FunctionCatalog implements FunctionLookup { this.plannerTypeInferenceUtil = plannerTypeInferenceUtil; } - public void registerScalarFunction(String name, ScalarFunction function) { + public void registerTempSystemScalarFunction(String name, ScalarFunction function) { UserFunctionsTypeHelper.validateInstantiation(function.getClass()); - registerFunction( + registerTempSystemFunction( name, new ScalarFunctionDefinition(name, function) ); } - public void registerTableFunction( + public void registerTempSystemTableFunction( String name, TableFunction function, TypeInformation resultType) { @@ -92,7 +91,7 @@ public class FunctionCatalog implements FunctionLookup { // check if class could be instantiated UserFunctionsTypeHelper.validateInstantiation(function.getClass()); - registerFunction( + registerTempSystemFunction( name, new TableFunctionDefinition( name, @@ -101,7 +100,7 @@ public class FunctionCatalog implements FunctionLookup { ); } - public void registerAggregateFunction( + public void registerTempSystemAggregateFunction( String name, UserDefinedAggregateFunction function, TypeInformation resultType, @@ -128,12 +127,71 @@ public class FunctionCatalog implements FunctionLookup { throw new TableException("Unknown function class: " + function.getClass()); } - registerFunction( + registerTempSystemFunction( name, definition ); } + public void registerTempCatalogScalarFunction(ObjectIdentifier oi, ScalarFunction function) { + UserFunctionsTypeHelper.validateInstantiation(function.getClass()); + registerTempCatalogFunction( + oi, + new ScalarFunctionDefinition(oi.getObjectName(), function) + ); + } + + public void registerTempCatalogTableFunction( + ObjectIdentifier oi, + TableFunction function, + TypeInformation resultType) { + // check if class not Scala object + UserFunctionsTypeHelper.validateNotSingleton(function.getClass()); + // check if class could be instantiated + UserFunctionsTypeHelper.validateInstantiation(function.getClass()); + + registerTempCatalogFunction( + oi, + new TableFunctionDefinition( + oi.getObjectName(), + function, + resultType) + ); + } + + public void registerTempCatalogAggregateFunction( + ObjectIdentifier oi, + UserDefinedAggregateFunction function, + TypeInformation resultType, + TypeInformation accType) { + // check if class not Scala object + UserFunctionsTypeHelper.validateNotSingleton(function.getClass()); + // check if class could be instantiated + UserFunctionsTypeHelper.validateInstantiation(function.getClass()); + + final FunctionDefinition definition; + if (function instanceof AggregateFunction) { + definition = new AggregateFunctionDefinition( + oi.getObjectName(), + (AggregateFunction) function, + resultType, + accType); + } else if (function instanceof TableAggregateFunction) { + definition = new TableAggregateFunctionDefinition( + oi.getObjectName(), + (TableAggregateFunction) function, + resultType, + accType); + } else { + throw new TableException("Unknown function class: " + function.getClass()); + } + + registerTempCatalogFunction( + oi, + definition + ); + } + public String[] getUserDefinedFunctions() { return getUserDefinedFunctionNames().toArray(new String[0]); } @@ -165,7 +223,7 @@ public class FunctionCatalog implements FunctionLookup { // Get functions registered in memory result.addAll( - userFunctions.values().stream() + tempSystemFunctions.values().stream() .map(FunctionDefinition::toString) .collect(Collectors.toSet())); @@ -204,7 +262,7 @@ public class FunctionCatalog implements FunctionLookup { } // If no corresponding function is found in catalog, check in-memory functions - userCandidate = userFunctions.get(functionName); + userCandidate = tempSystemFunctions.get(functionName); final Optional foundDefinition; if (userCandidate != null) { @@ -240,13 +298,24 @@ public class FunctionCatalog implements FunctionLookup { return plannerTypeInferenceUtil; } - private void registerFunction(String name, FunctionDefinition functionDefinition) { - // TODO: should register to catalog - userFunctions.put(normalizeName(name), functionDefinition); + private void registerTempSystemFunction(String name, FunctionDefinition functionDefinition) { + tempSystemFunctions.put(normalizeName(name), functionDefinition); + } + + private void registerTempCatalogFunction(ObjectIdentifier oi, FunctionDefinition functionDefinition) { + tempCatalogFunctions.put(normalizeObjectIdentifier(oi), functionDefinition); } @VisibleForTesting static String normalizeName(String name) { return name.toUpperCase(); } + + @VisibleForTesting + static ObjectIdentifier normalizeObjectIdentifier(ObjectIdentifier oi) { + return ObjectIdentifier.of( + oi.getCatalogName(), + oi.getDatabaseName(), + oi.getObjectName().toUpperCase()); + } } diff --git a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/utils/ApiExpressionUtils.java b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/utils/ApiExpressionUtils.java index 5d0c2c365ea88bac8019f71adc66a6a6f86924a9..99de217042904bbdd90f6be577f0d44f82cb59f7 100644 --- a/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/utils/ApiExpressionUtils.java +++ b/flink-table/flink-table-api-java/src/main/java/org/apache/flink/table/expressions/utils/ApiExpressionUtils.java @@ -50,7 +50,7 @@ public final class ApiExpressionUtils { public static final long MILLIS_PER_HOUR = 3600000L; // = 60 * 60 * 1000 - public static final long MILLIS_PER_DAY = 86400000; // = 24 * 60 * 60 * 1000 + public static final long MILLIS_PER_DAY = 86400000L; // = 24 * 60 * 60 * 1000 private ApiExpressionUtils() { // private diff --git a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImpl.scala b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImpl.scala index c45d324133d206da0e940fd40d31d5bf427b364d..1384be09e5a4a960f64076ba241bd4eb725db052 100644 --- a/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImpl.scala +++ b/flink-table/flink-table-api-scala-bridge/src/main/scala/org/apache/flink/table/api/scala/internal/StreamTableEnvironmentImpl.scala @@ -137,7 +137,7 @@ class StreamTableEnvironmentImpl ( override def registerFunction[T: TypeInformation](name: String, tf: TableFunction[T]): Unit = { val typeInfo = UserFunctionsTypeHelper .getReturnTypeOfTableFunction(tf, implicitly[TypeInformation[T]]) - functionCatalog.registerTableFunction( + functionCatalog.registerTempSystemTableFunction( name, tf, typeInfo @@ -152,7 +152,7 @@ class StreamTableEnvironmentImpl ( .getReturnTypeOfAggregateFunction(f, implicitly[TypeInformation[T]]) val accTypeInfo = UserFunctionsTypeHelper .getAccumulatorTypeOfAggregateFunction(f, implicitly[TypeInformation[ACC]]) - functionCatalog.registerAggregateFunction( + functionCatalog.registerTempSystemAggregateFunction( name, f, typeInfo, @@ -168,7 +168,7 @@ class StreamTableEnvironmentImpl ( .getReturnTypeOfAggregateFunction(f, implicitly[TypeInformation[T]]) val accTypeInfo = UserFunctionsTypeHelper .getAccumulatorTypeOfAggregateFunction(f, implicitly[TypeInformation[ACC]]) - functionCatalog.registerAggregateFunction( + functionCatalog.registerTempSystemAggregateFunction( name, f, typeInfo, diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/FunctionDefinition.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/FunctionDefinition.java index 16379285f2e287faaeaca4b0deb8fbd4fd52fd2e..e43e84121c3254d7fc501ce2014c41b16995be2d 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/FunctionDefinition.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/FunctionDefinition.java @@ -40,6 +40,13 @@ public interface FunctionDefinition { */ FunctionKind getKind(); + /** + * Returns the language of function this definition describes. + */ + default FunctionLanguage getLanguage() { + return FunctionLanguage.JVM; + } + /** * Returns the set of requirements this definition demands. */ diff --git a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncWatermarkResult.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/FunctionLanguage.java similarity index 65% rename from flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncWatermarkResult.java rename to flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/FunctionLanguage.java index 68bde3a57229d9c1d70b4805e387347aa5fbf900..7b36b132a4719f61f5a933bb9378bae751aa72e9 100644 --- a/flink-streaming-java/src/main/java/org/apache/flink/streaming/api/operators/async/queue/AsyncWatermarkResult.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/FunctionLanguage.java @@ -16,20 +16,17 @@ * limitations under the License. */ -package org.apache.flink.streaming.api.operators.async.queue; +package org.apache.flink.table.functions; -import org.apache.flink.annotation.Internal; -import org.apache.flink.streaming.api.watermark.Watermark; +import org.apache.flink.annotation.PublicEvolving; /** - * {@link AsyncResult} subclass for asynchronous result {@link Watermark}. + * Categorizes the language of a {@link FunctionDefinition}. */ -@Internal -public interface AsyncWatermarkResult extends AsyncResult { - /** - * Get the resulting watermark. - * - * @return the asynchronous result watermark - */ - Watermark getWatermark(); +@PublicEvolving +public enum FunctionLanguage { + + JVM, + + PYTHON } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonEnv.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonEnv.java index 5d2be94cb9ca8280d8d3e7f22920a6cad0304bf9..d01a08d95ccbf9fa627bdf0bf84509b691faf1df 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonEnv.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/PythonEnv.java @@ -31,38 +31,15 @@ public final class PythonEnv implements Serializable { private static final long serialVersionUID = 1L; - /** - * The path of the Python executable file used. - */ - private final String pythonExec; - - /** - * The command to start Python worker process. - */ - private final String pythonWorkerCmd; - /** * The execution type of the Python worker, it defines how to execute the Python functions. */ private final ExecType execType; - public PythonEnv( - String pythonExec, - String pythonWorkerCmd, - ExecType execType) { - this.pythonExec = Preconditions.checkNotNull(pythonExec); - this.pythonWorkerCmd = Preconditions.checkNotNull(pythonWorkerCmd); + public PythonEnv(ExecType execType) { this.execType = Preconditions.checkNotNull(execType); } - public String getPythonExec() { - return pythonExec; - } - - public String getPythonWorkerCmd() { - return pythonWorkerCmd; - } - public ExecType getExecType() { return execType; } diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/LeaderIdMismatchException.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/SimplePythonFunction.java similarity index 50% rename from flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/LeaderIdMismatchException.java rename to flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/SimplePythonFunction.java index 5caf1b296baa6bf38dfb935de6ab2a5d4048e60f..116151ad1d5fce33c5ebbd23fbe467fff584ed40 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/highavailability/LeaderIdMismatchException.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/python/SimplePythonFunction.java @@ -16,32 +16,41 @@ * limitations under the License. */ -package org.apache.flink.runtime.highavailability; +package org.apache.flink.table.functions.python; -import java.util.UUID; +import org.apache.flink.annotation.Internal; +import org.apache.flink.util.Preconditions; /** - * An exception thrown when the leader session id attached to a message does not match - * the leader session id that the receiver expects. + * A simple implementation of {@link PythonFunction}. */ -public class LeaderIdMismatchException extends Exception { +@Internal +public final class SimplePythonFunction implements PythonFunction { private static final long serialVersionUID = 1L; - private final UUID expected; - private final UUID actual; + /** + * Serialized representation of the user-defined python function. + */ + private final byte[] serializedPythonFunction; - public LeaderIdMismatchException(UUID expected, UUID actual) { - super("Leader session ID mismatch: expected=" + expected + ", actual=" + actual); - this.expected = expected; - this.actual = actual; + /** + * Python execution environment. + */ + private final PythonEnv pythonEnv; + + public SimplePythonFunction(byte[] serializedPythonFunction, PythonEnv pythonEnv) { + this.serializedPythonFunction = Preconditions.checkNotNull(serializedPythonFunction); + this.pythonEnv = Preconditions.checkNotNull(pythonEnv); } - public UUID getExpected() { - return expected; + @Override + public byte[] getSerializedPythonFunction() { + return serializedPythonFunction; } - public UUID getActual() { - return actual; + @Override + public PythonEnv getPythonEnv() { + return pythonEnv; } } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/MapType.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/MapType.java index 4d7726584e14dabfabe0b49bccf66f90afcae114..cd0ab71814551984bd4b53bb70c871b5b0920301 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/MapType.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/MapType.java @@ -91,6 +91,9 @@ public final class MapType extends LogicalType { @Override public boolean supportsInputConversion(Class clazz) { + if (Map.class.isAssignableFrom(clazz)) { + return true; + } return INPUT_OUTPUT_CONVERSION.contains(clazz.getName()); } diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/MultisetType.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/MultisetType.java index 54aa7e40100cb203daccea8672c5e9dc31b2f1e0..cd4b7ee6c2295dd09d2645a6fbe953221e41b78e 100644 --- a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/MultisetType.java +++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/types/logical/MultisetType.java @@ -82,6 +82,9 @@ public final class MultisetType extends LogicalType { @Override public boolean supportsInputConversion(Class clazz) { + if (Map.class.isAssignableFrom(clazz)) { + return true; + } return INPUT_OUTPUT_CONVERSION.contains(clazz.getName()); } diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/expressions/ExpressionTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/expressions/ExpressionTest.java index eb80d25bc032b3c9424d383ffd265044e8ab0321..c9d265b58ff0c5bd0dac33d7656b247e6e4cacbb 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/expressions/ExpressionTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/expressions/ExpressionTest.java @@ -30,6 +30,8 @@ import org.junit.rules.ExpectedException; import java.math.BigDecimal; import java.sql.Timestamp; import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; import static java.util.Arrays.asList; import static java.util.Collections.singletonList; @@ -86,6 +88,23 @@ public class ExpressionTest { new String[][]{null, null, {"1", "2", "3", "Dog's"}}, DataTypes.ARRAY(DataTypes.ARRAY(DataTypes.STRING()))) .toString()); + + final Map map = new HashMap<>(); + map.put("key1", 1); + map.put("key2", 2); + map.put("key3", 3); + assertEquals( + "{key1=1, key2=2, key3=3}", + new ValueLiteralExpression( + map, + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())) + .toString()); + assertEquals( + "{key1=1, key2=2, key3=3}", + new ValueLiteralExpression( + map, + DataTypes.MULTISET(DataTypes.STRING())) + .toString()); } @Test diff --git a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java index 6bbbc80c11b8af9c54fdc25b03ade31999f6bd97..5ce9937f6d07561072071ce40fd093650344a0b0 100644 --- a/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java +++ b/flink-table/flink-table-common/src/test/java/org/apache/flink/table/types/LogicalTypesTest.java @@ -68,7 +68,9 @@ import org.junit.Test; import java.math.BigDecimal; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Map; +import java.util.TreeMap; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -426,7 +428,7 @@ public class LogicalTypesTest { new MultisetType(new TimestampType()), "MULTISET", "MULTISET", - new Class[]{Map.class}, + new Class[]{Map.class, HashMap.class, TreeMap.class}, new Class[]{Map.class}, new LogicalType[]{new TimestampType()}, new MultisetType(new SmallIntType()) @@ -436,7 +438,7 @@ public class LogicalTypesTest { new MultisetType(new MultisetType(new TimestampType())), "MULTISET>", "MULTISET>", - new Class[]{Map.class}, + new Class[]{Map.class, HashMap.class, TreeMap.class}, new Class[]{Map.class}, new LogicalType[]{new MultisetType(new TimestampType())}, new MultisetType(new MultisetType(new SmallIntType())) @@ -449,7 +451,7 @@ public class LogicalTypesTest { new MapType(new VarCharType(20), new TimestampType()), "MAP", "MAP", - new Class[]{Map.class}, + new Class[]{Map.class, HashMap.class, TreeMap.class}, new Class[]{Map.class}, new LogicalType[]{new VarCharType(20), new TimestampType()}, new MapType(new VarCharType(99), new TimestampType()) diff --git a/flink-table/flink-table-planner-blink/pom.xml b/flink-table/flink-table-planner-blink/pom.xml index f9e7f2b73b47bf73bd1d8287754a2f84befbcc72..1d60b7ad18e094126df3d786068b23f8db85607e 100644 --- a/flink-table/flink-table-planner-blink/pom.xml +++ b/flink-table/flink-table-planner-blink/pom.xml @@ -162,14 +162,14 @@ under the License. org.apache.calcite calcite-core - 1.20.0 + ${calcite.version} newInput - // - // newProject - // | - // oldInput ----> newInput - // - // is transformed to - // - // oldInput ----> newProject - // | - // newInput - Map combinedMap = new HashMap<>(); - - for (Integer oldInputPos : frame.oldToNewOutputs.keySet()) { - combinedMap.put(oldInputPos, - mapNewInputToProjOutputs.get( - frame.oldToNewOutputs.get(oldInputPos))); - } - - register(oldInput, newProject, combinedMap, corDefOutputs); - - // now it's time to rewrite the Aggregate - final ImmutableBitSet newGroupSet = ImmutableBitSet.range(newGroupKeyCount); - List newAggCalls = new ArrayList<>(); - List oldAggCalls = rel.getAggCallList(); - - ImmutableList newGroupSets = null; - if (rel.getGroupType() != Aggregate.Group.SIMPLE) { - final ImmutableBitSet addedGroupSet = - ImmutableBitSet.range(oldGroupKeyCount, newGroupKeyCount); - final Iterable tmpGroupSets = - Iterables.transform(rel.getGroupSets(), - bitSet -> bitSet.union(addedGroupSet)); - newGroupSets = ImmutableBitSet.ORDERING.immutableSortedCopy(tmpGroupSets); - } - - int oldInputOutputFieldCount = rel.getGroupSet().cardinality(); - int newInputOutputFieldCount = newGroupSet.cardinality(); - - int i = -1; - for (AggregateCall oldAggCall : oldAggCalls) { - ++i; - List oldAggArgs = oldAggCall.getArgList(); - - List aggArgs = new ArrayList<>(); - - // Adjust the Aggregate argument positions. - // Note Aggregate does not change input ordering, so the input - // output position mapping can be used to derive the new positions - // for the argument. - for (int oldPos : oldAggArgs) { - aggArgs.add(combinedMap.get(oldPos)); - } - final int filterArg = oldAggCall.filterArg < 0 ? oldAggCall.filterArg - : combinedMap.get(oldAggCall.filterArg); - - newAggCalls.add( - oldAggCall.adaptTo(newProject, aggArgs, filterArg, - oldGroupKeyCount, newGroupKeyCount)); - - // The old to new output position mapping will be the same as that - // of newProject, plus any aggregates that the oldAgg produces. - outputMap.put( - oldInputOutputFieldCount + i, - newInputOutputFieldCount + i); - } - - relBuilder.push( - LogicalAggregate.create(newProject, newGroupSet, newGroupSets, newAggCalls)); - - if (!omittedConstants.isEmpty()) { - final List postProjects = new ArrayList<>(relBuilder.fields()); - for (Map.Entry entry - : omittedConstants.descendingMap().entrySet()) { - int index = entry.getKey() + frame.corDefOutputs.size(); - postProjects.add(index, entry.getValue()); - // Shift the outputs whose index equals with or bigger than the added index - // with 1 offset. - shiftMapping(outputMap, index, 1); - // Then add the constant key mapping. - outputMap.put(entry.getKey(), index); - } - relBuilder.project(postProjects); - } - - // Aggregate does not change input ordering so corVars will be - // located at the same position as the input newProject. - return register(rel, relBuilder.build(), outputMap, corDefOutputs); - } - - /** - * Shift the mapping to fixed offset from the {@code startIndex}. - * @param mapping the original mapping - * @param startIndex any output whose index equals with or bigger than the starting index - * would be shift - * @param offset shift offset - */ - private static void shiftMapping(Map mapping, int startIndex, int offset) { - for (Map.Entry entry : mapping.entrySet()) { - if (entry.getValue() >= startIndex) { - mapping.put(entry.getKey(), entry.getValue() + offset); - } else { - mapping.put(entry.getKey(), entry.getValue()); - } - } - } - - public Frame getInvoke(RelNode r, RelNode parent) { - final Frame frame = dispatcher.invoke(r); - if (frame != null) { - map.put(r, frame); - } - currentRel = parent; - return frame; - } - - /** Returns a literal output field, or null if it is not literal. */ - private static RexLiteral projectedLiteral(RelNode rel, int i) { - if (rel instanceof Project) { - final Project project = (Project) rel; - final RexNode node = project.getProjects().get(i); - if (node instanceof RexLiteral) { - return (RexLiteral) node; - } - } - return null; - } - - /** - * Rewrite LogicalProject. - * - * @param rel the project rel to rewrite - */ - public Frame decorrelateRel(LogicalProject rel) { - // - // Rewrite logic: - // - // 1. Pass along any correlated variables coming from the input. - // - - final RelNode oldInput = rel.getInput(); - Frame frame = getInvoke(oldInput, rel); - if (frame == null) { - // If input has not been rewritten, do not rewrite this rel. - return null; - } - final List oldProjects = rel.getProjects(); - final List relOutput = rel.getRowType().getFieldList(); - - // Project projects the original expressions, - // plus any correlated variables the input wants to pass along. - final List> projects = new ArrayList<>(); - - // If this Project has correlated reference, create value generator - // and produce the correlated variables in the new output. - if (cm.mapRefRelToCorRef.containsKey(rel)) { - frame = decorrelateInputWithValueGenerator(rel, frame); - } - - // Project projects the original expressions - final Map mapOldToNewOutputs = new HashMap<>(); - int newPos; - for (newPos = 0; newPos < oldProjects.size(); newPos++) { - projects.add( - newPos, - Pair.of( - decorrelateExpr(currentRel, map, cm, oldProjects.get(newPos)), - relOutput.get(newPos).getName())); - mapOldToNewOutputs.put(newPos, newPos); - } - - // Project any correlated variables the input wants to pass along. - final SortedMap corDefOutputs = new TreeMap<>(); - for (Map.Entry entry : frame.corDefOutputs.entrySet()) { - projects.add( - RexInputRef.of2(entry.getValue(), - frame.r.getRowType().getFieldList())); - corDefOutputs.put(entry.getKey(), newPos); - newPos++; - } - - RelNode newProject = relBuilder.push(frame.r) - .projectNamed(Pair.left(projects), Pair.right(projects), true) - .build(); - - return register(rel, newProject, mapOldToNewOutputs, corDefOutputs); - } - - /** - * Create RelNode tree that produces a list of correlated variables. - * - * @param correlations correlated variables to generate - * @param valueGenFieldOffset offset in the output that generated columns - * will start - * @param corDefOutputs output positions for the correlated variables - * generated - * @return RelNode the root of the resultant RelNode tree - */ - private RelNode createValueGenerator( - Iterable correlations, - int valueGenFieldOffset, - SortedMap corDefOutputs) { - final Map> mapNewInputToOutputs = new HashMap<>(); - - final Map mapNewInputToNewOffset = new HashMap<>(); - - // Input provides the definition of a correlated variable. - // Add to map all the referenced positions (relative to each input rel). - for (CorRef corVar : correlations) { - final int oldCorVarOffset = corVar.field; - - final RelNode oldInput = getCorRel(corVar); - assert oldInput != null; - final Frame frame = getFrame(oldInput, true); - assert frame != null; - final RelNode newInput = frame.r; - - final List newLocalOutputs; - if (!mapNewInputToOutputs.containsKey(newInput)) { - newLocalOutputs = new ArrayList<>(); - } else { - newLocalOutputs = mapNewInputToOutputs.get(newInput); - } - - final int newCorVarOffset = frame.oldToNewOutputs.get(oldCorVarOffset); - - // Add all unique positions referenced. - if (!newLocalOutputs.contains(newCorVarOffset)) { - newLocalOutputs.add(newCorVarOffset); - } - mapNewInputToOutputs.put(newInput, newLocalOutputs); - } - - int offset = 0; - - // Project only the correlated fields out of each input - // and join the project together. - // To make sure the plan does not change in terms of join order, - // join these rels based on their occurrence in corVar list which - // is sorted. - final Set joinedInputs = new HashSet<>(); - - RelNode r = null; - for (CorRef corVar : correlations) { - final RelNode oldInput = getCorRel(corVar); - assert oldInput != null; - final RelNode newInput = getFrame(oldInput, true).r; - assert newInput != null; - - if (!joinedInputs.contains(newInput)) { - RelNode project = - RelOptUtil.createProject(newInput, - mapNewInputToOutputs.get(newInput)); - RelNode distinct = relBuilder.push(project) - .distinct() - .build(); - RelOptCluster cluster = distinct.getCluster(); - - joinedInputs.add(newInput); - mapNewInputToNewOffset.put(newInput, offset); - offset += distinct.getRowType().getFieldCount(); - - if (r == null) { - r = distinct; - } else { - r = - LogicalJoin.create(r, distinct, - cluster.getRexBuilder().makeLiteral(true), - ImmutableSet.of(), JoinRelType.INNER); - } - } - } - - // Translate the positions of correlated variables to be relative to - // the join output, leaving room for valueGenFieldOffset because - // valueGenerators are joined with the original left input of the rel - // referencing correlated variables. - for (CorRef corRef : correlations) { - // The first input of a Correlate is always the rel defining - // the correlated variables. - final RelNode oldInput = getCorRel(corRef); - assert oldInput != null; - final Frame frame = getFrame(oldInput, true); - final RelNode newInput = frame.r; - assert newInput != null; - - final List newLocalOutputs = mapNewInputToOutputs.get(newInput); - - final int newLocalOutput = frame.oldToNewOutputs.get(corRef.field); - - // newOutput is the index of the corVar in the referenced - // position list plus the offset of referenced position list of - // each newInput. - final int newOutput = - newLocalOutputs.indexOf(newLocalOutput) - + mapNewInputToNewOffset.get(newInput) - + valueGenFieldOffset; - - corDefOutputs.put(corRef.def(), newOutput); - } - - return r; - } - - private Frame getFrame(RelNode r, boolean safe) { - final Frame frame = map.get(r); - if (frame == null && safe) { - return new Frame(r, r, ImmutableSortedMap.of(), - identityMap(r.getRowType().getFieldCount())); - } - return frame; - } - - private RelNode getCorRel(CorRef corVar) { - final RelNode r = cm.mapCorToCorRel.get(corVar.corr); - return r.getInput(0); - } - - /** Adds a value generator to satisfy the correlating variables used by - * a relational expression, if those variables are not already provided by - * its input. */ - private Frame maybeAddValueGenerator(RelNode rel, Frame frame) { - final CorelMap cm1 = new CorelMapBuilder().build(frame.r, rel); - if (!cm1.mapRefRelToCorRef.containsKey(rel)) { - return frame; - } - final Collection needs = cm1.mapRefRelToCorRef.get(rel); - final ImmutableSortedSet haves = frame.corDefOutputs.keySet(); - if (hasAll(needs, haves)) { - return frame; - } - return decorrelateInputWithValueGenerator(rel, frame); - } - - /** Returns whether all of a collection of {@link CorRef}s are satisfied - * by at least one of a collection of {@link CorDef}s. */ - private boolean hasAll(Collection corRefs, - Collection corDefs) { - for (CorRef corRef : corRefs) { - if (!has(corDefs, corRef)) { - return false; - } - } - return true; - } - - /** Returns whether a {@link CorrelationId} is satisfied by at least one of a - * collection of {@link CorDef}s. */ - private boolean has(Collection corDefs, CorRef corr) { - for (CorDef corDef : corDefs) { - if (corDef.corr.equals(corr.corr) && corDef.field == corr.field) { - return true; - } - } - return false; - } - - private Frame decorrelateInputWithValueGenerator(RelNode rel, Frame frame) { - // currently only handles one input - assert rel.getInputs().size() == 1; - RelNode oldInput = frame.r; - - final SortedMap corDefOutputs = - new TreeMap<>(frame.corDefOutputs); - - final Collection corVarList = cm.mapRefRelToCorRef.get(rel); - - // Try to populate correlation variables using local fields. - // This means that we do not need a value generator. - if (rel instanceof Filter) { - SortedMap map = new TreeMap<>(); - List projects = new ArrayList<>(); - for (CorRef correlation : corVarList) { - final CorDef def = correlation.def(); - if (corDefOutputs.containsKey(def) || map.containsKey(def)) { - continue; - } - try { - findCorrelationEquivalent(correlation, ((Filter) rel).getCondition()); - } catch (Util.FoundOne e) { - if (e.getNode() instanceof RexInputRef) { - map.put(def, ((RexInputRef) e.getNode()).getIndex()); - } else { - map.put(def, - frame.r.getRowType().getFieldCount() + projects.size()); - projects.add((RexNode) e.getNode()); - } - } - } - // If all correlation variables are now satisfied, skip creating a value - // generator. - if (map.size() == corVarList.size()) { - map.putAll(frame.corDefOutputs); - final RelNode r; - if (!projects.isEmpty()) { - relBuilder.push(oldInput) - .project(Iterables.concat(relBuilder.fields(), projects)); - r = relBuilder.build(); - } else { - r = oldInput; - } - return register(rel.getInput(0), r, - frame.oldToNewOutputs, map); - } - } - - int leftInputOutputCount = frame.r.getRowType().getFieldCount(); - - // can directly add positions into corDefOutputs since join - // does not change the output ordering from the inputs. - RelNode valueGen = - createValueGenerator(corVarList, leftInputOutputCount, corDefOutputs); - - RelNode join = - LogicalJoin.create(frame.r, valueGen, relBuilder.literal(true), - ImmutableSet.of(), JoinRelType.INNER); - - // Join or Filter does not change the old input ordering. All - // input fields from newLeftInput (i.e. the original input to the old - // Filter) are in the output and in the same position. - return register(rel.getInput(0), join, frame.oldToNewOutputs, - corDefOutputs); - } - - /** Finds a {@link RexInputRef} that is equivalent to a {@link CorRef}, - * and if found, throws a {@link org.apache.calcite.util.Util.FoundOne}. */ - private void findCorrelationEquivalent(CorRef correlation, RexNode e) - throws Util.FoundOne { - switch (e.getKind()) { - case EQUALS: - final RexCall call = (RexCall) e; - final List operands = call.getOperands(); - if (references(operands.get(0), correlation)) { - throw new Util.FoundOne(operands.get(1)); - } - if (references(operands.get(1), correlation)) { - throw new Util.FoundOne(operands.get(0)); - } - break; - case AND: - for (RexNode operand : ((RexCall) e).getOperands()) { - findCorrelationEquivalent(correlation, operand); - } - } - } - - private boolean references(RexNode e, CorRef correlation) { - switch (e.getKind()) { - case CAST: - final RexNode operand = ((RexCall) e).getOperands().get(0); - if (isWidening(e.getType(), operand.getType())) { - return references(operand, correlation); - } - return false; - case FIELD_ACCESS: - final RexFieldAccess f = (RexFieldAccess) e; - if (f.getField().getIndex() == correlation.field - && f.getReferenceExpr() instanceof RexCorrelVariable) { - if (((RexCorrelVariable) f.getReferenceExpr()).id == correlation.corr) { - return true; - } - } - // fall through - default: - return false; - } - } - - /** Returns whether one type is just a widening of another. - * - *

    For example:

      - *
    • {@code VARCHAR(10)} is a widening of {@code VARCHAR(5)}. - *
    • {@code VARCHAR(10)} is a widening of {@code VARCHAR(10) NOT NULL}. - *
    - */ - private boolean isWidening(RelDataType type, RelDataType type1) { - return type.getSqlTypeName() == type1.getSqlTypeName() - && type.getPrecision() >= type1.getPrecision(); - } - - /** - * Rewrite LogicalSnapshot. - * - * @param rel the snapshot rel to rewrite - */ - public Frame decorrelateRel(LogicalSnapshot rel) { - if (RexUtil.containsCorrelation(rel.getPeriod())) { - return null; - } - return decorrelateRel((RelNode) rel); - } - - /** - * Rewrite LogicalFilter. - * - * @param rel the filter rel to rewrite - */ - public Frame decorrelateRel(LogicalFilter rel) { - // - // Rewrite logic: - // - // 1. If a Filter references a correlated field in its filter - // condition, rewrite the Filter to be - // Filter - // Join(cross product) - // originalFilterInput - // ValueGenerator(produces distinct sets of correlated variables) - // and rewrite the correlated fieldAccess in the filter condition to - // reference the Join output. - // - // 2. If Filter does not reference correlated variables, simply - // rewrite the filter condition using new input. - // - - final RelNode oldInput = rel.getInput(); - Frame frame = getInvoke(oldInput, rel); - if (frame == null) { - // If input has not been rewritten, do not rewrite this rel. - return null; - } - - // If this Filter has correlated reference, create value generator - // and produce the correlated variables in the new output. - if (false) { - if (cm.mapRefRelToCorRef.containsKey(rel)) { - frame = decorrelateInputWithValueGenerator(rel, frame); - } - } else { - frame = maybeAddValueGenerator(rel, frame); - } - - final CorelMap cm2 = new CorelMapBuilder().build(rel); - - // Replace the filter expression to reference output of the join - // Map filter to the new filter over join - relBuilder.push(frame.r) - .filter(decorrelateExpr(currentRel, map, cm2, rel.getCondition())); - - // Filter does not change the input ordering. - // Filter rel does not permute the input. - // All corVars produced by filter will have the same output positions in the - // input rel. - return register(rel, relBuilder.build(), frame.oldToNewOutputs, - frame.corDefOutputs); - } - - /** - * Rewrite Correlate into a left outer join. - * - * @param rel Correlator - */ - public Frame decorrelateRel(LogicalCorrelate rel) { - // - // Rewrite logic: - // - // The original left input will be joined with the new right input that - // has generated correlated variables propagated up. For any generated - // corVars that are not used in the join key, pass them along to be - // joined later with the Correlates that produce them. - // - - // the right input to Correlate should produce correlated variables - final RelNode oldLeft = rel.getInput(0); - final RelNode oldRight = rel.getInput(1); - - final Frame leftFrame = getInvoke(oldLeft, rel); - final Frame rightFrame = getInvoke(oldRight, rel); - - if (leftFrame == null || rightFrame == null) { - // If any input has not been rewritten, do not rewrite this rel. - return null; - } - - if (rightFrame.corDefOutputs.isEmpty()) { - return null; - } - - assert rel.getRequiredColumns().cardinality() - <= rightFrame.corDefOutputs.keySet().size(); - - // Change correlator rel into a join. - // Join all the correlated variables produced by this correlator rel - // with the values generated and propagated from the right input - final SortedMap corDefOutputs = - new TreeMap<>(rightFrame.corDefOutputs); - final List conditions = new ArrayList<>(); - final List newLeftOutput = - leftFrame.r.getRowType().getFieldList(); - int newLeftFieldCount = newLeftOutput.size(); - - final List newRightOutput = - rightFrame.r.getRowType().getFieldList(); - - for (Map.Entry rightOutput - : new ArrayList<>(corDefOutputs.entrySet())) { - final CorDef corDef = rightOutput.getKey(); - if (!corDef.corr.equals(rel.getCorrelationId())) { - continue; - } - final int newLeftPos = leftFrame.oldToNewOutputs.get(corDef.field); - final int newRightPos = rightOutput.getValue(); - conditions.add( - relBuilder.call(SqlStdOperatorTable.EQUALS, - RexInputRef.of(newLeftPos, newLeftOutput), - new RexInputRef(newLeftFieldCount + newRightPos, - newRightOutput.get(newRightPos).getType()))); - - // remove this corVar from output position mapping - corDefOutputs.remove(corDef); - } - - // Update the output position for the corVars: only pass on the cor - // vars that are not used in the join key. - for (CorDef corDef : corDefOutputs.keySet()) { - int newPos = corDefOutputs.get(corDef) + newLeftFieldCount; - corDefOutputs.put(corDef, newPos); - } - - // then add any corVar from the left input. Do not need to change - // output positions. - corDefOutputs.putAll(leftFrame.corDefOutputs); - - // Create the mapping between the output of the old correlation rel - // and the new join rel - final Map mapOldToNewOutputs = new HashMap<>(); - - int oldLeftFieldCount = oldLeft.getRowType().getFieldCount(); - - int oldRightFieldCount = oldRight.getRowType().getFieldCount(); - //noinspection AssertWithSideEffects - assert rel.getRowType().getFieldCount() - == oldLeftFieldCount + oldRightFieldCount; - - // Left input positions are not changed. - mapOldToNewOutputs.putAll(leftFrame.oldToNewOutputs); - - // Right input positions are shifted by newLeftFieldCount. - for (int i = 0; i < oldRightFieldCount; i++) { - mapOldToNewOutputs.put(i + oldLeftFieldCount, - rightFrame.oldToNewOutputs.get(i) + newLeftFieldCount); - } - - final RexNode condition = - RexUtil.composeConjunction(relBuilder.getRexBuilder(), conditions); - RelNode newJoin = - LogicalJoin.create(leftFrame.r, rightFrame.r, condition, - ImmutableSet.of(), rel.getJoinType()); - - return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs); - } - - /** - * Rewrite LogicalJoin. - * - * @param rel Join - */ - public Frame decorrelateRel(LogicalJoin rel) { - // For SEMI/ANTI join decorrelate it's input directly, - // because the correlate variables can only be propagated from - // the left side, which is not supported yet. - if (!rel.getJoinType().projectsRight()) { - // fix CALCITE-3169 - return decorrelateRel((RelNode) rel); - } - - // - // Rewrite logic: - // - // 1. rewrite join condition. - // 2. map output positions and produce corVars if any. - // - - final RelNode oldLeft = rel.getInput(0); - final RelNode oldRight = rel.getInput(1); - - final Frame leftFrame = getInvoke(oldLeft, rel); - final Frame rightFrame = getInvoke(oldRight, rel); - - if (leftFrame == null || rightFrame == null) { - // If any input has not been rewritten, do not rewrite this rel. - return null; - } - - final RelNode newJoin = - LogicalJoin.create(leftFrame.r, rightFrame.r, - decorrelateExpr(currentRel, map, cm, rel.getCondition()), - ImmutableSet.of(), rel.getJoinType()); - - // Create the mapping between the output of the old correlation rel - // and the new join rel - Map mapOldToNewOutputs = new HashMap<>(); - - int oldLeftFieldCount = oldLeft.getRowType().getFieldCount(); - int newLeftFieldCount = leftFrame.r.getRowType().getFieldCount(); - - int oldRightFieldCount = oldRight.getRowType().getFieldCount(); - //noinspection AssertWithSideEffects - assert rel.getRowType().getFieldCount() - == oldLeftFieldCount + oldRightFieldCount; - - // Left input positions are not changed. - mapOldToNewOutputs.putAll(leftFrame.oldToNewOutputs); - - // Right input positions are shifted by newLeftFieldCount. - for (int i = 0; i < oldRightFieldCount; i++) { - mapOldToNewOutputs.put(i + oldLeftFieldCount, - rightFrame.oldToNewOutputs.get(i) + newLeftFieldCount); - } - - final SortedMap corDefOutputs = - new TreeMap<>(leftFrame.corDefOutputs); - - // Right input positions are shifted by newLeftFieldCount. - for (Map.Entry entry - : rightFrame.corDefOutputs.entrySet()) { - corDefOutputs.put(entry.getKey(), - entry.getValue() + newLeftFieldCount); - } - return register(rel, newJoin, mapOldToNewOutputs, corDefOutputs); - } - - private static RexInputRef getNewForOldInputRef(RelNode currentRel, - Map map, RexInputRef oldInputRef) { - assert currentRel != null; - - int oldOrdinal = oldInputRef.getIndex(); - int newOrdinal = 0; - - // determine which input rel oldOrdinal references, and adjust - // oldOrdinal to be relative to that input rel - RelNode oldInput = null; - - for (RelNode oldInput0 : currentRel.getInputs()) { - RelDataType oldInputType = oldInput0.getRowType(); - int n = oldInputType.getFieldCount(); - if (oldOrdinal < n) { - oldInput = oldInput0; - break; - } - RelNode newInput = map.get(oldInput0).r; - newOrdinal += newInput.getRowType().getFieldCount(); - oldOrdinal -= n; - } - - assert oldInput != null; - - final Frame frame = map.get(oldInput); - assert frame != null; - - // now oldOrdinal is relative to oldInput - int oldLocalOrdinal = oldOrdinal; - - // figure out the newLocalOrdinal, relative to the newInput. - int newLocalOrdinal = oldLocalOrdinal; - - if (!frame.oldToNewOutputs.isEmpty()) { - newLocalOrdinal = frame.oldToNewOutputs.get(oldLocalOrdinal); - } - - newOrdinal += newLocalOrdinal; - - return new RexInputRef(newOrdinal, - frame.r.getRowType().getFieldList().get(newLocalOrdinal).getType()); - } - - /** - * Pulls project above the join from its RHS input. Enforces nullability - * for join output. - * - * @param join Join - * @param project Original project as the right-hand input of the join - * @param nullIndicatorPos Position of null indicator - * @return the subtree with the new Project at the root - */ - private RelNode projectJoinOutputWithNullability( - LogicalJoin join, - LogicalProject project, - int nullIndicatorPos) { - final RelDataTypeFactory typeFactory = join.getCluster().getTypeFactory(); - final RelNode left = join.getLeft(); - final JoinRelType joinType = join.getJoinType(); - - RexInputRef nullIndicator = - new RexInputRef( - nullIndicatorPos, - typeFactory.createTypeWithNullability( - join.getRowType().getFieldList().get(nullIndicatorPos) - .getType(), - true)); - - // now create the new project - List> newProjExprs = new ArrayList<>(); - - // project everything from the LHS and then those from the original - // projRel - List leftInputFields = - left.getRowType().getFieldList(); - - for (int i = 0; i < leftInputFields.size(); i++) { - newProjExprs.add(RexInputRef.of2(i, leftInputFields)); - } - - // Marked where the projected expr is coming from so that the types will - // become nullable for the original projections which are now coming out - // of the nullable side of the OJ. - boolean projectPulledAboveLeftCorrelator = - joinType.generatesNullsOnRight(); - - for (Pair pair : project.getNamedProjects()) { - RexNode newProjExpr = - removeCorrelationExpr( - pair.left, - projectPulledAboveLeftCorrelator, - nullIndicator); - - newProjExprs.add(Pair.of(newProjExpr, pair.right)); - } - - return relBuilder.push(join) - .projectNamed(Pair.left(newProjExprs), Pair.right(newProjExprs), true) - .build(); - } - - /** - * Pulls a {@link Project} above a {@link Correlate} from its RHS input. - * Enforces nullability for join output. - * - * @param correlate Correlate - * @param project the original project as the RHS input of the join - * @param isCount Positions which are calls to the COUNT - * aggregation function - * @return the subtree with the new Project at the root - */ - private RelNode aggregateCorrelatorOutput( - Correlate correlate, - LogicalProject project, - Set isCount) { - final RelNode left = correlate.getLeft(); - final JoinRelType joinType = correlate.getJoinType(); - - // now create the new project - final List> newProjects = new ArrayList<>(); - - // Project everything from the LHS and then those from the original - // project - final List leftInputFields = - left.getRowType().getFieldList(); - - for (int i = 0; i < leftInputFields.size(); i++) { - newProjects.add(RexInputRef.of2(i, leftInputFields)); - } - - // Marked where the projected expr is coming from so that the types will - // become nullable for the original projections which are now coming out - // of the nullable side of the OJ. - boolean projectPulledAboveLeftCorrelator = - joinType.generatesNullsOnRight(); - - for (Pair pair : project.getNamedProjects()) { - RexNode newProjExpr = - removeCorrelationExpr( - pair.left, - projectPulledAboveLeftCorrelator, - isCount); - newProjects.add(Pair.of(newProjExpr, pair.right)); - } - - return relBuilder.push(correlate) - .projectNamed(Pair.left(newProjects), Pair.right(newProjects), true) - .build(); - } - - /** - * Checks whether the correlations in projRel and filter are related to - * the correlated variables provided by corRel. - * - * @param correlate Correlate - * @param project The original Project as the RHS input of the join - * @param filter Filter - * @param correlatedJoinKeys Correlated join keys - * @return true if filter and proj only references corVar provided by corRel - */ - private boolean checkCorVars( - LogicalCorrelate correlate, - LogicalProject project, - LogicalFilter filter, - List correlatedJoinKeys) { - if (filter != null) { - assert correlatedJoinKeys != null; - - // check that all correlated refs in the filter condition are - // used in the join(as field access). - Set corVarInFilter = - Sets.newHashSet(cm.mapRefRelToCorRef.get(filter)); - - for (RexFieldAccess correlatedJoinKey : correlatedJoinKeys) { - corVarInFilter.remove(cm.mapFieldAccessToCorRef.get(correlatedJoinKey)); - } - - if (!corVarInFilter.isEmpty()) { - return false; - } - - // Check that the correlated variables referenced in these - // comparisons do come from the Correlate. - corVarInFilter.addAll(cm.mapRefRelToCorRef.get(filter)); - - for (CorRef corVar : corVarInFilter) { - if (cm.mapCorToCorRel.get(corVar.corr) != correlate) { - return false; - } - } - } - - // if project has any correlated reference, make sure they are also - // provided by the current correlate. They will be projected out of the LHS - // of the correlate. - if ((project != null) && cm.mapRefRelToCorRef.containsKey(project)) { - for (CorRef corVar : cm.mapRefRelToCorRef.get(project)) { - if (cm.mapCorToCorRel.get(corVar.corr) != correlate) { - return false; - } - } - } - - return true; - } - - /** - * Remove correlated variables from the tree at root corRel - * - * @param correlate Correlate - */ - private void removeCorVarFromTree(LogicalCorrelate correlate) { - if (cm.mapCorToCorRel.get(correlate.getCorrelationId()) == correlate) { - cm.mapCorToCorRel.remove(correlate.getCorrelationId()); - } - } - - /** - * Projects all {@code input} output fields plus the additional expressions. - * - * @param input Input relational expression - * @param additionalExprs Additional expressions and names - * @return the new Project - */ - private RelNode createProjectWithAdditionalExprs( - RelNode input, - List> additionalExprs) { - final List fieldList = - input.getRowType().getFieldList(); - List> projects = new ArrayList<>(); - for (Ord field : Ord.zip(fieldList)) { - projects.add( - Pair.of( - (RexNode) relBuilder.getRexBuilder().makeInputRef( - field.e.getType(), field.i), - field.e.getName())); - } - projects.addAll(additionalExprs); - return relBuilder.push(input) - .projectNamed(Pair.left(projects), Pair.right(projects), true) - .build(); - } - - /* Returns an immutable map with the identity [0: 0, .., count-1: count-1]. */ - static Map identityMap(int count) { - ImmutableMap.Builder builder = ImmutableMap.builder(); - for (int i = 0; i < count; i++) { - builder.put(i, i); - } - return builder.build(); - } - - /** Registers a relational expression and the relational expression it became - * after decorrelation. */ - Frame register(RelNode rel, RelNode newRel, - Map oldToNewOutputs, - SortedMap corDefOutputs) { - final Frame frame = new Frame(rel, newRel, corDefOutputs, oldToNewOutputs); - map.put(rel, frame); - return frame; - } - - static boolean allLessThan(Collection integers, int limit, - Litmus ret) { - for (int value : integers) { - if (value >= limit) { - return ret.fail("out of range; value: {}, limit: {}", value, limit); - } - } - return ret.succeed(); - } - - private static RelNode stripHep(RelNode rel) { - if (rel instanceof HepRelVertex) { - HepRelVertex hepRelVertex = (HepRelVertex) rel; - rel = hepRelVertex.getCurrentRel(); - } - return rel; - } - - //~ Inner Classes ---------------------------------------------------------- - - /** Shuttle that decorrelates. */ - private static class DecorrelateRexShuttle extends RexShuttle { - private final RelNode currentRel; - private final Map map; - private final CorelMap cm; - - private DecorrelateRexShuttle(RelNode currentRel, - Map map, CorelMap cm) { - this.currentRel = Objects.requireNonNull(currentRel); - this.map = Objects.requireNonNull(map); - this.cm = Objects.requireNonNull(cm); - } - - @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { - int newInputOutputOffset = 0; - for (RelNode input : currentRel.getInputs()) { - final Frame frame = map.get(input); - - if (frame != null) { - // try to find in this input rel the position of corVar - final CorRef corRef = cm.mapFieldAccessToCorRef.get(fieldAccess); - - if (corRef != null) { - Integer newInputPos = frame.corDefOutputs.get(corRef.def()); - if (newInputPos != null) { - // This input does produce the corVar referenced. - return new RexInputRef(newInputPos + newInputOutputOffset, - frame.r.getRowType().getFieldList().get(newInputPos) - .getType()); - } - } - - // this input does not produce the corVar needed - newInputOutputOffset += frame.r.getRowType().getFieldCount(); - } else { - // this input is not rewritten - newInputOutputOffset += input.getRowType().getFieldCount(); - } - } - return fieldAccess; - } - - @Override public RexNode visitInputRef(RexInputRef inputRef) { - final RexInputRef ref = getNewForOldInputRef(currentRel, map, inputRef); - if (ref.getIndex() == inputRef.getIndex() - && ref.getType() == inputRef.getType()) { - return inputRef; // re-use old object, to prevent needless expr cloning - } - return ref; - } - } - - /** Shuttle that removes correlations. */ - private class RemoveCorrelationRexShuttle extends RexShuttle { - final RexBuilder rexBuilder; - final RelDataTypeFactory typeFactory; - final boolean projectPulledAboveLeftCorrelator; - final RexInputRef nullIndicator; - final ImmutableSet isCount; - - RemoveCorrelationRexShuttle( - RexBuilder rexBuilder, - boolean projectPulledAboveLeftCorrelator, - RexInputRef nullIndicator, - Set isCount) { - this.projectPulledAboveLeftCorrelator = - projectPulledAboveLeftCorrelator; - this.nullIndicator = nullIndicator; // may be null - this.isCount = ImmutableSet.copyOf(isCount); - this.rexBuilder = rexBuilder; - this.typeFactory = rexBuilder.getTypeFactory(); - } - - private RexNode createCaseExpression( - RexInputRef nullInputRef, - RexLiteral lit, - RexNode rexNode) { - RexNode[] caseOperands = new RexNode[3]; - - // Construct a CASE expression to handle the null indicator. - // - // This also covers the case where a left correlated sub-query - // projects fields from outer relation. Since LOJ cannot produce - // nulls on the LHS, the projection now need to make a nullable LHS - // reference using a nullability indicator. If this this indicator - // is null, it means the sub-query does not produce any value. As a - // result, any RHS ref by this usbquery needs to produce null value. - - // WHEN indicator IS NULL - caseOperands[0] = - rexBuilder.makeCall( - SqlStdOperatorTable.IS_NULL, - new RexInputRef( - nullInputRef.getIndex(), - typeFactory.createTypeWithNullability( - nullInputRef.getType(), - true))); - - // THEN CAST(NULL AS newInputTypeNullable) - caseOperands[1] = - rexBuilder.makeCast( - typeFactory.createTypeWithNullability( - rexNode.getType(), - true), - lit); - - // ELSE cast (newInput AS newInputTypeNullable) END - caseOperands[2] = - rexBuilder.makeCast( - typeFactory.createTypeWithNullability( - rexNode.getType(), - true), - rexNode); - - return rexBuilder.makeCall( - SqlStdOperatorTable.CASE, - caseOperands); - } - - @Override public RexNode visitFieldAccess(RexFieldAccess fieldAccess) { - if (cm.mapFieldAccessToCorRef.containsKey(fieldAccess)) { - // if it is a corVar, change it to be input ref. - CorRef corVar = cm.mapFieldAccessToCorRef.get(fieldAccess); - - // corVar offset should point to the leftInput of currentRel, - // which is the Correlate. - RexNode newRexNode = - new RexInputRef(corVar.field, fieldAccess.getType()); - - if (projectPulledAboveLeftCorrelator - && (nullIndicator != null)) { - // need to enforce nullability by applying an additional - // cast operator over the transformed expression. - newRexNode = - createCaseExpression( - nullIndicator, - rexBuilder.constantNull(), - newRexNode); - } - return newRexNode; - } - return fieldAccess; - } - - @Override public RexNode visitInputRef(RexInputRef inputRef) { - if (currentRel instanceof LogicalCorrelate) { - // if this rel references corVar - // and now it needs to be rewritten - // it must have been pulled above the Correlate - // replace the input ref to account for the LHS of the - // Correlate - final int leftInputFieldCount = - ((LogicalCorrelate) currentRel).getLeft().getRowType() - .getFieldCount(); - RelDataType newType = inputRef.getType(); - - if (projectPulledAboveLeftCorrelator) { - newType = - typeFactory.createTypeWithNullability(newType, true); - } - - int pos = inputRef.getIndex(); - RexInputRef newInputRef = - new RexInputRef(leftInputFieldCount + pos, newType); - - if ((isCount != null) && isCount.contains(pos)) { - return createCaseExpression( - newInputRef, - rexBuilder.makeExactLiteral(BigDecimal.ZERO), - newInputRef); - } else { - return newInputRef; - } - } - return inputRef; - } - - @Override public RexNode visitLiteral(RexLiteral literal) { - // Use nullIndicator to decide whether to project null. - // Do nothing if the literal is null. - if (!RexUtil.isNull(literal) - && projectPulledAboveLeftCorrelator - && (nullIndicator != null)) { - return createCaseExpression( - nullIndicator, - rexBuilder.constantNull(), - literal); - } - return literal; - } - - @Override public RexNode visitCall(final RexCall call) { - RexNode newCall; - - boolean[] update = {false}; - List clonedOperands = visitList(call.operands, update); - if (update[0]) { - SqlOperator operator = call.getOperator(); - - boolean isSpecialCast = false; - if (operator instanceof SqlFunction) { - SqlFunction function = (SqlFunction) operator; - if (function.getKind() == SqlKind.CAST) { - if (call.operands.size() < 2) { - isSpecialCast = true; - } - } - } - - final RelDataType newType; - if (!isSpecialCast) { - // TODO: ideally this only needs to be called if the result - // type will also change. However, since that requires - // support from type inference rules to tell whether a rule - // decides return type based on input types, for now all - // operators will be recreated with new type if any operand - // changed, unless the operator has "built-in" type. - newType = rexBuilder.deriveReturnType(operator, clonedOperands); - } else { - // Use the current return type when creating a new call, for - // operators with return type built into the operator - // definition, and with no type inference rules, such as - // cast function with less than 2 operands. - - // TODO: Comments in RexShuttle.visitCall() mention other - // types in this category. Need to resolve those together - // and preferably in the base class RexShuttle. - newType = call.getType(); - } - newCall = - rexBuilder.makeCall( - newType, - operator, - clonedOperands); - } else { - newCall = call; - } - - if (projectPulledAboveLeftCorrelator && (nullIndicator != null)) { - return createCaseExpression( - nullIndicator, - rexBuilder.constantNull(), - newCall); - } - return newCall; - } - } - - /** - * Rule to remove single_value rel. For cases like - * - *
    AggRel single_value proj/filter/agg/ join on unique LHS key - * AggRel single group
    - */ - private final class RemoveSingleAggregateRule extends RelOptRule { - RemoveSingleAggregateRule(RelBuilderFactory relBuilderFactory) { - super( - operand( - LogicalAggregate.class, - operand( - LogicalProject.class, - operand(LogicalAggregate.class, any()))), - relBuilderFactory, null); - } - - public void onMatch(RelOptRuleCall call) { - LogicalAggregate singleAggregate = call.rel(0); - LogicalProject project = call.rel(1); - LogicalAggregate aggregate = call.rel(2); - - // check singleAggRel is single_value agg - if ((!singleAggregate.getGroupSet().isEmpty()) - || (singleAggregate.getAggCallList().size() != 1) - || !(singleAggregate.getAggCallList().get(0).getAggregation() - instanceof SqlSingleValueAggFunction)) { - return; - } - - // check projRel only projects one expression - // check this project only projects one expression, i.e. scalar - // sub-queries. - List projExprs = project.getProjects(); - if (projExprs.size() != 1) { - return; - } - - // check the input to project is an aggregate on the entire input - if (!aggregate.getGroupSet().isEmpty()) { - return; - } - - // singleAggRel produces a nullable type, so create the new - // projection that casts proj expr to a nullable type. - final RelBuilder relBuilder = call.builder(); - final RelDataType type = - relBuilder.getTypeFactory() - .createTypeWithNullability(projExprs.get(0).getType(), true); - final RexNode cast = - relBuilder.getRexBuilder().makeCast(type, projExprs.get(0)); - relBuilder.push(aggregate) - .project(cast); - call.transformTo(relBuilder.build()); - } - } - - /** Planner rule that removes correlations for scalar projects. */ - private final class RemoveCorrelationForScalarProjectRule extends RelOptRule { - RemoveCorrelationForScalarProjectRule(RelBuilderFactory relBuilderFactory) { - super( - operand(LogicalCorrelate.class, - operand(RelNode.class, any()), - operand(LogicalAggregate.class, - operand(LogicalProject.class, - operand(RelNode.class, any())))), - relBuilderFactory, null); - } - - public void onMatch(RelOptRuleCall call) { - final LogicalCorrelate correlate = call.rel(0); - final RelNode left = call.rel(1); - final LogicalAggregate aggregate = call.rel(2); - final LogicalProject project = call.rel(3); - RelNode right = call.rel(4); - final RelOptCluster cluster = correlate.getCluster(); - - setCurrent(call.getPlanner().getRoot(), correlate); - - // Check for this pattern. - // The pattern matching could be simplified if rules can be applied - // during decorrelation. - // - // Correlate(left correlation, condition = true) - // leftInput - // Aggregate (groupby (0) single_value()) - // Project-A (may reference corVar) - // rightInput - final JoinRelType joinType = correlate.getJoinType(); - - // corRel.getCondition was here, however Correlate was updated so it - // never includes a join condition. The code was not modified for brevity. - RexNode joinCond = relBuilder.literal(true); - if ((joinType != JoinRelType.LEFT) - || (joinCond != relBuilder.literal(true))) { - return; - } - - // check that the agg is of the following type: - // doing a single_value() on the entire input - if ((!aggregate.getGroupSet().isEmpty()) - || (aggregate.getAggCallList().size() != 1) - || !(aggregate.getAggCallList().get(0).getAggregation() - instanceof SqlSingleValueAggFunction)) { - return; - } - - // check this project only projects one expression, i.e. scalar - // sub-queries. - if (project.getProjects().size() != 1) { - return; - } - - int nullIndicatorPos; - - if ((right instanceof LogicalFilter) - && cm.mapRefRelToCorRef.containsKey(right)) { - // rightInput has this shape: - // - // Filter (references corVar) - // filterInput - - // If rightInput is a filter and contains correlated - // reference, make sure the correlated keys in the filter - // condition forms a unique key of the RHS. - - LogicalFilter filter = (LogicalFilter) right; - right = filter.getInput(); - - assert right instanceof HepRelVertex; - right = ((HepRelVertex) right).getCurrentRel(); - - // check filter input contains no correlation - if (RelOptUtil.getVariablesUsed(right).size() > 0) { - return; - } - - // extract the correlation out of the filter - - // First breaking up the filter conditions into equality - // comparisons between rightJoinKeys (from the original - // filterInput) and correlatedJoinKeys. correlatedJoinKeys - // can be expressions, while rightJoinKeys need to be input - // refs. These comparisons are AND'ed together. - List tmpRightJoinKeys = new ArrayList<>(); - List correlatedJoinKeys = new ArrayList<>(); - RelOptUtil.splitCorrelatedFilterCondition( - filter, - tmpRightJoinKeys, - correlatedJoinKeys, - false); - - // check that the columns referenced in these comparisons form - // an unique key of the filterInput - final List rightJoinKeys = new ArrayList<>(); - for (RexNode key : tmpRightJoinKeys) { - assert key instanceof RexInputRef; - rightJoinKeys.add((RexInputRef) key); - } - - // check that the columns referenced in rightJoinKeys form an - // unique key of the filterInput - if (rightJoinKeys.isEmpty()) { - return; - } - - // The join filters out the nulls. So, it's ok if there are - // nulls in the join keys. - final RelMetadataQuery mq = call.getMetadataQuery(); - if (!RelMdUtil.areColumnsDefinitelyUniqueWhenNullsFiltered(mq, right, - rightJoinKeys)) { - SQL2REL_LOGGER.debug("{} are not unique keys for {}", - rightJoinKeys.toString(), right.toString()); - return; - } - - RexUtil.FieldAccessFinder visitor = - new RexUtil.FieldAccessFinder(); - RexUtil.apply(visitor, correlatedJoinKeys, null); - List correlatedKeyList = - visitor.getFieldAccessList(); - - if (!checkCorVars(correlate, project, filter, correlatedKeyList)) { - return; - } - - // Change the plan to this structure. - // Note that the Aggregate is removed. - // - // Project-A' (replace corVar to input ref from the Join) - // Join (replace corVar to input ref from leftInput) - // leftInput - // rightInput (previously filterInput) - - // Change the filter condition into a join condition - joinCond = - removeCorrelationExpr(filter.getCondition(), false); - - nullIndicatorPos = - left.getRowType().getFieldCount() - + rightJoinKeys.get(0).getIndex(); - } else if (cm.mapRefRelToCorRef.containsKey(project)) { - // check filter input contains no correlation - if (RelOptUtil.getVariablesUsed(right).size() > 0) { - return; - } - - if (!checkCorVars(correlate, project, null, null)) { - return; - } - - // Change the plan to this structure. - // - // Project-A' (replace corVar to input ref from Join) - // Join (left, condition = true) - // leftInput - // Aggregate(groupby(0), single_value(0), s_v(1)....) - // Project-B (everything from input plus literal true) - // projectInput - - // make the new Project to provide a null indicator - right = - createProjectWithAdditionalExprs(right, - ImmutableList.of( - Pair.of(relBuilder.literal(true), "nullIndicator"))); - - // make the new aggRel - right = - RelOptUtil.createSingleValueAggRel(cluster, right); - - // The last field: - // single_value(true) - // is the nullIndicator - nullIndicatorPos = - left.getRowType().getFieldCount() - + right.getRowType().getFieldCount() - 1; - } else { - return; - } - - // make the new join rel - LogicalJoin join = - LogicalJoin.create(left, right, joinCond, - ImmutableSet.of(), joinType); - - RelNode newProject = - projectJoinOutputWithNullability(join, project, nullIndicatorPos); - - call.transformTo(newProject); - - removeCorVarFromTree(correlate); - } - } - - /** Planner rule that removes correlations for scalar aggregates. */ - private final class RemoveCorrelationForScalarAggregateRule - extends RelOptRule { - RemoveCorrelationForScalarAggregateRule(RelBuilderFactory relBuilderFactory) { - super( - operand(LogicalCorrelate.class, - operand(RelNode.class, any()), - operand(LogicalProject.class, - operandJ(LogicalAggregate.class, null, Aggregate::isSimple, - operand(LogicalProject.class, - operand(RelNode.class, any()))))), - relBuilderFactory, null); - } - - public void onMatch(RelOptRuleCall call) { - final LogicalCorrelate correlate = call.rel(0); - final RelNode left = call.rel(1); - final LogicalProject aggOutputProject = call.rel(2); - final LogicalAggregate aggregate = call.rel(3); - final LogicalProject aggInputProject = call.rel(4); - RelNode right = call.rel(5); - final RelBuilder builder = call.builder(); - final RexBuilder rexBuilder = builder.getRexBuilder(); - final RelOptCluster cluster = correlate.getCluster(); - - setCurrent(call.getPlanner().getRoot(), correlate); - - // check for this pattern - // The pattern matching could be simplified if rules can be applied - // during decorrelation, - // - // CorrelateRel(left correlation, condition = true) - // leftInput - // Project-A (a RexNode) - // Aggregate (groupby (0), agg0(), agg1()...) - // Project-B (references coVar) - // rightInput - - // check aggOutputProject projects only one expression - final List aggOutputProjects = aggOutputProject.getProjects(); - if (aggOutputProjects.size() != 1) { - return; - } - - final JoinRelType joinType = correlate.getJoinType(); - // corRel.getCondition was here, however Correlate was updated so it - // never includes a join condition. The code was not modified for brevity. - RexNode joinCond = rexBuilder.makeLiteral(true); - if ((joinType != JoinRelType.LEFT) - || (joinCond != rexBuilder.makeLiteral(true))) { - return; - } - - // check that the agg is on the entire input - if (!aggregate.getGroupSet().isEmpty()) { - return; - } - - final List aggInputProjects = aggInputProject.getProjects(); - - final List aggCalls = aggregate.getAggCallList(); - final Set isCountStar = new HashSet<>(); - - // mark if agg produces count(*) which needs to reference the - // nullIndicator after the transformation. - int k = -1; - for (AggregateCall aggCall : aggCalls) { - ++k; - if ((aggCall.getAggregation() instanceof SqlCountAggFunction) - && (aggCall.getArgList().size() == 0)) { - isCountStar.add(k); - } - } - - if ((right instanceof LogicalFilter) - && cm.mapRefRelToCorRef.containsKey(right)) { - // rightInput has this shape: - // - // Filter (references corVar) - // filterInput - LogicalFilter filter = (LogicalFilter) right; - right = filter.getInput(); - - assert right instanceof HepRelVertex; - right = ((HepRelVertex) right).getCurrentRel(); - - // check filter input contains no correlation - if (RelOptUtil.getVariablesUsed(right).size() > 0) { - return; - } - - // check filter condition type First extract the correlation out - // of the filter - - // First breaking up the filter conditions into equality - // comparisons between rightJoinKeys(from the original - // filterInput) and correlatedJoinKeys. correlatedJoinKeys - // can only be RexFieldAccess, while rightJoinKeys can be - // expressions. These comparisons are AND'ed together. - List rightJoinKeys = new ArrayList<>(); - List tmpCorrelatedJoinKeys = new ArrayList<>(); - RelOptUtil.splitCorrelatedFilterCondition( - filter, - rightJoinKeys, - tmpCorrelatedJoinKeys, - true); - - // make sure the correlated reference forms a unique key check - // that the columns referenced in these comparisons form an - // unique key of the leftInput - List correlatedJoinKeys = new ArrayList<>(); - List correlatedInputRefJoinKeys = new ArrayList<>(); - for (RexNode joinKey : tmpCorrelatedJoinKeys) { - assert joinKey instanceof RexFieldAccess; - correlatedJoinKeys.add((RexFieldAccess) joinKey); - RexNode correlatedInputRef = - removeCorrelationExpr(joinKey, false); - assert correlatedInputRef instanceof RexInputRef; - correlatedInputRefJoinKeys.add( - (RexInputRef) correlatedInputRef); - } - - // check that the columns referenced in rightJoinKeys form an - // unique key of the filterInput - if (correlatedInputRefJoinKeys.isEmpty()) { - return; - } - - // The join filters out the nulls. So, it's ok if there are - // nulls in the join keys. - final RelMetadataQuery mq = call.getMetadataQuery(); - if (!RelMdUtil.areColumnsDefinitelyUniqueWhenNullsFiltered(mq, left, - correlatedInputRefJoinKeys)) { - SQL2REL_LOGGER.debug("{} are not unique keys for {}", - correlatedJoinKeys.toString(), left.toString()); - return; - } - - // check corVar references are valid - if (!checkCorVars(correlate, - aggInputProject, - filter, - correlatedJoinKeys)) { - return; - } - - // Rewrite the above plan: - // - // Correlate(left correlation, condition = true) - // leftInput - // Project-A (a RexNode) - // Aggregate (groupby(0), agg0(),agg1()...) - // Project-B (may reference corVar) - // Filter (references corVar) - // rightInput (no correlated reference) - // - - // to this plan: - // - // Project-A' (all gby keys + rewritten nullable ProjExpr) - // Aggregate (groupby(all left input refs) - // agg0(rewritten expression), - // agg1()...) - // Project-B' (rewritten original projected exprs) - // Join(replace corVar w/ input ref from leftInput) - // leftInput - // rightInput - // - - // In the case where agg is count(*) or count($corVar), it is - // changed to count(nullIndicator). - // Note: any non-nullable field from the RHS can be used as - // the indicator however a "true" field is added to the - // projection list from the RHS for simplicity to avoid - // searching for non-null fields. - // - // Project-A' (all gby keys + rewritten nullable ProjExpr) - // Aggregate (groupby(all left input refs), - // count(nullIndicator), other aggs...) - // Project-B' (all left input refs plus - // the rewritten original projected exprs) - // Join(replace corVar to input ref from leftInput) - // leftInput - // Project (everything from rightInput plus - // the nullIndicator "true") - // rightInput - // - - // first change the filter condition into a join condition - joinCond = - removeCorrelationExpr(filter.getCondition(), false); - } else if (cm.mapRefRelToCorRef.containsKey(aggInputProject)) { - // check rightInput contains no correlation - if (RelOptUtil.getVariablesUsed(right).size() > 0) { - return; - } - - // check corVar references are valid - if (!checkCorVars(correlate, aggInputProject, null, null)) { - return; - } - - int nFields = left.getRowType().getFieldCount(); - ImmutableBitSet allCols = ImmutableBitSet.range(nFields); - - // leftInput contains unique keys - // i.e. each row is distinct and can group by on all the left - // fields - final RelMetadataQuery mq = call.getMetadataQuery(); - if (!RelMdUtil.areColumnsDefinitelyUnique(mq, left, allCols)) { - SQL2REL_LOGGER.debug("There are no unique keys for {}", left); - return; - } - // - // Rewrite the above plan: - // - // CorrelateRel(left correlation, condition = true) - // leftInput - // Project-A (a RexNode) - // Aggregate (groupby(0), agg0(), agg1()...) - // Project-B (references coVar) - // rightInput (no correlated reference) - // - - // to this plan: - // - // Project-A' (all gby keys + rewritten nullable ProjExpr) - // Aggregate (groupby(all left input refs) - // agg0(rewritten expression), - // agg1()...) - // Project-B' (rewritten original projected exprs) - // Join (LOJ cond = true) - // leftInput - // rightInput - // - - // In the case where agg is count($corVar), it is changed to - // count(nullIndicator). - // Note: any non-nullable field from the RHS can be used as - // the indicator however a "true" field is added to the - // projection list from the RHS for simplicity to avoid - // searching for non-null fields. - // - // Project-A' (all gby keys + rewritten nullable ProjExpr) - // Aggregate (groupby(all left input refs), - // count(nullIndicator), other aggs...) - // Project-B' (all left input refs plus - // the rewritten original projected exprs) - // Join (replace corVar to input ref from leftInput) - // leftInput - // Project (everything from rightInput plus - // the nullIndicator "true") - // rightInput - } else { - return; - } - - RelDataType leftInputFieldType = left.getRowType(); - int leftInputFieldCount = leftInputFieldType.getFieldCount(); - int joinOutputProjExprCount = - leftInputFieldCount + aggInputProjects.size() + 1; - - right = - createProjectWithAdditionalExprs(right, - ImmutableList.of( - Pair.of(rexBuilder.makeLiteral(true), - "nullIndicator"))); - - LogicalJoin join = - LogicalJoin.create(left, right, joinCond, - ImmutableSet.of(), joinType); - - // To the consumer of joinOutputProjRel, nullIndicator is located - // at the end - int nullIndicatorPos = join.getRowType().getFieldCount() - 1; - - RexInputRef nullIndicator = - new RexInputRef( - nullIndicatorPos, - cluster.getTypeFactory().createTypeWithNullability( - join.getRowType().getFieldList() - .get(nullIndicatorPos).getType(), - true)); - - // first project all group-by keys plus the transformed agg input - List joinOutputProjects = new ArrayList<>(); - - // LOJ Join preserves LHS types - for (int i = 0; i < leftInputFieldCount; i++) { - joinOutputProjects.add( - rexBuilder.makeInputRef( - leftInputFieldType.getFieldList().get(i).getType(), i)); - } - - for (RexNode aggInputProjExpr : aggInputProjects) { - joinOutputProjects.add( - removeCorrelationExpr(aggInputProjExpr, - joinType.generatesNullsOnRight(), - nullIndicator)); - } - - joinOutputProjects.add( - rexBuilder.makeInputRef(join, nullIndicatorPos)); - - final RelNode joinOutputProject = builder.push(join) - .project(joinOutputProjects) - .build(); - - // nullIndicator is now at a different location in the output of - // the join - nullIndicatorPos = joinOutputProjExprCount - 1; - - final int groupCount = leftInputFieldCount; - - List newAggCalls = new ArrayList<>(); - k = -1; - for (AggregateCall aggCall : aggCalls) { - ++k; - final List argList; - - if (isCountStar.contains(k)) { - // this is a count(*), transform it to count(nullIndicator) - // the null indicator is located at the end - argList = Collections.singletonList(nullIndicatorPos); - } else { - argList = new ArrayList<>(); - - for (int aggArg : aggCall.getArgList()) { - argList.add(aggArg + groupCount); - } - } - - int filterArg = aggCall.filterArg < 0 ? aggCall.filterArg - : aggCall.filterArg + groupCount; - newAggCalls.add( - aggCall.adaptTo(joinOutputProject, argList, filterArg, - aggregate.getGroupCount(), groupCount)); - } - - ImmutableBitSet groupSet = - ImmutableBitSet.range(groupCount); - LogicalAggregate newAggregate = - LogicalAggregate.create(joinOutputProject, groupSet, null, - newAggCalls); - List newAggOutputProjectList = new ArrayList<>(); - for (int i : groupSet) { - newAggOutputProjectList.add( - rexBuilder.makeInputRef(newAggregate, i)); - } - - RexNode newAggOutputProjects = - removeCorrelationExpr(aggOutputProjects.get(0), false); - newAggOutputProjectList.add( - rexBuilder.makeCast( - cluster.getTypeFactory().createTypeWithNullability( - newAggOutputProjects.getType(), - true), - newAggOutputProjects)); - - builder.push(newAggregate) - .project(newAggOutputProjectList); - call.transformTo(builder.build()); - - removeCorVarFromTree(correlate); - } - } - - // REVIEW jhyde 29-Oct-2007: This rule is non-static, depends on the state - // of members in RelDecorrelator, and has side-effects in the decorrelator. - // This breaks the contract of a planner rule, and the rule will not be - // reusable in other planners. - - // REVIEW jvs 29-Oct-2007: Shouldn't it also be incorporating - // the flavor attribute into the description? - - /** Planner rule that adjusts projects when counts are added. */ - private final class AdjustProjectForCountAggregateRule extends RelOptRule { - final boolean flavor; - - AdjustProjectForCountAggregateRule(boolean flavor, - RelBuilderFactory relBuilderFactory) { - super( - flavor - ? operand(LogicalCorrelate.class, - operand(RelNode.class, any()), - operand(LogicalProject.class, - operand(LogicalAggregate.class, any()))) - : operand(LogicalCorrelate.class, - operand(RelNode.class, any()), - operand(LogicalAggregate.class, any())), - relBuilderFactory, null); - this.flavor = flavor; - } - - public void onMatch(RelOptRuleCall call) { - final LogicalCorrelate correlate = call.rel(0); - final RelNode left = call.rel(1); - final LogicalProject aggOutputProject; - final LogicalAggregate aggregate; - if (flavor) { - aggOutputProject = call.rel(2); - aggregate = call.rel(3); - } else { - aggregate = call.rel(2); - - // Create identity projection - final List> projects = new ArrayList<>(); - final List fields = - aggregate.getRowType().getFieldList(); - for (int i = 0; i < fields.size(); i++) { - projects.add(RexInputRef.of2(projects.size(), fields)); - } - final RelBuilder relBuilder = call.builder(); - relBuilder.push(aggregate) - .projectNamed(Pair.left(projects), Pair.right(projects), true); - aggOutputProject = (LogicalProject) relBuilder.build(); - } - onMatch2(call, correlate, left, aggOutputProject, aggregate); - } - - private void onMatch2( - RelOptRuleCall call, - LogicalCorrelate correlate, - RelNode leftInput, - LogicalProject aggOutputProject, - LogicalAggregate aggregate) { - if (generatedCorRels.contains(correlate)) { - // This Correlate was generated by a previous invocation of - // this rule. No further work to do. - return; - } - - setCurrent(call.getPlanner().getRoot(), correlate); - - // check for this pattern - // The pattern matching could be simplified if rules can be applied - // during decorrelation, - // - // CorrelateRel(left correlation, condition = true) - // leftInput - // Project-A (a RexNode) - // Aggregate (groupby (0), agg0(), agg1()...) - - // check aggOutputProj projects only one expression - List aggOutputProjExprs = aggOutputProject.getProjects(); - if (aggOutputProjExprs.size() != 1) { - return; - } - - JoinRelType joinType = correlate.getJoinType(); - // corRel.getCondition was here, however Correlate was updated so it - // never includes a join condition. The code was not modified for brevity. - RexNode joinCond = relBuilder.literal(true); - if ((joinType != JoinRelType.LEFT) - || (joinCond != relBuilder.literal(true))) { - return; - } - - // check that the agg is on the entire input - if (!aggregate.getGroupSet().isEmpty()) { - return; - } - - List aggCalls = aggregate.getAggCallList(); - Set isCount = new HashSet<>(); - - // remember the count() positions - int i = -1; - for (AggregateCall aggCall : aggCalls) { - ++i; - if (aggCall.getAggregation() instanceof SqlCountAggFunction) { - isCount.add(i); - } - } - - // now rewrite the plan to - // - // Project-A' (all LHS plus transformed original projections, - // replacing references to count() with case statement) - // Correlate(left correlation, condition = true) - // leftInput - // Aggregate(groupby (0), agg0(), agg1()...) - // - LogicalCorrelate newCorrelate = - LogicalCorrelate.create(leftInput, aggregate, - correlate.getCorrelationId(), correlate.getRequiredColumns(), - correlate.getJoinType()); - - // remember this rel so we don't fire rule on it again - // REVIEW jhyde 29-Oct-2007: rules should not save state; rule - // should recognize patterns where it does or does not need to do - // work - generatedCorRels.add(newCorrelate); - - // need to update the mapCorToCorRel Update the output position - // for the corVars: only pass on the corVars that are not used in - // the join key. - if (cm.mapCorToCorRel.get(correlate.getCorrelationId()) == correlate) { - cm.mapCorToCorRel.put(correlate.getCorrelationId(), newCorrelate); - } - - RelNode newOutput = - aggregateCorrelatorOutput(newCorrelate, aggOutputProject, isCount); - - call.transformTo(newOutput); - } - } - - /** - * A unique reference to a correlation field. - * - *

    For instance, if a RelNode references emp.name multiple times, it would - * result in multiple {@code CorRef} objects that differ just in - * {@link CorRef#uniqueKey}. - */ - static class CorRef implements Comparable { - public final int uniqueKey; - public final CorrelationId corr; - public final int field; - - CorRef(CorrelationId corr, int field, int uniqueKey) { - this.corr = corr; - this.field = field; - this.uniqueKey = uniqueKey; - } - - @Override public String toString() { - return corr.getName() + '.' + field; - } - - @Override public int hashCode() { - return Objects.hash(uniqueKey, corr, field); - } - - @Override public boolean equals(Object o) { - return this == o - || o instanceof CorRef - && uniqueKey == ((CorRef) o).uniqueKey - && corr == ((CorRef) o).corr - && field == ((CorRef) o).field; - } - - public int compareTo(@Nonnull CorRef o) { - int c = corr.compareTo(o.corr); - if (c != 0) { - return c; - } - c = Integer.compare(field, o.field); - if (c != 0) { - return c; - } - return Integer.compare(uniqueKey, o.uniqueKey); - } - - public CorDef def() { - return new CorDef(corr, field); - } - } - - /** A correlation and a field. */ - static class CorDef implements Comparable { - public final CorrelationId corr; - public final int field; - - CorDef(CorrelationId corr, int field) { - this.corr = corr; - this.field = field; - } - - @Override public String toString() { - return corr.getName() + '.' + field; - } - - @Override public int hashCode() { - return Objects.hash(corr, field); - } - - @Override public boolean equals(Object o) { - return this == o - || o instanceof CorDef - && corr == ((CorDef) o).corr - && field == ((CorDef) o).field; - } - - public int compareTo(@Nonnull CorDef o) { - int c = corr.compareTo(o.corr); - if (c != 0) { - return c; - } - return Integer.compare(field, o.field); - } - } - - /** A map of the locations of - * {@link org.apache.calcite.rel.logical.LogicalCorrelate} - * in a tree of {@link RelNode}s. - * - *

    It is used to drive the decorrelation process. - * Treat it as immutable; rebuild if you modify the tree. - * - *

    There are three maps:

      - * - *
    1. {@link #mapRefRelToCorRef} maps a {@link RelNode} to the correlated - * variables it references; - * - *
    2. {@link #mapCorToCorRel} maps a correlated variable to the - * {@link Correlate} providing it; - * - *
    3. {@link #mapFieldAccessToCorRef} maps a rex field access to - * the corVar it represents. Because typeFlattener does not clone or - * modify a correlated field access this map does not need to be - * updated. - * - *
    */ - private static class CorelMap { - private final Multimap mapRefRelToCorRef; - private final SortedMap mapCorToCorRel; - private final Map mapFieldAccessToCorRef; - - // TODO: create immutable copies of all maps - private CorelMap(Multimap mapRefRelToCorRef, - SortedMap mapCorToCorRel, - Map mapFieldAccessToCorRef) { - this.mapRefRelToCorRef = mapRefRelToCorRef; - this.mapCorToCorRel = mapCorToCorRel; - this.mapFieldAccessToCorRef = ImmutableMap.copyOf(mapFieldAccessToCorRef); - } - - @Override public String toString() { - return "mapRefRelToCorRef=" + mapRefRelToCorRef - + "\nmapCorToCorRel=" + mapCorToCorRel - + "\nmapFieldAccessToCorRef=" + mapFieldAccessToCorRef - + "\n"; - } - - @Override public boolean equals(Object obj) { - return obj == this - || obj instanceof CorelMap - && mapRefRelToCorRef.equals(((CorelMap) obj).mapRefRelToCorRef) - && mapCorToCorRel.equals(((CorelMap) obj).mapCorToCorRel) - && mapFieldAccessToCorRef.equals( - ((CorelMap) obj).mapFieldAccessToCorRef); - } - - @Override public int hashCode() { - return Objects.hash(mapRefRelToCorRef, mapCorToCorRel, - mapFieldAccessToCorRef); - } - - /** Creates a CorelMap with given contents. */ - public static CorelMap of( - SortedSetMultimap mapRefRelToCorVar, - SortedMap mapCorToCorRel, - Map mapFieldAccessToCorVar) { - return new CorelMap(mapRefRelToCorVar, mapCorToCorRel, - mapFieldAccessToCorVar); - } - - /** - * Returns whether there are any correlating variables in this statement. - * - * @return whether there are any correlating variables - */ - public boolean hasCorrelation() { - return !mapCorToCorRel.isEmpty(); - } - } - - /** Builds a {@link org.apache.calcite.sql2rel.RelDecorrelator.CorelMap}. */ - private static class CorelMapBuilder extends RelShuttleImpl { - final SortedMap mapCorToCorRel = - new TreeMap<>(); - - final SortedSetMultimap mapRefRelToCorRef = - MultimapBuilder.SortedSetMultimapBuilder.hashKeys() - .treeSetValues() - .build(); - - final Map mapFieldAccessToCorVar = new HashMap<>(); - - final Holder offset = Holder.of(0); - int corrIdGenerator = 0; - - /** Creates a CorelMap by iterating over a {@link RelNode} tree. */ - CorelMap build(RelNode... rels) { - for (RelNode rel : rels) { - stripHep(rel).accept(this); - } - return new CorelMap(mapRefRelToCorRef, mapCorToCorRel, - mapFieldAccessToCorVar); - } - - @Override public RelNode visit(LogicalJoin join) { - try { - stack.push(join); - join.getCondition().accept(rexVisitor(join)); - } finally { - stack.pop(); - } - return visitJoin(join); - } - - @Override protected RelNode visitChild(RelNode parent, int i, - RelNode input) { - return super.visitChild(parent, i, stripHep(input)); - } - - @Override public RelNode visit(LogicalCorrelate correlate) { - mapCorToCorRel.put(correlate.getCorrelationId(), correlate); - return visitJoin(correlate); - } - - private RelNode visitJoin(BiRel join) { - final int x = offset.get(); - visitChild(join, 0, join.getLeft()); - offset.set(x + join.getLeft().getRowType().getFieldCount()); - visitChild(join, 1, join.getRight()); - offset.set(x); - return join; - } - - @Override public RelNode visit(final LogicalFilter filter) { - try { - stack.push(filter); - filter.getCondition().accept(rexVisitor(filter)); - } finally { - stack.pop(); - } - return super.visit(filter); - } - - @Override public RelNode visit(LogicalProject project) { - try { - stack.push(project); - for (RexNode node : project.getProjects()) { - node.accept(rexVisitor(project)); - } - } finally { - stack.pop(); - } - return super.visit(project); - } - - private RexVisitorImpl rexVisitor(final RelNode rel) { - return new RexVisitorImpl(true) { - @Override public Void visitFieldAccess(RexFieldAccess fieldAccess) { - final RexNode ref = fieldAccess.getReferenceExpr(); - if (ref instanceof RexCorrelVariable) { - final RexCorrelVariable var = (RexCorrelVariable) ref; - if (mapFieldAccessToCorVar.containsKey(fieldAccess)) { - // for cases where different Rel nodes are referring to - // same correlation var (e.g. in case of NOT IN) - // avoid generating another correlation var - // and record the 'rel' is using the same correlation - mapRefRelToCorRef.put(rel, - mapFieldAccessToCorVar.get(fieldAccess)); - } else { - final CorRef correlation = - new CorRef(var.id, fieldAccess.getField().getIndex(), - corrIdGenerator++); - mapFieldAccessToCorVar.put(fieldAccess, correlation); - mapRefRelToCorRef.put(rel, correlation); - } - } - return super.visitFieldAccess(fieldAccess); - } - - @Override public Void visitSubQuery(RexSubQuery subQuery) { - subQuery.rel.accept(CorelMapBuilder.this); - return super.visitSubQuery(subQuery); - } - }; - } - } - - /** Frame describing the relational expression after decorrelation - * and where to find the output fields and correlation variables - * among its output fields. */ - static class Frame { - final RelNode r; - final ImmutableSortedMap corDefOutputs; - final ImmutableSortedMap oldToNewOutputs; - - Frame(RelNode oldRel, RelNode r, SortedMap corDefOutputs, - Map oldToNewOutputs) { - this.r = Objects.requireNonNull(r); - this.corDefOutputs = ImmutableSortedMap.copyOf(corDefOutputs); - this.oldToNewOutputs = ImmutableSortedMap.copyOf(oldToNewOutputs); - assert allLessThan(this.corDefOutputs.values(), - r.getRowType().getFieldCount(), Litmus.THROW); - assert allLessThan(this.oldToNewOutputs.keySet(), - oldRel.getRowType().getFieldCount(), Litmus.THROW); - assert allLessThan(this.oldToNewOutputs.values(), - r.getRowType().getFieldCount(), Litmus.THROW); - } - } -} - -// End RelDecorrelator.java diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/operations/SqlToOperationConverter.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/operations/SqlToOperationConverter.java index f26154f01bc73c24618f0797814c2666e9be9d0f..3a129653679437f986191a46ca1cc480a5ef0aa3 100644 --- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/operations/SqlToOperationConverter.java +++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/operations/SqlToOperationConverter.java @@ -33,7 +33,6 @@ import org.apache.flink.table.operations.ddl.CreateTableOperation; import org.apache.flink.table.operations.ddl.DropTableOperation; import org.apache.flink.table.planner.calcite.FlinkPlannerImpl; import org.apache.flink.table.planner.calcite.FlinkTypeFactory; -import org.apache.flink.table.planner.calcite.FlinkTypeSystem; import org.apache.flink.table.runtime.types.LogicalTypeDataTypeConverter; import org.apache.calcite.rel.RelRoot; @@ -43,7 +42,6 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -100,38 +98,27 @@ public class SqlToOperationConverter { */ private Operation convertCreateTable(SqlCreateTable sqlCreateTable) { // primary key and unique keys are not supported - if ((sqlCreateTable.getPrimaryKeyList() != null - && sqlCreateTable.getPrimaryKeyList().size() > 0) - || (sqlCreateTable.getUniqueKeysList() != null - && sqlCreateTable.getUniqueKeysList().size() > 0)) { + if ((sqlCreateTable.getPrimaryKeyList().size() > 0) + || (sqlCreateTable.getUniqueKeysList().size() > 0)) { throw new SqlConversionException("Primary key and unique key are not supported yet."); } // set with properties - SqlNodeList propertyList = sqlCreateTable.getPropertyList(); Map properties = new HashMap<>(); - if (propertyList != null) { - propertyList.getList().forEach(p -> - properties.put(((SqlTableOption) p).getKeyString().toLowerCase(), - ((SqlTableOption) p).getValueString())); - } + sqlCreateTable.getPropertyList().getList().forEach(p -> + properties.put(((SqlTableOption) p).getKeyString().toLowerCase(), + ((SqlTableOption) p).getValueString())); - TableSchema tableSchema = createTableSchema(sqlCreateTable, - new FlinkTypeFactory(new FlinkTypeSystem())); // need to make type factory singleton ? - String tableComment = ""; - if (sqlCreateTable.getComment() != null) { - tableComment = sqlCreateTable.getComment().getNlsString().getValue(); - } + TableSchema tableSchema = createTableSchema(sqlCreateTable); + String tableComment = sqlCreateTable.getComment().map(comment -> + comment.getNlsString().getValue()).orElse(null); // set partition key - List partitionKeys = new ArrayList<>(); - SqlNodeList partitionKey = sqlCreateTable.getPartitionKeyList(); - if (partitionKey != null) { - partitionKeys = partitionKey - .getList() - .stream() - .map(p -> ((SqlIdentifier) p).getSimple()) - .collect(Collectors.toList()); - } + List partitionKeys = sqlCreateTable.getPartitionKeyList() + .getList() + .stream() + .map(p -> ((SqlIdentifier) p).getSimple()) + .collect(Collectors.toList()); + CatalogTable catalogTable = new CatalogTableImpl(tableSchema, partitionKeys, properties, @@ -180,11 +167,9 @@ public class SqlToOperationConverter { *

    The returned table schema contains columns (a:int, b:varchar, c:timestamp). * * @param sqlCreateTable sql create table node. - * @param factory FlinkTypeFactory instance. * @return TableSchema */ - private TableSchema createTableSchema(SqlCreateTable sqlCreateTable, - FlinkTypeFactory factory) { + private TableSchema createTableSchema(SqlCreateTable sqlCreateTable) { // setup table columns SqlNodeList columnList = sqlCreateTable.getColumnList(); TableSchema physicalSchema = null; @@ -194,8 +179,10 @@ public class SqlToOperationConverter { .filter(n -> n instanceof SqlTableColumn).collect(Collectors.toList()); for (SqlNode node : physicalColumns) { SqlTableColumn column = (SqlTableColumn) node; - final RelDataType relType = column.getType().deriveType(factory, - column.getType().getNullable()); + final RelDataType relType = column.getType() + .deriveType( + flinkPlanner.getOrCreateSqlValidator(), + column.getType().getNullable()); builder.field(column.getName().getSimple(), LogicalTypeDataTypeConverter.fromLogicalTypeToDataType( FlinkTypeFactory.toLogicalType(relType))); diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateExpandDistinctAggregatesRule.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateExpandDistinctAggregatesRule.java index e824bd3796ac935703b0dfd6d1a4360259fee768..8023100a6d144df0ebded4abf0fd0f24ce114ba4 100644 --- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateExpandDistinctAggregatesRule.java +++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateExpandDistinctAggregatesRule.java @@ -25,6 +25,7 @@ import org.apache.calcite.linq4j.Ord; import org.apache.calcite.plan.Contexts; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelCollations; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Aggregate.Group; @@ -44,6 +45,7 @@ import org.apache.calcite.tools.RelBuilder; import org.apache.calcite.tools.RelBuilderFactory; import org.apache.calcite.util.ImmutableBitSet; import org.apache.calcite.util.ImmutableIntList; +import org.apache.calcite.util.Optionality; import org.apache.calcite.util.Pair; import org.apache.calcite.util.Util; @@ -149,7 +151,10 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule } // Find all of the agg expressions. We use a LinkedHashSet to ensure determinism. - int nonDistinctAggCallCount = 0; // find all aggregate calls without distinct + // Find all aggregate calls without distinct + int nonDistinctAggCallCount = 0; + // Find all aggregate calls without distinct but ignore MAX, MIN, BIT_AND, BIT_OR + int nonDistinctAggCallExcludingIgnoredCount = 0; int filterCount = 0; int unsupportedNonDistinctAggCallCount = 0; final Set, Integer>> argLists = new LinkedHashSet<>(); @@ -171,6 +176,11 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule default: ++unsupportedNonDistinctAggCallCount; } + if (aggCall.getAggregation().getDistinctOptionality() == Optionality.IGNORED) { + argLists.add(Pair.of(aggCall.getArgList(), aggCall.filterArg)); + } else { + ++nonDistinctAggCallExcludingIgnoredCount; + } } else { argLists.add(Pair.of(aggCall.getArgList(), aggCall.filterArg)); } @@ -182,7 +192,11 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule // If all of the agg expressions are distinct and have the same // arguments then we can use a more efficient form. - if (nonDistinctAggCallCount == 0 + + // MAX, MIN, BIT_AND, BIT_OR always ignore distinct attribute, + // when they are mixed in with other distinct agg calls, + // we can still use this promotion. + if (nonDistinctAggCallExcludingIgnoredCount == 0 && argLists.size() == 1 && aggregate.getGroupType() == Group.SIMPLE) { final Pair, Integer> pair = @@ -217,9 +231,8 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule final List refs = new ArrayList<>(); final List fieldNames = aggregate.getRowType().getFieldNames(); final ImmutableBitSet groupSet = aggregate.getGroupSet(); - final int groupAndIndicatorCount = - aggregate.getGroupCount() + aggregate.getIndicatorCount(); - for (int i : Util.range(groupAndIndicatorCount)) { + final int groupCount = aggregate.getGroupCount(); + for (int i : Util.range(groupCount)) { refs.add(RexInputRef.of(i, aggFields)); } @@ -234,8 +247,8 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule } refs.add( new RexInputRef( - groupAndIndicatorCount + newAggCallList.size(), - aggFields.get(groupAndIndicatorCount + i).getType())); + groupCount + newAggCallList.size(), + aggFields.get(groupCount + i).getType())); newAggCallList.add(aggCall); } @@ -319,7 +332,8 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule if (!aggCall.isDistinct()) { final AggregateCall newCall = AggregateCall.create(aggCall.getAggregation(), false, - aggCall.isApproximate(), aggCall.getArgList(), -1, + aggCall.isApproximate(), false, aggCall.getArgList(), -1, + RelCollations.EMPTY, ImmutableBitSet.of(bottomGroupSet).cardinality(), relBuilder.peek(), null, aggCall.name); bottomAggregateCalls.add(newCall); @@ -329,7 +343,7 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule relBuilder.push( aggregate.copy( aggregate.getTraitSet(), relBuilder.build(), - false, ImmutableBitSet.of(bottomGroupSet), null, bottomAggregateCalls)); + ImmutableBitSet.of(bottomGroupSet), null, bottomAggregateCalls)); // Add aggregate A (see the reference example above), the top aggregate // to handle the rest of the aggregation that the bottom aggregate hasn't handled @@ -347,8 +361,10 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule AggregateCall.create(aggCall.getAggregation(), false, aggCall.isApproximate(), + false, newArgList, -1, + RelCollations.EMPTY, originalGroupSet.cardinality(), relBuilder.peek(), aggCall.getType(), @@ -362,13 +378,14 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule if (aggCall.getAggregation().getKind() == SqlKind.COUNT) { newCall = AggregateCall.create(new SqlSumEmptyIsZeroAggFunction(), false, - aggCall.isApproximate(), newArgs, -1, - originalGroupSet.cardinality(), relBuilder.peek(), - aggCall.getType(), aggCall.getName()); + aggCall.isApproximate(), false, newArgs, -1, + RelCollations.EMPTY, originalGroupSet.cardinality(), + relBuilder.peek(), aggCall.getType(), aggCall.getName()); } else { newCall = AggregateCall.create(aggCall.getAggregation(), false, - aggCall.isApproximate(), newArgs, -1, + aggCall.isApproximate(), false, newArgs, -1, + RelCollations.EMPTY, originalGroupSet.cardinality(), relBuilder.peek(), aggCall.getType(), aggCall.name); } @@ -391,8 +408,8 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule } relBuilder.push( aggregate.copy(aggregate.getTraitSet(), - relBuilder.build(), aggregate.indicator, - ImmutableBitSet.of(topGroupSet), null, topAggregateCalls)); + relBuilder.build(), ImmutableBitSet.of(topGroupSet), + null, topAggregateCalls)); return relBuilder; } @@ -436,7 +453,8 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule final int z = groupCount + distinctAggCalls.size(); distinctAggCalls.add( AggregateCall.create(SqlStdOperatorTable.GROUPING, false, false, - ImmutableIntList.copyOf(fullGroupSet), -1, groupSets.size(), + false, ImmutableIntList.copyOf(fullGroupSet), -1, + RelCollations.EMPTY, groupSets.size(), relBuilder.peek(), null, "$g")); for (Ord groupSet : Ord.zip(groupSets)) { filters.put(groupSet.e, z + groupSet.i); @@ -493,7 +511,8 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule } final AggregateCall newCall = AggregateCall.create(aggregation, false, aggCall.isApproximate(), - newArgList, newFilterArg, aggregate.getGroupCount(), distinct, + false, newArgList, newFilterArg, + RelCollations.EMPTY, aggregate.getGroupCount(), distinct, null, aggCall.name); newCalls.add(newCall); aggCallIdx++; @@ -614,8 +633,7 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule final int cardinality = aggregate.getGroupSet().cardinality(); relBuilder.push( aggregate.copy(aggregate.getTraitSet(), relBuilder.build(), - aggregate.indicator, ImmutableBitSet.range(cardinality), null, - newAggCalls)); + ImmutableBitSet.range(cardinality), null, newAggCalls)); return relBuilder; } @@ -704,9 +722,8 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule final List aggCallList = new ArrayList<>(); final List aggCalls = aggregate.getAggCallList(); - final int groupAndIndicatorCount = - aggregate.getGroupCount() + aggregate.getIndicatorCount(); - int i = groupAndIndicatorCount - 1; + final int groupCount = aggregate.getGroupCount(); + int i = groupCount - 1; for (AggregateCall aggCall : aggCalls) { ++i; @@ -732,16 +749,17 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule aggCall.filterArg >= 0 ? sourceOf.get(aggCall.filterArg) : -1; final AggregateCall newAggCall = AggregateCall.create(aggCall.getAggregation(), false, - aggCall.isApproximate(), newArgs, - newFilterArg, aggCall.getType(), aggCall.getName()); + aggCall.isApproximate(), false, newArgs, + newFilterArg, RelCollations.EMPTY, + aggCall.getType(), aggCall.getName()); assert refs.get(i) == null; if (n == 0) { refs.set(i, - new RexInputRef(groupAndIndicatorCount + aggCallList.size(), + new RexInputRef(groupCount + aggCallList.size(), newAggCall.getType())); } else { refs.set(i, - new RexInputRef(leftFields.size() + groupAndIndicatorCount + new RexInputRef(leftFields.size() + groupCount + aggCallList.size(), newAggCall.getType())); } aggCallList.add(newAggCall); @@ -754,16 +772,9 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule final ImmutableBitSet newGroupSet = aggregate.getGroupSet().permute(map); assert newGroupSet .equals(ImmutableBitSet.range(aggregate.getGroupSet().cardinality())); - com.google.common.collect.ImmutableList newGroupingSets = null; - if (aggregate.indicator) { - newGroupingSets = - ImmutableBitSet.ORDERING.immutableSortedCopy( - ImmutableBitSet.permute(aggregate.getGroupSets(), map)); - } - relBuilder.push( aggregate.copy(aggregate.getTraitSet(), relBuilder.build(), - aggregate.indicator, newGroupSet, newGroupingSets, aggCallList)); + newGroupSet, null, aggCallList)); // If there's no left child yet, no need to create the join if (n == 0) { @@ -776,7 +787,7 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule final List distinctFields = relBuilder.peek().getRowType().getFieldList(); final List conditions = com.google.common.collect.Lists.newArrayList(); - for (i = 0; i < groupAndIndicatorCount; ++i) { + for (i = 0; i < groupCount; ++i) { // null values form its own group // use "is not distinct from" so that the join condition // allows null values to match. @@ -805,7 +816,8 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule // arguments. If we're rewriting aggregates whose args are {sal}, we will // rewrite COUNT(DISTINCT sal) and SUM(DISTINCT sal) but ignore // COUNT(DISTINCT gender) or SUM(sal). - if (!aggCall.isDistinct()) { + if (!aggCall.isDistinct() + && aggCall.getAggregation().getDistinctOptionality() != Optionality.IGNORED) { continue; } if (!aggCall.getArgList().equals(argList)) { @@ -821,8 +833,8 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule } final AggregateCall newAggCall = AggregateCall.create(aggCall.getAggregation(), false, - aggCall.isApproximate(), newArgs, -1, - aggCall.getType(), aggCall.getName()); + aggCall.isApproximate(), false, newArgs, -1, + RelCollations.EMPTY, aggCall.getType(), aggCall.getName()); newAggCalls.set(i, newAggCall); } } @@ -896,10 +908,7 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule RexNode condition = rexBuilder.makeCall(SqlStdOperatorTable.CASE, filterRef, argRef.left, - rexBuilder.ensureType(argRef.left.getType(), - rexBuilder.makeCast(argRef.left.getType(), - rexBuilder.constantNull()), - true)); + rexBuilder.makeNullLiteral(argRef.left.getType())); sourceOf.put(arg, projects.size()); projects.add(Pair.of(condition, "i$" + argRef.right)); continue; @@ -915,7 +924,7 @@ public final class FlinkAggregateExpandDistinctAggregatesRule extends RelOptRule // Get the distinct values of the GROUP BY fields and the arguments // to the agg functions. relBuilder.push( - aggregate.copy(aggregate.getTraitSet(), relBuilder.build(), false, + aggregate.copy(aggregate.getTraitSet(), relBuilder.build(), ImmutableBitSet.range(projects.size()), null, com.google.common.collect.ImmutableList.of())); return relBuilder; diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRule.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRule.java deleted file mode 100644 index 23ec403a11af83ed13e2cdf4bf6e14de65331a93..0000000000000000000000000000000000000000 --- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRule.java +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.table.planner.plan.rules.logical; - -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptRuleOperand; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.EquiJoin; -import org.apache.calcite.rel.core.Filter; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.type.RelDataType; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.tools.RelBuilderFactory; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; - -/** - * This rules is copied from Calcite's {@link org.apache.calcite.rel.rules.FilterJoinRule}. - * NOTES: This file should be deleted when CALCITE-3170 is fixed, - * and please make sure to synchronize with FlinkFilterJoinRule in flink planner when changing this class. - * Modification: - * - Handles the ON condition of anti-join can not be pushed down, lines added (192-198) - */ - -/** - * Planner rule that pushes filters above and - * within a join node into the join node and/or its children nodes. - */ -public abstract class FlinkFilterJoinRule extends RelOptRule { - /** Predicate that always returns true. With this predicate, every filter - * will be pushed into the ON clause. */ - public static final Predicate TRUE_PREDICATE = (join, joinType, exp) -> true; - - /** Rule that pushes predicates from a Filter into the Join below them. */ - public static final FlinkFilterJoinRule FILTER_ON_JOIN = - new FlinkFilterIntoJoinRule(true, RelFactories.LOGICAL_BUILDER, - TRUE_PREDICATE); - - /** Dumber version of {@link #FILTER_ON_JOIN}. Not intended for production - * use, but keeps some tests working for which {@code FILTER_ON_JOIN} is too - * smart. */ - public static final FlinkFilterJoinRule DUMB_FILTER_ON_JOIN = - new FlinkFilterIntoJoinRule(false, RelFactories.LOGICAL_BUILDER, - TRUE_PREDICATE); - - /** Rule that pushes predicates in a Join into the inputs to the join. */ - public static final FlinkFilterJoinRule JOIN = - new FlinkJoinConditionPushRule(RelFactories.LOGICAL_BUILDER, TRUE_PREDICATE); - - /** Whether to try to strengthen join-type. */ - private final boolean smart; - - /** Predicate that returns whether a filter is valid in the ON clause of a - * join for this particular kind of join. If not, Calcite will push it back to - * above the join. */ - private final Predicate predicate; - - //~ Constructors ----------------------------------------------------------- - - /** - * Creates a FilterProjectTransposeRule with an explicit root operand and - * factories. - */ - protected FlinkFilterJoinRule(RelOptRuleOperand operand, String id, - boolean smart, RelBuilderFactory relBuilderFactory, Predicate predicate) { - super(operand, relBuilderFactory, "FlinkFilterJoinRule:" + id); - this.smart = smart; - this.predicate = Objects.requireNonNull(predicate); - } - - /** - * Creates a FlinkFilterJoinRule with an explicit root operand and - * factories. - */ - @Deprecated // to be removed before 2.0 - protected FlinkFilterJoinRule(RelOptRuleOperand operand, String id, - boolean smart, RelFactories.FilterFactory filterFactory, - RelFactories.ProjectFactory projectFactory) { - this(operand, id, smart, RelBuilder.proto(filterFactory, projectFactory), - TRUE_PREDICATE); - } - - /** - * Creates a FilterProjectTransposeRule with an explicit root operand and - * factories. - */ - @Deprecated // to be removed before 2.0 - protected FlinkFilterJoinRule(RelOptRuleOperand operand, String id, - boolean smart, RelFactories.FilterFactory filterFactory, - RelFactories.ProjectFactory projectFactory, - Predicate predicate) { - this(operand, id, smart, RelBuilder.proto(filterFactory, projectFactory), - predicate); - } - - //~ Methods ---------------------------------------------------------------- - - protected void perform(RelOptRuleCall call, Filter filter, - Join join) { - final List joinFilters = - RelOptUtil.conjunctions(join.getCondition()); - final List origJoinFilters = com.google.common.collect.ImmutableList.copyOf(joinFilters); - - // If there is only the joinRel, - // make sure it does not match a cartesian product joinRel - // (with "true" condition), otherwise this rule will be applied - // again on the new cartesian product joinRel. - if (filter == null && joinFilters.isEmpty()) { - return; - } - - final List aboveFilters = - filter != null - ? RelOptUtil.conjunctions(filter.getCondition()) - : new ArrayList<>(); - final com.google.common.collect.ImmutableList origAboveFilters = - com.google.common.collect.ImmutableList.copyOf(aboveFilters); - - // Simplify Outer Joins - JoinRelType joinType = join.getJoinType(); - if (smart - && !origAboveFilters.isEmpty() - && join.getJoinType() != JoinRelType.INNER) { - joinType = RelOptUtil.simplifyJoin(join, origAboveFilters, joinType); - } - - final List leftFilters = new ArrayList<>(); - final List rightFilters = new ArrayList<>(); - - // TODO - add logic to derive additional filters. E.g., from - // (t1.a = 1 AND t2.a = 2) OR (t1.b = 3 AND t2.b = 4), you can - // derive table filters: - // (t1.a = 1 OR t1.b = 3) - // (t2.a = 2 OR t2.b = 4) - - // Try to push down above filters. These are typically where clause - // filters. They can be pushed down if they are not on the NULL - // generating side. - boolean filterPushed = false; - if (RelOptUtil.classifyFilters( - join, - aboveFilters, - joinType, - !(join instanceof EquiJoin), - !joinType.generatesNullsOnLeft(), - !joinType.generatesNullsOnRight(), - joinFilters, - leftFilters, - rightFilters)) { - filterPushed = true; - } - - // Move join filters up if needed - validateJoinFilters(aboveFilters, joinFilters, join, joinType); - - // If no filter got pushed after validate, reset filterPushed flag - if (leftFilters.isEmpty() - && rightFilters.isEmpty() - && joinFilters.size() == origJoinFilters.size()) { - if (com.google.common.collect.Sets.newHashSet(joinFilters) - .equals(com.google.common.collect.Sets.newHashSet(origJoinFilters))) { - filterPushed = false; - } - } - - boolean isAntiJoin = joinType == JoinRelType.ANTI; - - // Try to push down filters in ON clause. A ON clause filter can only be - // pushed down if it does not affect the non-matching set, i.e. it is - // not on the side which is preserved. - // A ON clause filter of anti-join can not be pushed down. - if (!isAntiJoin && RelOptUtil.classifyFilters( - join, - joinFilters, - joinType, - false, - !joinType.generatesNullsOnRight(), - !joinType.generatesNullsOnLeft(), - joinFilters, - leftFilters, - rightFilters)) { - filterPushed = true; - } - - // if nothing actually got pushed and there is nothing leftover, - // then this rule is a no-op - if ((!filterPushed - && joinType == join.getJoinType()) - || (joinFilters.isEmpty() - && leftFilters.isEmpty() - && rightFilters.isEmpty())) { - return; - } - - // create Filters on top of the children if any filters were - // pushed to them - final RexBuilder rexBuilder = join.getCluster().getRexBuilder(); - final RelBuilder relBuilder = call.builder(); - final RelNode leftRel = - relBuilder.push(join.getLeft()).filter(leftFilters).build(); - final RelNode rightRel = - relBuilder.push(join.getRight()).filter(rightFilters).build(); - - // create the new join node referencing the new children and - // containing its new join filters (if there are any) - final com.google.common.collect.ImmutableList fieldTypes = - com.google.common.collect.ImmutableList.builder() - .addAll(RelOptUtil.getFieldTypeList(leftRel.getRowType())) - .addAll(RelOptUtil.getFieldTypeList(rightRel.getRowType())).build(); - final RexNode joinFilter = - RexUtil.composeConjunction(rexBuilder, - RexUtil.fixUp(rexBuilder, joinFilters, fieldTypes)); - - // If nothing actually got pushed and there is nothing leftover, - // then this rule is a no-op - if (joinFilter.isAlwaysTrue() - && leftFilters.isEmpty() - && rightFilters.isEmpty() - && joinType == join.getJoinType()) { - return; - } - - RelNode newJoinRel = - join.copy( - join.getTraitSet(), - joinFilter, - leftRel, - rightRel, - joinType, - join.isSemiJoinDone()); - call.getPlanner().onCopy(join, newJoinRel); - if (!leftFilters.isEmpty()) { - call.getPlanner().onCopy(filter, leftRel); - } - if (!rightFilters.isEmpty()) { - call.getPlanner().onCopy(filter, rightRel); - } - - relBuilder.push(newJoinRel); - - // Create a project on top of the join if some of the columns have become - // NOT NULL due to the join-type getting stricter. - relBuilder.convert(join.getRowType(), false); - - // create a FilterRel on top of the join if needed - relBuilder.filter( - RexUtil.fixUp(rexBuilder, aboveFilters, - RelOptUtil.getFieldTypeList(relBuilder.peek().getRowType()))); - - call.transformTo(relBuilder.build()); - } - - /** - * Validates that target execution framework can satisfy join filters. - * - *

    If the join filter cannot be satisfied (for example, if it is - * {@code l.c1 > r.c2} and the join only supports equi-join), removes the - * filter from {@code joinFilters} and adds it to {@code aboveFilters}. - * - *

    The default implementation does nothing; i.e. the join can handle all - * conditions. - * - * @param aboveFilters Filter above Join - * @param joinFilters Filters in join condition - * @param join Join - * @param joinType JoinRelType could be different from type in Join due to - * outer join simplification. - */ - protected void validateJoinFilters(List aboveFilters, - List joinFilters, Join join, JoinRelType joinType) { - final Iterator filterIter = joinFilters.iterator(); - while (filterIter.hasNext()) { - RexNode exp = filterIter.next(); - if (!predicate.apply(join, joinType, exp)) { - aboveFilters.add(exp); - filterIter.remove(); - } - } - } - - /** Rule that pushes parts of the join condition to its inputs. */ - public static class FlinkJoinConditionPushRule extends FlinkFilterJoinRule { - public FlinkJoinConditionPushRule(RelBuilderFactory relBuilderFactory, - Predicate predicate) { - super(RelOptRule.operand(Join.class, RelOptRule.any()), - "FlinkFilterJoinRule:no-filter", true, relBuilderFactory, - predicate); - } - - @Deprecated // to be removed before 2.0 - public FlinkJoinConditionPushRule(RelFactories.FilterFactory filterFactory, - RelFactories.ProjectFactory projectFactory, Predicate predicate) { - this(RelBuilder.proto(filterFactory, projectFactory), predicate); - } - - @Override public void onMatch(RelOptRuleCall call) { - Join join = call.rel(0); - perform(call, null, join); - } - } - - /** Rule that tries to push filter expressions into a join - * condition and into the inputs of the join. */ - public static class FlinkFilterIntoJoinRule extends FlinkFilterJoinRule { - public FlinkFilterIntoJoinRule(boolean smart, - RelBuilderFactory relBuilderFactory, Predicate predicate) { - super( - operand(Filter.class, - operand(Join.class, RelOptRule.any())), - "FlinkFilterJoinRule:filter", smart, relBuilderFactory, - predicate); - } - - @Deprecated // to be removed before 2.0 - public FlinkFilterIntoJoinRule(boolean smart, - RelFactories.FilterFactory filterFactory, - RelFactories.ProjectFactory projectFactory, - Predicate predicate) { - this(smart, RelBuilder.proto(filterFactory, projectFactory), predicate); - } - - @Override public void onMatch(RelOptRuleCall call) { - Filter filter = call.rel(0); - Join join = call.rel(1); - perform(call, filter, join); - } - } - - /** Predicate that returns whether a filter is valid in the ON clause of a - * join for this particular kind of join. If not, Calcite will push it back to - * above the join. */ - public interface Predicate { - boolean apply(Join join, JoinRelType joinType, RexNode exp); - } -} - -// End FlinkFilterJoinRule.java diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinPushExpressionsRule.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinPushExpressionsRule.java index 310fe8b37d46adad7afaa050a65f38e19258b0ff..61b42e71ad29a9330bde05127e27b09e8e933a1b 100644 --- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinPushExpressionsRule.java +++ b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinPushExpressionsRule.java @@ -17,10 +17,9 @@ package org.apache.flink.table.planner.plan.rules.logical; -import org.apache.flink.table.planner.plan.utils.FlinkRelOptUtil; - import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.RelFactories; @@ -66,7 +65,7 @@ public class FlinkJoinPushExpressionsRule extends RelOptRule { Join join = call.rel(0); // Push expression in join condition into Project below Join. - RelNode newJoin = FlinkRelOptUtil.pushDownJoinConditions(join, call.builder()); + RelNode newJoin = RelOptUtil.pushDownJoinConditions(join, call.builder()); // If the join is the same, we bail out if (newJoin instanceof Join) { diff --git a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRule.java b/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRule.java deleted file mode 100644 index 0d4a9547b0e7cbc4108894a93a448fbafc989a81..0000000000000000000000000000000000000000 --- a/flink-table/flink-table-planner-blink/src/main/java/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRule.java +++ /dev/null @@ -1,594 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.table.planner.plan.rules.logical; - -import org.apache.calcite.plan.RelOptRule; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelOptUtil; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.core.Join; -import org.apache.calcite.rel.core.JoinRelType; -import org.apache.calcite.rel.core.RelFactories; -import org.apache.calcite.rel.logical.LogicalJoin; -import org.apache.calcite.rel.rules.MultiJoin; -import org.apache.calcite.rel.type.RelDataTypeField; -import org.apache.calcite.rex.RexBuilder; -import org.apache.calcite.rex.RexInputRef; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexUtil; -import org.apache.calcite.rex.RexVisitorImpl; -import org.apache.calcite.tools.RelBuilderFactory; -import org.apache.calcite.util.ImmutableBitSet; -import org.apache.calcite.util.ImmutableIntList; -import org.apache.calcite.util.Pair; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * This rule is copied from Calcite's {@link org.apache.calcite.rel.rules.JoinToMultiJoinRule}. - * This file should be removed while upgrading Calcite version to 1.21. [CALCITE-3225] - * Modification: - * - Does not match SEMI/ANTI join. lines changed (142-145) - * - lines changed (440-451) - */ - -/** - * Planner rule to flatten a tree of - * {@link org.apache.calcite.rel.logical.LogicalJoin}s - * into a single {@link MultiJoin} with N inputs. - * - *

    An input is not flattened if - * the input is a null generating input in an outer join, i.e., either input in - * a full outer join, the right hand side of a left outer join, or the left hand - * side of a right outer join. - * - *

    Join conditions are also pulled up from the inputs into the topmost - * {@link MultiJoin}, - * unless the input corresponds to a null generating input in an outer join, - * - *

    Outer join information is also stored in the {@link MultiJoin}. A - * boolean flag indicates if the join is a full outer join, and in the case of - * left and right outer joins, the join type and outer join conditions are - * stored in arrays in the {@link MultiJoin}. This outer join information is - * associated with the null generating input in the outer join. So, in the case - * of a a left outer join between A and B, the information is associated with B, - * not A. - * - *

    Here are examples of the {@link MultiJoin}s constructed after this rule - * has been applied on following join trees. - * - *

      - *
    • A JOIN B → MJ(A, B) - * - *
    • A JOIN B JOIN C → MJ(A, B, C) - * - *
    • A LEFT JOIN B → MJ(A, B), left outer join on input#1 - * - *
    • A RIGHT JOIN B → MJ(A, B), right outer join on input#0 - * - *
    • A FULL JOIN B → MJ[full](A, B) - * - *
    • A LEFT JOIN (B JOIN C) → MJ(A, MJ(B, C))), left outer join on - * input#1 in the outermost MultiJoin - * - *
    • (A JOIN B) LEFT JOIN C → MJ(A, B, C), left outer join on input#2 - * - *
    • (A LEFT JOIN B) JOIN C → MJ(MJ(A, B), C), left outer join on input#1 - * of the inner MultiJoin TODO - * - *
    • A LEFT JOIN (B FULL JOIN C) → MJ(A, MJ[full](B, C)), left outer join - * on input#1 in the outermost MultiJoin - * - *
    • (A LEFT JOIN B) FULL JOIN (C RIGHT JOIN D) → - * MJ[full](MJ(A, B), MJ(C, D)), left outer join on input #1 in the first - * inner MultiJoin and right outer join on input#0 in the second inner - * MultiJoin - *
    - * - *

    The constructor is parameterized to allow any sub-class of - * {@link org.apache.calcite.rel.core.Join}, not just - * {@link org.apache.calcite.rel.logical.LogicalJoin}.

    - * - * @see org.apache.calcite.rel.rules.FilterMultiJoinMergeRule - * @see org.apache.calcite.rel.rules.ProjectMultiJoinMergeRule - */ -public class FlinkJoinToMultiJoinRule extends RelOptRule { - public static final FlinkJoinToMultiJoinRule INSTANCE = - new FlinkJoinToMultiJoinRule(LogicalJoin.class, RelFactories.LOGICAL_BUILDER); - - //~ Constructors ----------------------------------------------------------- - - @Deprecated // to be removed before 2.0 - public FlinkJoinToMultiJoinRule(Class clazz) { - this(clazz, RelFactories.LOGICAL_BUILDER); - } - - /** - * Creates a FlinkJoinToMultiJoinRule. - */ - public FlinkJoinToMultiJoinRule(Class clazz, - RelBuilderFactory relBuilderFactory) { - super( - operand(clazz, - operand(RelNode.class, any()), - operand(RelNode.class, any())), - relBuilderFactory, null); - } - - //~ Methods ---------------------------------------------------------------- - - @Override - public boolean matches(RelOptRuleCall call) { - final Join origJoin = call.rel(0); - return origJoin.getJoinType() != JoinRelType.SEMI && origJoin.getJoinType() != JoinRelType.ANTI; - } - - public void onMatch(RelOptRuleCall call) { - final Join origJoin = call.rel(0); - final RelNode left = call.rel(1); - final RelNode right = call.rel(2); - - // combine the children MultiJoin inputs into an array of inputs - // for the new MultiJoin - final List projFieldsList = new ArrayList<>(); - final List joinFieldRefCountsList = new ArrayList<>(); - final List newInputs = - combineInputs( - origJoin, - left, - right, - projFieldsList, - joinFieldRefCountsList); - - // combine the outer join information from the left and right - // inputs, and include the outer join information from the current - // join, if it's a left/right outer join - final List> joinSpecs = new ArrayList<>(); - combineOuterJoins( - origJoin, - newInputs, - left, - right, - joinSpecs); - - // pull up the join filters from the children MultiJoinRels and - // combine them with the join filter associated with this LogicalJoin to - // form the join filter for the new MultiJoin - List newJoinFilters = combineJoinFilters(origJoin, left, right); - - // add on the join field reference counts for the join condition - // associated with this LogicalJoin - final com.google.common.collect.ImmutableMap newJoinFieldRefCountsMap = - addOnJoinFieldRefCounts(newInputs, - origJoin.getRowType().getFieldCount(), - origJoin.getCondition(), - joinFieldRefCountsList); - - List newPostJoinFilters = - combinePostJoinFilters(origJoin, left, right); - - final RexBuilder rexBuilder = origJoin.getCluster().getRexBuilder(); - RelNode multiJoin = - new MultiJoin( - origJoin.getCluster(), - newInputs, - RexUtil.composeConjunction(rexBuilder, newJoinFilters), - origJoin.getRowType(), - origJoin.getJoinType() == JoinRelType.FULL, - Pair.right(joinSpecs), - Pair.left(joinSpecs), - projFieldsList, - newJoinFieldRefCountsMap, - RexUtil.composeConjunction(rexBuilder, newPostJoinFilters, true)); - - call.transformTo(multiJoin); - } - - /** - * Combines the inputs into a LogicalJoin into an array of inputs. - * - * @param join original join - * @param left left input into join - * @param right right input into join - * @param projFieldsList returns a list of the new combined projection - * fields - * @param joinFieldRefCountsList returns a list of the new combined join - * field reference counts - * @return combined left and right inputs in an array - */ - private List combineInputs( - Join join, - RelNode left, - RelNode right, - List projFieldsList, - List joinFieldRefCountsList) { - final List newInputs = new ArrayList<>(); - - // leave the null generating sides of an outer join intact; don't - // pull up those children inputs into the array we're constructing - if (canCombine(left, join.getJoinType().generatesNullsOnLeft())) { - final MultiJoin leftMultiJoin = (MultiJoin) left; - for (int i = 0; i < left.getInputs().size(); i++) { - newInputs.add(leftMultiJoin.getInput(i)); - projFieldsList.add(leftMultiJoin.getProjFields().get(i)); - joinFieldRefCountsList.add( - leftMultiJoin.getJoinFieldRefCountsMap().get(i).toIntArray()); - } - } else { - newInputs.add(left); - projFieldsList.add(null); - joinFieldRefCountsList.add( - new int[left.getRowType().getFieldCount()]); - } - - if (canCombine(right, join.getJoinType().generatesNullsOnRight())) { - final MultiJoin rightMultiJoin = (MultiJoin) right; - for (int i = 0; i < right.getInputs().size(); i++) { - newInputs.add(rightMultiJoin.getInput(i)); - projFieldsList.add( - rightMultiJoin.getProjFields().get(i)); - joinFieldRefCountsList.add( - rightMultiJoin.getJoinFieldRefCountsMap().get(i).toIntArray()); - } - } else { - newInputs.add(right); - projFieldsList.add(null); - joinFieldRefCountsList.add( - new int[right.getRowType().getFieldCount()]); - } - - return newInputs; - } - - /** - * Combines the outer join conditions and join types from the left and right - * join inputs. If the join itself is either a left or right outer join, - * then the join condition corresponding to the join is also set in the - * position corresponding to the null-generating input into the join. The - * join type is also set. - * - * @param joinRel join rel - * @param combinedInputs the combined inputs to the join - * @param left left child of the joinrel - * @param right right child of the joinrel - * @param joinSpecs the list where the join types and conditions will be - * copied - */ - private void combineOuterJoins( - Join joinRel, - List combinedInputs, - RelNode left, - RelNode right, - List> joinSpecs) { - JoinRelType joinType = joinRel.getJoinType(); - boolean leftCombined = - canCombine(left, joinType.generatesNullsOnLeft()); - boolean rightCombined = - canCombine(right, joinType.generatesNullsOnRight()); - switch (joinType) { - case LEFT: - if (leftCombined) { - copyOuterJoinInfo( - (MultiJoin) left, - joinSpecs, - 0, - null, - null); - } else { - joinSpecs.add(Pair.of(JoinRelType.INNER, (RexNode) null)); - } - joinSpecs.add(Pair.of(joinType, joinRel.getCondition())); - break; - case RIGHT: - joinSpecs.add(Pair.of(joinType, joinRel.getCondition())); - if (rightCombined) { - copyOuterJoinInfo( - (MultiJoin) right, - joinSpecs, - left.getRowType().getFieldCount(), - right.getRowType().getFieldList(), - joinRel.getRowType().getFieldList()); - } else { - joinSpecs.add(Pair.of(JoinRelType.INNER, (RexNode) null)); - } - break; - default: - if (leftCombined) { - copyOuterJoinInfo( - (MultiJoin) left, - joinSpecs, - 0, - null, - null); - } else { - joinSpecs.add(Pair.of(JoinRelType.INNER, (RexNode) null)); - } - if (rightCombined) { - copyOuterJoinInfo( - (MultiJoin) right, - joinSpecs, - left.getRowType().getFieldCount(), - right.getRowType().getFieldList(), - joinRel.getRowType().getFieldList()); - } else { - joinSpecs.add(Pair.of(JoinRelType.INNER, (RexNode) null)); - } - } - } - - /** - * Copies outer join data from a source MultiJoin to a new set of arrays. - * Also adjusts the conditions to reflect the new position of an input if - * that input ends up being shifted to the right. - * - * @param multiJoin the source MultiJoin - * @param destJoinSpecs the list where the join types and conditions will - * be copied - * @param adjustmentAmount if > 0, the amount the RexInputRefs in the join - * conditions need to be adjusted by - * @param srcFields the source fields that the original join conditions - * are referencing - * @param destFields the destination fields that the new join conditions - */ - private void copyOuterJoinInfo( - MultiJoin multiJoin, - List> destJoinSpecs, - int adjustmentAmount, - List srcFields, - List destFields) { - final List> srcJoinSpecs = - Pair.zip( - multiJoin.getJoinTypes(), - multiJoin.getOuterJoinConditions()); - - if (adjustmentAmount == 0) { - destJoinSpecs.addAll(srcJoinSpecs); - } else { - assert srcFields != null; - assert destFields != null; - int nFields = srcFields.size(); - int[] adjustments = new int[nFields]; - for (int idx = 0; idx < nFields; idx++) { - adjustments[idx] = adjustmentAmount; - } - for (Pair src - : srcJoinSpecs) { - destJoinSpecs.add( - Pair.of( - src.left, - src.right == null - ? null - : src.right.accept( - new RelOptUtil.RexInputConverter( - multiJoin.getCluster().getRexBuilder(), - srcFields, destFields, adjustments)))); - } - } - } - - /** - * Combines the join filters from the left and right inputs (if they are - * MultiJoinRels) with the join filter in the joinrel into a single AND'd - * join filter, unless the inputs correspond to null generating inputs in an - * outer join. - * - * @param joinRel join rel - * @param left left child of the join - * @param right right child of the join - * @return combined join filters AND-ed together - */ - private List combineJoinFilters( - Join joinRel, - RelNode left, - RelNode right) { - JoinRelType joinType = joinRel.getJoinType(); - - // AND the join condition if this isn't a left or right outer join; - // in those cases, the outer join condition is already tracked - // separately - final List filters = new ArrayList<>(); - if ((joinType != JoinRelType.LEFT) && (joinType != JoinRelType.RIGHT)) { - filters.add(joinRel.getCondition()); - } - if (canCombine(left, joinType.generatesNullsOnLeft())) { - filters.add(((MultiJoin) left).getJoinFilter()); - } - // Need to adjust the RexInputs of the right child, since - // those need to shift over to the right - if (canCombine(right, joinType.generatesNullsOnRight())) { - MultiJoin multiJoin = (MultiJoin) right; - filters.add( - shiftRightFilter(joinRel, left, multiJoin, - multiJoin.getJoinFilter())); - } - - return filters; - } - - /** - * Returns whether an input can be merged into a given relational expression - * without changing semantics. - * - * @param input input into a join - * @param nullGenerating true if the input is null generating - * @return true if the input can be combined into a parent MultiJoin - */ - private boolean canCombine(RelNode input, boolean nullGenerating) { - return input instanceof MultiJoin - && !((MultiJoin) input).isFullOuterJoin() - && !(containsOuter((MultiJoin) input)) - && !nullGenerating; - } - - private boolean containsOuter(MultiJoin multiJoin) { - for (JoinRelType joinType : multiJoin.getJoinTypes()) { - if (joinType.isOuterJoin()) { - return true; - } - } - return false; - } - - /** - * Shifts a filter originating from the right child of the LogicalJoin to the - * right, to reflect the filter now being applied on the resulting - * MultiJoin. - * - * @param joinRel the original LogicalJoin - * @param left the left child of the LogicalJoin - * @param right the right child of the LogicalJoin - * @param rightFilter the filter originating from the right child - * @return the adjusted right filter - */ - private RexNode shiftRightFilter( - Join joinRel, - RelNode left, - MultiJoin right, - RexNode rightFilter) { - if (rightFilter == null) { - return null; - } - - int nFieldsOnLeft = left.getRowType().getFieldList().size(); - int nFieldsOnRight = right.getRowType().getFieldList().size(); - int[] adjustments = new int[nFieldsOnRight]; - for (int i = 0; i < nFieldsOnRight; i++) { - adjustments[i] = nFieldsOnLeft; - } - rightFilter = - rightFilter.accept( - new RelOptUtil.RexInputConverter( - joinRel.getCluster().getRexBuilder(), - right.getRowType().getFieldList(), - joinRel.getRowType().getFieldList(), - adjustments)); - return rightFilter; - } - - /** - * Adds on to the existing join condition reference counts the references - * from the new join condition. - * - * @param multiJoinInputs inputs into the new MultiJoin - * @param nTotalFields total number of fields in the MultiJoin - * @param joinCondition the new join condition - * @param origJoinFieldRefCounts existing join condition reference counts - * - * @return Map containing the new join condition - */ - private com.google.common.collect.ImmutableMap addOnJoinFieldRefCounts( - List multiJoinInputs, - int nTotalFields, - RexNode joinCondition, - List origJoinFieldRefCounts) { - // count the input references in the join condition - int[] joinCondRefCounts = new int[nTotalFields]; - joinCondition.accept(new FlinkJoinToMultiJoinRule.InputReferenceCounter(joinCondRefCounts)); - - // first, make a copy of the ref counters - final Map refCountsMap = new HashMap<>(); - int nInputs = multiJoinInputs.size(); - int currInput = 0; - for (int[] origRefCounts : origJoinFieldRefCounts) { - refCountsMap.put( - currInput, - origRefCounts.clone()); - currInput++; - } - - // add on to the counts for each input into the MultiJoin the - // reference counts computed for the current join condition - currInput = -1; - int startField = 0; - int nFields = 0; - for (int i = 0; i < nTotalFields; i++) { - if (joinCondRefCounts[i] == 0) { - continue; - } - while (i >= (startField + nFields)) { - startField += nFields; - currInput++; - assert currInput < nInputs; - nFields = - multiJoinInputs.get(currInput).getRowType().getFieldCount(); - } - int[] refCounts = refCountsMap.get(currInput); - refCounts[i - startField] += joinCondRefCounts[i]; - } - - final com.google.common.collect.ImmutableMap.Builder builder = - com.google.common.collect.ImmutableMap.builder(); - for (Map.Entry entry : refCountsMap.entrySet()) { - builder.put(entry.getKey(), ImmutableIntList.of(entry.getValue())); - } - return builder.build(); - } - - /** - * Combines the post-join filters from the left and right inputs (if they - * are MultiJoinRels) into a single AND'd filter. - * - * @param joinRel the original LogicalJoin - * @param left left child of the LogicalJoin - * @param right right child of the LogicalJoin - * @return combined post-join filters AND'd together - */ - private List combinePostJoinFilters( - Join joinRel, - RelNode left, - RelNode right) { - final List filters = new ArrayList<>(); - if (right instanceof MultiJoin) { - final MultiJoin multiRight = (MultiJoin) right; - filters.add( - shiftRightFilter(joinRel, left, multiRight, - multiRight.getPostJoinFilter())); - } - - if (left instanceof MultiJoin) { - filters.add(((MultiJoin) left).getPostJoinFilter()); - } - - return filters; - } - - //~ Inner Classes ---------------------------------------------------------- - - /** - * Visitor that keeps a reference count of the inputs used by an expression. - */ - private class InputReferenceCounter extends RexVisitorImpl { - private final int[] refCounts; - - InputReferenceCounter(int[] refCounts) { - super(true); - this.refCounts = refCounts; - } - - public Void visitInputRef(RexInputRef inputRef) { - refCounts[inputRef.getIndex()]++; - return null; - } - } -} - -// End FlinkJoinToMultiJoinRule.java diff --git a/flink-table/flink-table-planner-blink/src/main/resources/META-INF/NOTICE b/flink-table/flink-table-planner-blink/src/main/resources/META-INF/NOTICE index 131460d5ce9dc8903688e1117a6627cd851ea88d..c0f55ce3f7250157fad67950aed21c34935e2121 100644 --- a/flink-table/flink-table-planner-blink/src/main/resources/META-INF/NOTICE +++ b/flink-table/flink-table-planner-blink/src/main/resources/META-INF/NOTICE @@ -12,8 +12,8 @@ This project bundles the following dependencies under the Apache Software Licens - com.fasterxml.jackson.core:jackson-databind:2.9.6 - com.jayway.jsonpath:json-path:2.4.0 - joda-time:joda-time:2.5 -- org.apache.calcite:calcite-core:1.20.0 -- org.apache.calcite:calcite-linq4j:1.20.0 +- org.apache.calcite:calcite-core:1.21.0 +- org.apache.calcite:calcite-linq4j:1.21.0 - org.apache.calcite.avatica:avatica-core:1.15.0 - commons-codec:commons-codec:1.10 diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkLogicalRelFactories.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkLogicalRelFactories.scala index 4410cf2a84962d3a93c3ab2ea4ed9e63eacf5ace..d1b2fad7fb05db5a299301d64efa4043e0c863f8 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkLogicalRelFactories.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkLogicalRelFactories.scala @@ -145,8 +145,12 @@ object FlinkLogicalRelFactories { * Implementation of [[FilterFactory]] that returns a [[FlinkLogicalCalc]]. */ class FilterFactoryImpl extends FilterFactory { - def createFilter(input: RelNode, condition: RexNode): RelNode = { + override def createFilter( + input: RelNode, + condition: RexNode, + variablesSet: util.Set[CorrelationId]): RelNode = { // Create a program containing a filter. + // Ignore the variablesSet for current implementation. val rexBuilder = input.getCluster.getRexBuilder val inputRowType = input.getRowType val programBuilder = new RexProgramBuilder(inputRowType, rexBuilder) diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkPlannerImpl.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkPlannerImpl.scala index 0f8bd03051fb4964c75522a9b20ef106cacff8cb..f85062fd4e3c47d3fb406848c14f6b0a9c64016f 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkPlannerImpl.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkPlannerImpl.scala @@ -85,6 +85,30 @@ class FlinkPlannerImpl( hints.toArray } + /** + * Get the [[FlinkCalciteSqlValidator]] instance from this planner, create a new instance + * if current validator has not been initialized, or returns the validator + * instance directly. + * + *

    The validator instance creation is not thread safe. + * + * @return a new validator instance or current existed one + */ + def getOrCreateSqlValidator(): FlinkCalciteSqlValidator = { + if (validator == null) { + val catalogReader = catalogReaderSupplier.apply(false) + validator = new FlinkCalciteSqlValidator( + operatorTable, + catalogReader, + typeFactory) + validator.setIdentifierExpansion(true) + validator.setDefaultNullCollation(FlinkPlannerImpl.defaultNullCollation) + // Disable implicit type coercion for now. + validator.setEnableTypeCoercion(false) + } + validator + } + def parse(sql: String): SqlNode = { try { ready() @@ -112,14 +136,8 @@ class FlinkPlannerImpl( || sqlNode.getKind == SqlKind.INSERT) { return sqlNode } - validator = new FlinkCalciteSqlValidator( - operatorTable, - catalogReader, - typeFactory) - validator.setIdentifierExpansion(true) - validator.setDefaultNullCollation(FlinkPlannerImpl.defaultNullCollation) try { - validator.validate(sqlNode) + getOrCreateSqlValidator().validate(sqlNode) } catch { case e: RuntimeException => throw new ValidationException(s"SQL validation failed. ${e.getMessage}", e) @@ -131,7 +149,7 @@ class FlinkPlannerImpl( assert(validatedSqlNode != null) val sqlToRelConverter: SqlToRelConverter = new SqlToRelConverter( new ViewExpanderImpl, - validator, + getOrCreateSqlValidator(), catalogReaderSupplier.apply(false), cluster, convertletTable, diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkRelBuilder.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkRelBuilder.scala index a138d215861607f379829e9c8a8b3a7094e86898..c493907ffd4f1e6b8fa5a9b0cc158f0bbbcf8650 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkRelBuilder.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkRelBuilder.scala @@ -77,9 +77,9 @@ class FlinkRelBuilder( Util.first(context.unwrap(classOf[SinkFactory]), FlinkRelFactories.DEFAULT_SINK_FACTORY) } - def getRelOptSchema: RelOptSchema = relOptSchema + override def getRelOptSchema: RelOptSchema = relOptSchema - def getCluster: RelOptCluster = relOptCluster + override def getCluster: RelOptCluster = relOptCluster override def getTypeFactory: FlinkTypeFactory = super.getTypeFactory.asInstanceOf[FlinkTypeFactory] diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeFactory.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeFactory.scala index 9bd22c85d7fd47f14614de5c8b6681a147a2ed1e..1c61c00a741fb441d2db7dd2c5a812655fe52ab6 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeFactory.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeFactory.scala @@ -34,7 +34,7 @@ import org.apache.calcite.rel.RelNode import org.apache.calcite.rel.`type`._ import org.apache.calcite.sql.SqlIntervalQualifier import org.apache.calcite.sql.`type`.SqlTypeName._ -import org.apache.calcite.sql.`type`.{BasicSqlType, MapSqlType, SqlTypeName, SqlTypeUtil} +import org.apache.calcite.sql.`type`.{BasicSqlType, MapSqlType, SqlTypeName} import org.apache.calcite.sql.parser.SqlParserPos import org.apache.calcite.util.ConversionUtil @@ -327,26 +327,6 @@ class FlinkTypeFactory(typeSystem: RelDataTypeSystem) extends JavaTypeFactoryImp override def getDefaultCharset: Charset = { Charset.forName(ConversionUtil.NATIVE_UTF16_CHARSET_NAME) } - - /** - * Calcite's default impl for division is apparently borrowed from T-SQL, - * but the details are a little different, e.g. when Decimal(34,0)/Decimal(10,0) - * To avoid confusion, follow the exact T-SQL behavior. - * Note that for (+-*), Calcite is also different from T-SQL; - * however, Calcite conforms to SQL2003 while T-SQL does not. - * therefore we keep Calcite's behavior on (+-*). - */ - override def createDecimalQuotient(type1: RelDataType, type2: RelDataType): RelDataType = { - if (SqlTypeUtil.isExactNumeric(type1) && SqlTypeUtil.isExactNumeric(type2) && - (SqlTypeUtil.isDecimal(type1) || SqlTypeUtil.isDecimal(type2))) { - val result = FlinkTypeSystem.inferDivisionType( - type1.getPrecision, type1.getScale, - type2.getPrecision, type2.getScale) - createSqlType(SqlTypeName.DECIMAL, result.getPrecision, result.getScale) - } else { - null - } - } } object FlinkTypeFactory { diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeSystem.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeSystem.scala index e8374241375fce02c14beb751e1040f45117fd87..0258f28e919baf94bc7e949f7498844494dab4f0 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeSystem.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/FlinkTypeSystem.scala @@ -19,10 +19,10 @@ package org.apache.flink.table.planner.calcite import org.apache.flink.table.runtime.typeutils.TypeCheckUtils -import org.apache.flink.table.types.logical.{DecimalType, DoubleType, LogicalType} +import org.apache.flink.table.types.logical.{DecimalType, LogicalType} import org.apache.calcite.rel.`type`.{RelDataType, RelDataTypeFactory, RelDataTypeSystemImpl} -import org.apache.calcite.sql.`type`.SqlTypeName +import org.apache.calcite.sql.`type`.{SqlTypeName, SqlTypeUtil} /** * Custom type system for Flink. @@ -77,6 +77,29 @@ class FlinkTypeSystem extends RelDataTypeSystemImpl { typeFactory.asInstanceOf[FlinkTypeFactory].createFieldTypeFromLogicalType( sumType.copy(argType.isNullable)) } + + /** + * Calcite's default impl for division is apparently borrowed from T-SQL, + * but the details are a little different, e.g. when Decimal(34,0)/Decimal(10,0) + * To avoid confusion, follow the exact T-SQL behavior. + * Note that for (+-*), Calcite is also different from T-SQL; + * however, Calcite conforms to SQL2003 while T-SQL does not. + * therefore we keep Calcite's behavior on (+-*). + */ + override def deriveDecimalDivideType( + typeFactory: RelDataTypeFactory, + type1: RelDataType, + type2: RelDataType): RelDataType = { + if (SqlTypeUtil.isExactNumeric(type1) && SqlTypeUtil.isExactNumeric(type2) && + (SqlTypeUtil.isDecimal(type1) || SqlTypeUtil.isDecimal(type2))) { + val result = FlinkTypeSystem.inferDivisionType( + type1.getPrecision, type1.getScale, + type2.getPrecision, type2.getScale) + typeFactory.createSqlType(SqlTypeName.DECIMAL, result.getPrecision, result.getScale) + } else { + null + } + } } object FlinkTypeSystem { diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.scala index 57974f319fb3a74677b83aee2d36b34159dcf7fe..fb582d29dbd4c504a69f2580b65e9b44251d33af 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/calcite/RelTimeIndicatorConverter.scala @@ -80,9 +80,6 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle { val measures = matchRel.getMeasures .mapValues(_.accept(materializer)) - val partitionKeys = matchRel.getPartitionKeys - .map(_.accept(materializer)) - .map(materializerUtils.materialize) val interval = if (matchRel.getInterval != null) { matchRel.getInterval.accept(materializer) } else { @@ -108,7 +105,7 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle { matchRel.getAfter, matchRel.getSubsets.asInstanceOf[java.util.Map[String, java.util.TreeSet[String]]], matchRel.isAllRows, - partitionKeys, + matchRel.getPartitionKeys, matchRel.getOrderKeys, interval) } diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala index 34796eb00ef8526c5cb35bd7e21f8910e789df9a..f73abfcc85feb9898b1685917f2f9f665e46df57 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExprCodeGenerator.scala @@ -558,6 +558,11 @@ class ExprCodeGenerator(ctx: CodeGeneratorContext, nullableInput: Boolean) val right = operands(1) generateEquals(ctx, left, right) + case IS_NOT_DISTINCT_FROM => + val left = operands.head + val right = operands(1) + generateIsNotDistinctFrom(ctx, left, right) + case NOT_EQUALS => val left = operands.head val right = operands(1) diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala index bb5ff90cbbff885612f11e733e5e6ac73a0b0c62..d12701193a4a4d5bb17dc9e4b556b94ef6e3566a 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/ExpressionReducer.scala @@ -24,9 +24,10 @@ import org.apache.flink.metrics.MetricGroup import org.apache.flink.table.api.{TableConfig, TableException} import org.apache.flink.table.dataformat.BinaryStringUtil.safeToString import org.apache.flink.table.dataformat.{BinaryString, Decimal, GenericRow} -import org.apache.flink.table.functions.{FunctionContext, UserDefinedFunction} +import org.apache.flink.table.functions.{FunctionContext, FunctionLanguage, UserDefinedFunction} import org.apache.flink.table.planner.calcite.FlinkTypeFactory import org.apache.flink.table.planner.codegen.FunctionCodeGenerator.generateFunction +import org.apache.flink.table.planner.plan.utils.PythonUtil import org.apache.flink.table.runtime.functions.SqlDateTimeUtils import org.apache.flink.table.types.logical.RowType @@ -39,6 +40,7 @@ import java.io.File import java.util.TimeZone import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer /** * Evaluates constant expressions with code generator. @@ -60,8 +62,16 @@ class ExpressionReducer( constExprs: java.util.List[RexNode], reducedValues: java.util.List[RexNode]): Unit = { + val pythonUDFExprs = new ListBuffer[RexNode]() + val literals = constExprs.asScala.map(e => (e.getType.getSqlTypeName, e)).flatMap { + // Skip expressions that contain python functions because it's quite expensive to + // call Python UDFs during optimization phase. They will be optimized during the runtime. + case (_, e) if PythonUtil.containsFunctionOf(e, FunctionLanguage.PYTHON) => + pythonUDFExprs += e + None + // we don't support object literals yet, we skip those constant expressions case (SqlTypeName.ANY, _) | (SqlTypeName.ROW, _) | @@ -120,59 +130,65 @@ class ExpressionReducer( var reducedIdx = 0 while (i < constExprs.size()) { val unreduced = constExprs.get(i) - unreduced.getType.getSqlTypeName match { - // we insert the original expression for object literals - case SqlTypeName.ANY | - SqlTypeName.ROW | - SqlTypeName.ARRAY | - SqlTypeName.MAP | - SqlTypeName.MULTISET => - reducedValues.add(unreduced) - case SqlTypeName.VARCHAR | SqlTypeName.CHAR => - val escapeVarchar = StringEscapeUtils - .escapeJava(safeToString(reduced.getField(reducedIdx).asInstanceOf[BinaryString])) - reducedValues.add(maySkipNullLiteralReduce(rexBuilder, escapeVarchar, unreduced)) - reducedIdx += 1 - case SqlTypeName.VARBINARY | SqlTypeName.BINARY => - val reducedValue = reduced.getField(reducedIdx) - val value = if (null != reducedValue) { - new ByteString(reduced.getField(reducedIdx).asInstanceOf[Array[Byte]]) - } else { - reducedValue - } - reducedValues.add(maySkipNullLiteralReduce(rexBuilder, value, unreduced)) - reducedIdx += 1 - case SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE => - val value = if (!reduced.isNullAt(reducedIdx)) { - val mills = reduced.getField(reducedIdx).asInstanceOf[Long] - Long.box(SqlDateTimeUtils.timestampWithLocalZoneToTimestamp( - mills, TimeZone.getTimeZone(config.getLocalTimeZone))) - } else { - null - } - reducedValues.add(maySkipNullLiteralReduce(rexBuilder, value, unreduced)) - reducedIdx += 1 - case SqlTypeName.DECIMAL => - val reducedValue = reduced.getField(reducedIdx) - val value = if (reducedValue != null) { - reducedValue.asInstanceOf[Decimal].toBigDecimal - } else { - reducedValue - } - reducedValues.add(maySkipNullLiteralReduce(rexBuilder, value, unreduced)) - reducedIdx += 1 - case _ => - val reducedValue = reduced.getField(reducedIdx) - // RexBuilder handle double literal incorrectly, convert it into BigDecimal manually - val value = if (reducedValue != null && - unreduced.getType.getSqlTypeName == SqlTypeName.DOUBLE) { - new java.math.BigDecimal(reducedValue.asInstanceOf[Number].doubleValue()) - } else { - reducedValue - } - - reducedValues.add(maySkipNullLiteralReduce(rexBuilder, value, unreduced)) - reducedIdx += 1 + // use eq to compare reference + if (pythonUDFExprs.exists(_ eq unreduced)) { + // if contains python function then just insert the original expression. + reducedValues.add(unreduced) + } else { + unreduced.getType.getSqlTypeName match { + // we insert the original expression for object literals + case SqlTypeName.ANY | + SqlTypeName.ROW | + SqlTypeName.ARRAY | + SqlTypeName.MAP | + SqlTypeName.MULTISET => + reducedValues.add(unreduced) + case SqlTypeName.VARCHAR | SqlTypeName.CHAR => + val escapeVarchar = StringEscapeUtils + .escapeJava(safeToString(reduced.getField(reducedIdx).asInstanceOf[BinaryString])) + reducedValues.add(maySkipNullLiteralReduce(rexBuilder, escapeVarchar, unreduced)) + reducedIdx += 1 + case SqlTypeName.VARBINARY | SqlTypeName.BINARY => + val reducedValue = reduced.getField(reducedIdx) + val value = if (null != reducedValue) { + new ByteString(reduced.getField(reducedIdx).asInstanceOf[Array[Byte]]) + } else { + reducedValue + } + reducedValues.add(maySkipNullLiteralReduce(rexBuilder, value, unreduced)) + reducedIdx += 1 + case SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE => + val value = if (!reduced.isNullAt(reducedIdx)) { + val mills = reduced.getField(reducedIdx).asInstanceOf[Long] + Long.box(SqlDateTimeUtils.timestampWithLocalZoneToTimestamp( + mills, TimeZone.getTimeZone(config.getLocalTimeZone))) + } else { + null + } + reducedValues.add(maySkipNullLiteralReduce(rexBuilder, value, unreduced)) + reducedIdx += 1 + case SqlTypeName.DECIMAL => + val reducedValue = reduced.getField(reducedIdx) + val value = if (reducedValue != null) { + reducedValue.asInstanceOf[Decimal].toBigDecimal + } else { + reducedValue + } + reducedValues.add(maySkipNullLiteralReduce(rexBuilder, value, unreduced)) + reducedIdx += 1 + case _ => + val reducedValue = reduced.getField(reducedIdx) + // RexBuilder handle double literal incorrectly, convert it into BigDecimal manually + val value = if (reducedValue != null && + unreduced.getType.getSqlTypeName == SqlTypeName.DOUBLE) { + new java.math.BigDecimal(reducedValue.asInstanceOf[Number].doubleValue()) + } else { + reducedValue + } + + reducedValues.add(maySkipNullLiteralReduce(rexBuilder, value, unreduced)) + reducedIdx += 1 + } } i += 1 } diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/MatchCodeGenerator.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/MatchCodeGenerator.scala index 0b98f00a5792aaa592f4cab00a2d3bd3cf430420..4e492b4b973e50562afdefdd91d8f4ce5f09949b 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/MatchCodeGenerator.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/MatchCodeGenerator.scala @@ -47,6 +47,7 @@ import org.apache.calcite.rel.core.AggregateCall import org.apache.calcite.rex._ import org.apache.calcite.sql.SqlAggFunction import org.apache.calcite.tools.RelBuilder +import org.apache.calcite.util.ImmutableBitSet import java.lang.{Long => JLong} import java.util @@ -200,7 +201,7 @@ class MatchCodeGenerator( */ def generateOneRowPerMatchExpression( returnType: RowType, - partitionKeys: util.List[RexNode], + partitionKeys: ImmutableBitSet, measures: util.Map[String, RexNode]) : PatternProcessFunctionRunner = { val resultExpression = generateOneRowPerMatchExpression( @@ -296,7 +297,7 @@ class MatchCodeGenerator( } private def generateOneRowPerMatchExpression( - partitionKeys: java.util.List[RexNode], + partitionKeys: ImmutableBitSet, measures: java.util.Map[String, RexNode], returnType: RowType): GeneratedExpression = { @@ -304,11 +305,12 @@ class MatchCodeGenerator( // 1) the partition columns; // 2) the columns defined in the measures clause. val resultExprs = - partitionKeys.asScala.map { case inputRef: RexInputRef => - generatePartitionKeyAccess(inputRef) - } ++ returnType.getFieldNames.filter(measures.containsKey(_)).map { fieldName => - generateExpression(measures.get(fieldName)) - } + partitionKeys.toArray.map(generatePartitionKeyAccess) ++ + returnType.getFieldNames + .filter(measures.containsKey(_)) + .map { fieldName => + generateExpression(measures.get(fieldName)) + } val resultCodeGenerator = new ExprCodeGenerator(ctx, nullableInput) .bindInput(input1Type, inputTerm = input1Term) @@ -398,16 +400,16 @@ class MatchCodeGenerator( /** * Extracts partition keys from any element of the match * - * @param partitionKey partition key to be extracted + * @param partitionKeyIdx partition key index * @return generated code for the given key */ - private def generatePartitionKeyAccess(partitionKey: RexInputRef): GeneratedExpression = { + private def generatePartitionKeyAccess(partitionKeyIdx: Int): GeneratedExpression = { val keyRow = generateKeyRow() GenerateUtils.generateFieldAccess( ctx, keyRow.resultType, keyRow.resultTerm, - partitionKey.getIndex + partitionKeyIdx ) } diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/ScalarOperatorGens.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/ScalarOperatorGens.scala index 5cc541287bf13b25e646734263a821d418fc1487..06d89043018b6239500caccdb378ad5bf24e6cf4 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/ScalarOperatorGens.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/codegen/calls/ScalarOperatorGens.scala @@ -390,6 +390,16 @@ object ScalarOperatorGens { } } + def generateIsNotDistinctFrom( + ctx: CodeGeneratorContext, + left: GeneratedExpression, + right: GeneratedExpression) + : GeneratedExpression = { + generateOr(ctx, + generateAnd(ctx, generateIsNull(ctx, left), generateIsNull(ctx, right)), + generateEquals(ctx, left, right)) + } + def generateNotEquals( ctx: CodeGeneratorContext, left: GeneratedExpression, diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/functions/utils/AggSqlFunction.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/functions/utils/AggSqlFunction.scala index c6ca9a38f6d38626a89954444da4d4c9bd25463b..7effbba7e5015f4999b010fda7f878cfd4601874 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/functions/utils/AggSqlFunction.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/functions/utils/AggSqlFunction.scala @@ -61,8 +61,8 @@ class AggSqlFunction( new SqlIdentifier(name, SqlParserPos.ZERO), returnTypeInfer.getOrElse(createReturnTypeInference( fromDataTypeToLogicalType(externalResultType), typeFactory)), - createOperandTypeInference(name, aggregateFunction, typeFactory), - createOperandTypeChecker(name, aggregateFunction), + createOperandTypeInference(name, aggregateFunction, typeFactory, externalAccType), + createOperandTypeChecker(name, aggregateFunction, externalAccType), // Do not need to provide a calcite aggregateFunction here. Flink aggregateion function // will be generated when translating the calcite relnode to flink runtime execution plan null, @@ -107,7 +107,8 @@ object AggSqlFunction { private[flink] def createOperandTypeInference( name: String, aggregateFunction: UserDefinedAggregateFunction[_, _], - typeFactory: FlinkTypeFactory): SqlOperandTypeInference = { + typeFactory: FlinkTypeFactory, + externalAccType: DataType): SqlOperandTypeInference = { /** * Operand type inference based on [[AggregateFunction]] given information. */ @@ -117,13 +118,14 @@ object AggSqlFunction { returnType: RelDataType, operandTypes: Array[RelDataType]): Unit = { - val operandTypeInfo = getOperandType(callBinding) + val operandLogicalType = getOperandType(callBinding) + val actualSignature = externalAccType.getLogicalType +: operandLogicalType - val foundSignature = getAccumulateMethodSignature(aggregateFunction, operandTypeInfo) + val foundSignature = getAccumulateMethodSignature(aggregateFunction, operandLogicalType) .getOrElse( throw new ValidationException( s"Given parameters of function '$name' do not match any signature. \n" + - s"Actual: ${signatureInternalToString(operandTypeInfo)} \n" + + s"Actual: ${signatureInternalToString(actualSignature)} \n" + s"Expected: ${signaturesToString(aggregateFunction, "accumulate")}")) val inferredTypes = getParameterTypes(aggregateFunction, foundSignature.drop(1)) @@ -156,7 +158,8 @@ object AggSqlFunction { private[flink] def createOperandTypeChecker( name: String, - aggregateFunction: UserDefinedAggregateFunction[_, _]): SqlOperandTypeChecker = { + aggregateFunction: UserDefinedAggregateFunction[_, _], + externalAccType: DataType): SqlOperandTypeChecker = { val methods = checkAndExtractMethods(aggregateFunction, "accumulate") @@ -194,15 +197,17 @@ object AggSqlFunction { override def checkOperandTypes( callBinding: SqlCallBinding, throwOnFailure: Boolean): Boolean = { - val operandTypeInfo = getOperandType(callBinding) - val foundSignature = getAccumulateMethodSignature(aggregateFunction, operandTypeInfo) + val operandLogicalType = getOperandType(callBinding) + val actualSignature = externalAccType.getLogicalType +: operandLogicalType + + val foundSignature = getAccumulateMethodSignature(aggregateFunction, operandLogicalType) if (foundSignature.isEmpty) { if (throwOnFailure) { throw new ValidationException( s"Given parameters of function '$name' do not match any signature. \n" + - s"Actual: ${signatureInternalToString(operandTypeInfo)} \n" + + s"Actual: ${signatureInternalToString(actualSignature)} \n" + s"Expected: ${signaturesToString(aggregateFunction, "accumulate")}") } else { false diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/functions/utils/ScalarSqlFunction.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/functions/utils/ScalarSqlFunction.scala index 159d4f1dc0dcfe522bf09e90f94383266f528ae4..8e5609cacabcb2474bc1f313e9e723a3e1b557c0 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/functions/utils/ScalarSqlFunction.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/functions/utils/ScalarSqlFunction.scala @@ -47,7 +47,7 @@ import scala.collection.JavaConverters._ class ScalarSqlFunction( name: String, displayName: String, - scalarFunction: ScalarFunction, + val scalarFunction: ScalarFunction, typeFactory: FlinkTypeFactory, returnTypeInfer: Option[SqlReturnTypeInference] = None) extends SqlFunction( diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/logical/MatchRecognize.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/logical/MatchRecognize.scala index e8aa1919949296aea839e01cb4c268ba460a8b96..430b21d7caaf204f3492ffe54406543e43dedfcf 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/logical/MatchRecognize.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/logical/MatchRecognize.scala @@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableMap import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rel.{RelCollation, RelNode} import org.apache.calcite.rex.RexNode +import org.apache.calcite.util.ImmutableBitSet import java.util @@ -37,6 +38,6 @@ case class MatchRecognize( after: RexNode, subsets: ImmutableMap[String, util.SortedSet[String]], allRows: Boolean, - partitionKeys: util.List[RexNode], + partitionKeys: ImmutableBitSet, orderKeys: RelCollation, interval: RexNode) diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalMatch.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalMatch.scala index 59fed4bcc6b4e2292e4119519f0395586afc8246..796114dbd5bc81db34fd437fc41613dce1f54440 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalMatch.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/logical/FlinkLogicalMatch.scala @@ -27,6 +27,7 @@ import org.apache.calcite.rel.core.Match import org.apache.calcite.rel.logical.LogicalMatch import org.apache.calcite.rel.{RelCollation, RelNode} import org.apache.calcite.rex.RexNode +import org.apache.calcite.util.ImmutableBitSet import java.util @@ -43,7 +44,7 @@ class FlinkLogicalMatch( after: RexNode, subsets: util.Map[String, _ <: util.SortedSet[String]], allRows: Boolean, - partitionKeys: util.List[RexNode], + partitionKeys: ImmutableBitSet, orderKeys: RelCollation, interval: RexNode) extends Match( @@ -64,36 +65,23 @@ class FlinkLogicalMatch( interval) with FlinkLogicalRel { - override def copy( - input: RelNode, - rowType: RelDataType, - pattern: RexNode, - strictStart: Boolean, - strictEnd: Boolean, - patternDefinitions: util.Map[String, RexNode], - measures: util.Map[String, RexNode], - after: RexNode, - subsets: util.Map[String, _ <: util.SortedSet[String]], - allRows: Boolean, - partitionKeys: util.List[RexNode], - orderKeys: RelCollation, - interval: RexNode): Match = { + override def copy(traitSet: RelTraitSet, inputs: util.List[RelNode]): RelNode = { new FlinkLogicalMatch( cluster, traitSet, - input, + inputs.get(0), rowType, pattern, strictStart, strictEnd, - patternDefinitions, - measures, - after, - subsets, + getPatternDefinitions, + getMeasures, + getAfter, + getSubsets, allRows, - partitionKeys, - orderKeys, - interval) + getPartitionKeys, + getOrderKeys, + getInterval) } } diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamExecMatch.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamExecMatch.scala index 8415da3559c4f95eb6b1c5bd20cb486742ad6ca4..56b0e30ee3afd3d84c8d6f285b8501fb9248c510 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamExecMatch.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/nodes/physical/stream/StreamExecMatch.scala @@ -105,7 +105,7 @@ class StreamExecMatch( val fieldNames = inputRowType.getFieldNames.toList super.explainTerms(pw) .itemIf("partitionBy", - fieldToString(getPartitionKeyIndexes, inputRowType), + fieldToString(logicalMatch.partitionKeys.toArray, inputRowType), !logicalMatch.partitionKeys.isEmpty) .itemIf("orderBy", collationToString(logicalMatch.orderKeys, inputRowType), @@ -307,16 +307,12 @@ class StreamExecMatch( (timestampedInputTransform, eventComparator) } - private def getPartitionKeyIndexes: Array[Int] = { - logicalMatch.partitionKeys.map { - case inputRef: RexInputRef => inputRef.getIndex - }.toArray - } - private def setKeySelector( transform: OneInputTransformation[BaseRow, _], inputTypeInfo: BaseRowTypeInfo): Unit = { - val selector = KeySelectorUtil.getBaseRowSelector(getPartitionKeyIndexes, inputTypeInfo) + val selector = KeySelectorUtil.getBaseRowSelector( + logicalMatch.partitionKeys.toArray, + inputTypeInfo) transform.setStateKeySelector(selector) transform.setStateKeyType(selector.getProducedType) } diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkBatchProgram.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkBatchProgram.scala index 3dfe2ec426e045d0155aeb5ac3963e91f3a628fd..326b026e05ae7084e1962532d94e831e7abbd149 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkBatchProgram.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkBatchProgram.scala @@ -144,7 +144,7 @@ object FlinkBatchProgram { .addProgram(FlinkHepRuleSetProgramBuilder.newBuilder .setHepRulesExecutionType(HEP_RULES_EXECUTION_TYPE.RULE_COLLECTION) .setHepMatchOrder(HepMatchOrder.BOTTOM_UP) - .add(FlinkBatchRuleSets.JOIN_REORDER_PERPARE_RULES) + .add(FlinkBatchRuleSets.JOIN_REORDER_PREPARE_RULES) .build(), "merge join into MultiJoin") .addProgram(FlinkHepRuleSetProgramBuilder.newBuilder .setHepRulesExecutionType(HEP_RULES_EXECUTION_TYPE.RULE_SEQUENCE) diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkStreamProgram.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkStreamProgram.scala index 422ec7a6c0d6f0c04ab222691fdc24925f149a96..38dfa62ceabcd7dc522a2b73ed3701555d1c7c45 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkStreamProgram.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/optimize/program/FlinkStreamProgram.scala @@ -140,7 +140,7 @@ object FlinkStreamProgram { .addProgram(FlinkHepRuleSetProgramBuilder.newBuilder .setHepRulesExecutionType(HEP_RULES_EXECUTION_TYPE.RULE_COLLECTION) .setHepMatchOrder(HepMatchOrder.BOTTOM_UP) - .add(FlinkStreamRuleSets.JOIN_REORDER_PERPARE_RULES) + .add(FlinkStreamRuleSets.JOIN_REORDER_PREPARE_RULES) .build(), "merge join into MultiJoin") .addProgram(FlinkHepRuleSetProgramBuilder.newBuilder .setHepRulesExecutionType(HEP_RULES_EXECUTION_TYPE.RULE_SEQUENCE) diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala index 076242edc3ec805deafd8f1be2cb8013c9957f32..10aaef49ce0c844676c901ce8faeed8ab12e5d05 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkBatchRuleSets.scala @@ -130,9 +130,9 @@ object FlinkBatchRuleSets { */ private val FILTER_RULES: RuleSet = RuleSets.ofList( // push a filter into a join - FlinkFilterJoinRule.FILTER_ON_JOIN, + FilterJoinRule.FILTER_ON_JOIN, // push filter into the children of a join - FlinkFilterJoinRule.JOIN, + FilterJoinRule.JOIN, // push filter through an aggregation FilterAggregateTransposeRule.INSTANCE, // push a filter past a project @@ -221,9 +221,9 @@ object FlinkBatchRuleSets { FILTER_RULES.asScala ).asJava) - val JOIN_REORDER_PERPARE_RULES: RuleSet = RuleSets.ofList( + val JOIN_REORDER_PREPARE_RULES: RuleSet = RuleSets.ofList( // merge join to MultiJoin - FlinkJoinToMultiJoinRule.INSTANCE, + JoinToMultiJoinRule.INSTANCE, // merge project to MultiJoin ProjectMultiJoinMergeRule.INSTANCE, // merge filter to MultiJoin diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala index 1f9b613f4d47a99290080f4717c4eaab7984728a..626800394934dc9d8024de96f03bad9224ebb95e 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/FlinkStreamRuleSets.scala @@ -134,9 +134,9 @@ object FlinkStreamRuleSets { */ private val FILTER_RULES: RuleSet = RuleSets.ofList( // push a filter into a join - FlinkFilterJoinRule.FILTER_ON_JOIN, + FilterJoinRule.FILTER_ON_JOIN, // push filter into the children of a join - FlinkFilterJoinRule.JOIN, + FilterJoinRule.JOIN, // push filter through an aggregation FilterAggregateTransposeRule.INSTANCE, // push a filter past a project @@ -204,13 +204,13 @@ object FlinkStreamRuleSets { ProjectSetOpTransposeRule.INSTANCE ) - val JOIN_REORDER_PERPARE_RULES: RuleSet = RuleSets.ofList( + val JOIN_REORDER_PREPARE_RULES: RuleSet = RuleSets.ofList( // merge project to MultiJoin ProjectMultiJoinMergeRule.INSTANCE, // merge filter to MultiJoin FilterMultiJoinMergeRule.INSTANCE, // merge join to MultiJoin - FlinkJoinToMultiJoinRule.INSTANCE + JoinToMultiJoinRule.INSTANCE ) val JOIN_REORDER_RULES: RuleSet = RuleSets.ofList( diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushFilterIntoTableSourceScanRule.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushFilterIntoTableSourceScanRule.scala index a4363e2650dcda2bc5fea2d3fa1c10d2278fe98b..a70c3ca8e7b7530412eb6aa4dbfe177d9b50c06c 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushFilterIntoTableSourceScanRule.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushFilterIntoTableSourceScanRule.scala @@ -19,6 +19,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.table.api.config.OptimizerConfigOptions +import org.apache.flink.table.api.TableException import org.apache.flink.table.expressions.Expression import org.apache.flink.table.planner.calcite.FlinkContext import org.apache.flink.table.planner.expressions.converter.ExpressionConverter @@ -103,7 +104,17 @@ class PushFilterIntoTableSourceScanRule extends RelOptRule( val remainingPredicates = new util.LinkedList[Expression]() predicates.foreach(e => remainingPredicates.add(e)) - val newRelOptTable = applyPredicate(remainingPredicates, relOptTable, relBuilder.getTypeFactory) + val newRelOptTable: FlinkRelOptTable = + applyPredicate(remainingPredicates, relOptTable, relBuilder.getTypeFactory) + val newTableSource = newRelOptTable.unwrap(classOf[TableSourceTable[_]]).tableSource + val oldTableSource = relOptTable.unwrap(classOf[TableSourceTable[_]]).tableSource + + if (newTableSource.asInstanceOf[FilterableTableSource[_]].isFilterPushedDown + && newTableSource.explainSource().equals(oldTableSource.explainSource)) { + throw new TableException("Failed to push filter into table source! " + + "table source with pushdown capability must override and change " + + "explainSource() API to explain the pushdown applied!") + } val newScan = new LogicalTableScan(scan.getCluster, scan.getTraitSet, newRelOptTable) diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushPartitionIntoTableSourceScanRule.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushPartitionIntoTableSourceScanRule.scala index 1a9f8cd0e81b1181f6503628e9fa4b00d4c2c413..c752647bbf326e767f529210f841d1bab9610de7 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushPartitionIntoTableSourceScanRule.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushPartitionIntoTableSourceScanRule.scala @@ -23,12 +23,12 @@ import org.apache.flink.table.planner.plan.schema.{FlinkRelOptTable, TableSource import org.apache.flink.table.planner.plan.stats.FlinkStatistic import org.apache.flink.table.planner.plan.utils.{FlinkRelOptUtil, PartitionPruner, RexNodeExtractor} import org.apache.flink.table.sources.PartitionableTableSource - import org.apache.calcite.plan.RelOptRule.{none, operand} import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall} import org.apache.calcite.rel.core.Filter import org.apache.calcite.rel.logical.LogicalTableScan import org.apache.calcite.rex.{RexInputRef, RexNode, RexShuttle} +import org.apache.flink.table.api.TableException import scala.collection.JavaConversions._ @@ -114,6 +114,12 @@ class PushPartitionIntoTableSourceScanRule extends RelOptRule( val newTableSource = tableSource.applyPartitionPruning(remainingPartitions) + if (newTableSource.explainSource().equals(tableSourceTable.tableSource.explainSource())) { + throw new TableException("Failed to push partition into table source! " + + "table source with pushdown capability must override and change " + + "explainSource() API to explain the pushdown applied!") + } + val statistic = tableSourceTable.statistic val newStatistic = if (remainingPartitions.size() == allPartitions.size()) { // Keep all Statistics if no predicates can be pushed down diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRule.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRule.scala index 6aad23c40eec27171ebdd70adeeb40873c71cf98..e4973886ef5061037eca4d3b2663bfad9af5c441 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRule.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/logical/PushProjectIntoTableSourceScanRule.scala @@ -67,13 +67,23 @@ class PushProjectIntoTableSourceScanRule extends RelOptRule( val relOptTable = scan.getTable.asInstanceOf[FlinkRelOptTable] val tableSourceTable = relOptTable.unwrap(classOf[TableSourceTable[_]]) val oldTableSource = tableSourceTable.tableSource - val newTableSource = oldTableSource match { + val (newTableSource, isProjectSuccess) = oldTableSource match { case nested: NestedFieldsProjectableTableSource[_] => val nestedFields = RexNodeExtractor.extractRefNestedInputFields( project.getProjects, usedFields) - nested.projectNestedFields(usedFields, nestedFields) + (nested.projectNestedFields(usedFields, nestedFields), true) case projecting: ProjectableTableSource[_] => - projecting.projectFields(usedFields) + (projecting.projectFields(usedFields), true) + case nonProjecting: TableSource[_] => + // projection cannot be pushed to TableSource + (nonProjecting, false) + } + + if (isProjectSuccess + && newTableSource.explainSource().equals(oldTableSource.explainSource())) { + throw new TableException("Failed to push project into table source! " + + "table source with pushdown capability must override and change " + + "explainSource() API to explain the pushdown applied!") } // check that table schema of the new table source is identical to original diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamExecMatchRule.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamExecMatchRule.scala index d51f65ce8c82e1a67c12cb3632a0b10aa77216c9..f3bf625b823676b966809ee25baf792cea68a3da 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamExecMatchRule.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/rules/physical/stream/StreamExecMatchRule.scala @@ -29,10 +29,9 @@ import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet} import org.apache.calcite.rel.RelNode import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rel.convert.ConverterRule -import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode} +import org.apache.calcite.rex.{RexCall, RexNode} import org.apache.calcite.sql.SqlAggFunction - -import java.util.{List => JList} +import org.apache.calcite.util.ImmutableBitSet import scala.collection.JavaConverters._ import scala.collection.mutable @@ -99,43 +98,33 @@ class StreamExecMatchRule if (logicalMatch.isAllRows) { throw new TableException("All rows per match mode is not supported yet.") } else { - val refNameFinder = new RefNameFinder(logicalMatch.getInput.getRowType) validateAmbiguousColumnsOnRowPerMatch( logicalMatch.getPartitionKeys, logicalMatch.getMeasures.keySet().asScala, - logicalMatch.getRowType, - refNameFinder) + logicalMatch.getInput.getRowType, + logicalMatch.getRowType) } } private def validateAmbiguousColumnsOnRowPerMatch( - partitionKeys: JList[RexNode], + partitionKeys: ImmutableBitSet, measuresNames: mutable.Set[String], - expectedSchema: RelDataType, - refNameFinder: RefNameFinder) + inputSchema: RelDataType, + expectedSchema: RelDataType) : Unit = { - val actualSize = partitionKeys.size() + measuresNames.size + val actualSize = partitionKeys.toArray.length + measuresNames.size val expectedSize = expectedSchema.getFieldCount if (actualSize != expectedSize) { //try to find ambiguous column - val ambiguousColumns = partitionKeys.asScala.map(_.accept(refNameFinder)) + val ambiguousColumns = partitionKeys.toArray + .map(inputSchema.getFieldList.get(_).getName) .filter(measuresNames.contains).mkString("{", ", ", "}") throw new ValidationException(s"Columns ambiguously defined: $ambiguousColumns") } } - private class RefNameFinder(inputSchema: RelDataType) extends RexDefaultVisitor[String] { - - override def visitInputRef(inputRef: RexInputRef): String = { - inputSchema.getFieldList.get(inputRef.getIndex).getName - } - - override def visitNode(rexNode: RexNode): String = - throw new TableException(s"PARTITION BY clause accepts only input reference. Found $rexNode") - } - private class AggregationsValidator extends RexDefaultVisitor[Object] { override def visitCall(call: RexCall): AnyRef = { diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelOptUtil.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelOptUtil.scala index acf20ecf1daea70780f745f1748cca1b7dd868cf..f3632ea6f10009e379f3d169808a6875cfd54e6b 100644 --- a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelOptUtil.scala +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/FlinkRelOptUtil.scala @@ -22,20 +22,15 @@ import org.apache.flink.table.planner.calcite.{FlinkContext, FlinkPlannerImpl, F import org.apache.flink.table.planner.plan.`trait`.{MiniBatchInterval, MiniBatchMode} import org.apache.flink.table.planner.{JBoolean, JByte, JDouble, JFloat, JLong, JShort} -import com.google.common.collect.Lists import org.apache.calcite.config.NullCollation import org.apache.calcite.plan.RelOptUtil import org.apache.calcite.rel.RelFieldCollation.{Direction, NullDirection} -import org.apache.calcite.rel.`type`.RelDataTypeField -import org.apache.calcite.rel.core.{Join, JoinRelType} import org.apache.calcite.rel.{RelFieldCollation, RelNode} import org.apache.calcite.rex.{RexBuilder, RexCall, RexInputRef, RexLiteral, RexNode, RexUtil, RexVisitorImpl} import org.apache.calcite.sql.SqlExplainLevel import org.apache.calcite.sql.SqlKind._ import org.apache.calcite.sql.`type`.SqlTypeName._ -import org.apache.calcite.tools.RelBuilder -import org.apache.calcite.util.mapping.Mappings -import org.apache.calcite.util.{ImmutableBitSet, Pair, Util} +import org.apache.calcite.util.ImmutableBitSet import org.apache.commons.math3.util.ArithmeticUtils import java.io.{PrintWriter, StringWriter} @@ -186,169 +181,6 @@ object FlinkRelOptUtil { } } - /** - * Pushes down expressions in "equal" join condition. - * - * NOTES: This method should be deleted when CALCITE-3171 is fixed. - * - *

    For example, given - * "emp JOIN dept ON emp.deptno + 1 = dept.deptno", adds a project above - * "emp" that computes the expression - * "emp.deptno + 1". The resulting join condition is a simple combination - * of AND, equals, and input fields, plus the remaining non-equal conditions. - * - * @param originalJoin Join whose condition is to be pushed down - * @param relBuilder Factory to create project operator - */ - def pushDownJoinConditions(originalJoin: Join, relBuilder: RelBuilder): RelNode = { - var joinCond: RexNode = originalJoin.getCondition - val joinType: JoinRelType = originalJoin.getJoinType - - val extraLeftExprs: util.List[RexNode] = new util.ArrayList[RexNode] - val extraRightExprs: util.List[RexNode] = new util.ArrayList[RexNode] - val leftCount: Int = originalJoin.getLeft.getRowType.getFieldCount - val rightCount: Int = originalJoin.getRight.getRowType.getFieldCount - - // You cannot push a 'get' because field names might change. - // - // Pushing sub-queries is OK in principle (if they don't reference both - // sides of the join via correlating variables) but we'd rather not do it - // yet. - if (!containsGet(joinCond) && RexUtil.SubQueryFinder.find(joinCond) == null) { - joinCond = pushDownEqualJoinConditions( - joinCond, leftCount, rightCount, extraLeftExprs, extraRightExprs) - } - relBuilder.push(originalJoin.getLeft) - if (!extraLeftExprs.isEmpty) { - val fields: util.List[RelDataTypeField] = relBuilder.peek.getRowType.getFieldList - val pairs: util.List[Pair[RexNode, String]] = new util.AbstractList[Pair[RexNode, String]]() { - override def size: Int = leftCount + extraLeftExprs.size - - override def get(index: Int): Pair[RexNode, String] = if (index < leftCount) { - val field: RelDataTypeField = fields.get(index) - Pair.of(new RexInputRef(index, field.getType), field.getName) - } - else Pair.of(extraLeftExprs.get(index - leftCount), null) - } - relBuilder.project(Pair.left(pairs), Pair.right(pairs)) - } - - relBuilder.push(originalJoin.getRight) - if (!extraRightExprs.isEmpty) { - val fields: util.List[RelDataTypeField] = relBuilder.peek.getRowType.getFieldList - val newLeftCount: Int = leftCount + extraLeftExprs.size - val pairs: util.List[Pair[RexNode, String]] = new util.AbstractList[Pair[RexNode, String]]() { - override def size: Int = rightCount + extraRightExprs.size - - override def get(index: Int): Pair[RexNode, String] = if (index < rightCount) { - val field: RelDataTypeField = fields.get(index) - Pair.of(new RexInputRef(index, field.getType), field.getName) - } - else Pair.of(RexUtil.shift(extraRightExprs.get(index - rightCount), -newLeftCount), null) - } - relBuilder.project(Pair.left(pairs), Pair.right(pairs)) - } - - val right: RelNode = relBuilder.build - val left: RelNode = relBuilder.build - relBuilder.push(originalJoin.copy(originalJoin.getTraitSet, joinCond, left, right, joinType, - originalJoin.isSemiJoinDone)) - - // handle SEMI/ANTI join here - var mapping: Mappings.TargetMapping = null - if (!originalJoin.getJoinType.projectsRight()) { - if (!extraLeftExprs.isEmpty) { - mapping = Mappings.createShiftMapping(leftCount + extraLeftExprs.size, 0, 0, leftCount) - } - } else { - if (!extraLeftExprs.isEmpty || !extraRightExprs.isEmpty) { - mapping = Mappings.createShiftMapping( - leftCount + extraLeftExprs.size + rightCount + extraRightExprs.size, - 0, 0, leftCount, leftCount, leftCount + extraLeftExprs.size, rightCount) - } - } - - if (mapping != null) { - relBuilder.project(relBuilder.fields(mapping.inverse)) - } - relBuilder.build - } - - private def containsGet(node: RexNode) = try { - node.accept(new RexVisitorImpl[Void](true) { - override def visitCall(call: RexCall): Void = { - if (call.getOperator eq RexBuilder.GET_OPERATOR) { - throw Util.FoundOne.NULL - } - super.visitCall(call) - } - }) - false - } catch { - case _: Util.FoundOne => - true - } - - /** - * Pushes down parts of a join condition. - * - *

    For example, given - * "emp JOIN dept ON emp.deptno + 1 = dept.deptno", adds a project above - * "emp" that computes the expression - * "emp.deptno + 1". The resulting join condition is a simple combination - * of AND, equals, and input fields. - */ - private def pushDownEqualJoinConditions( - node: RexNode, - leftCount: Int, - rightCount: Int, - extraLeftExprs: util.List[RexNode], - extraRightExprs: util.List[RexNode]): RexNode = - node.getKind match { - case AND | EQUALS => - val call = node.asInstanceOf[RexCall] - val list = new util.ArrayList[RexNode] - val operands = Lists.newArrayList(call.getOperands) - // do not use `operands.zipWithIndex.foreach` - operands.indices.foreach { i => - val operand = operands.get(i) - val left2 = leftCount + extraLeftExprs.size - val right2 = rightCount + extraRightExprs.size - val e = pushDownEqualJoinConditions( - operand, leftCount, rightCount, extraLeftExprs, extraRightExprs) - val remainingOperands = Util.skip(operands, i + 1) - val left3 = leftCount + extraLeftExprs.size - fix(remainingOperands, left2, left3) - fix(list, left2, left3) - list.add(e) - } - - if (!(list == call.getOperands)) { - call.clone(call.getType, list) - } else { - call - } - case OR | INPUT_REF | LITERAL | NOT => node - case _ => - if (node.accept(new TimeIndicatorExprFinder)) { - node - } else { - val bits = RelOptUtil.InputFinder.bits(node) - val mid = leftCount + extraLeftExprs.size - Side.of(bits, mid) match { - case Side.LEFT => - fix(extraRightExprs, mid, mid + 1) - extraLeftExprs.add(node) - new RexInputRef(mid, node.getType) - case Side.RIGHT => - val index2 = mid + rightCount + extraRightExprs.size - extraRightExprs.add(node) - new RexInputRef(index2, node.getType) - case _ => node - } - } - } - private def fix(operands: util.List[RexNode], before: Int, after: Int): Unit = { if (before == after) { return diff --git a/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/PythonUtil.scala b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/PythonUtil.scala new file mode 100644 index 0000000000000000000000000000000000000000..141d1dc3cb447fd72c6fbecee0cb590d112bd8ef --- /dev/null +++ b/flink-table/flink-table-planner-blink/src/main/scala/org/apache/flink/table/planner/plan/utils/PythonUtil.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.planner.plan.utils + +import org.apache.calcite.rex.{RexCall, RexNode} +import org.apache.flink.table.functions.FunctionLanguage +import org.apache.flink.table.planner.functions.utils.ScalarSqlFunction + +import scala.collection.JavaConversions._ + +object PythonUtil { + /** + * Checks whether it contains the specified kind of function in the specified node. + * + * @param node the RexNode to check + * @param language the expected kind of function to find + * @param recursive whether check the inputs of the specified node + * @return true if it contains the specified kind of function in the specified node. + */ + def containsFunctionOf( + node: RexNode, + language: FunctionLanguage, + recursive: Boolean = true): Boolean = { + node.accept(new FunctionFinder(language, recursive)) + } + + /** + * Checks whether it contains the specified kind of function in a RexNode. + * + * @param expectedLanguage the expected kind of function to find + * @param recursive whether check the inputs + */ + private class FunctionFinder(expectedLanguage: FunctionLanguage, recursive: Boolean) + extends RexDefaultVisitor[Boolean] { + + override def visitCall(call: RexCall): Boolean = { + call.getOperator match { + case sfc: ScalarSqlFunction if sfc.scalarFunction.getLanguage == + FunctionLanguage.PYTHON => + findInternal(FunctionLanguage.PYTHON, call) + case _ => + findInternal(FunctionLanguage.JVM, call) + } + } + + override def visitNode(rexNode: RexNode): Boolean = false + + private def findInternal(actualLanguage: FunctionLanguage, call: RexCall): Boolean = + actualLanguage == expectedLanguage || + (recursive && call.getOperands.exists(_.accept(this))) + } +} diff --git a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/BatchAbstractTestBase.java b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/BatchAbstractTestBase.java index 490b5dc17ddcc3844721b5e1155308dcd53c30cb..209e9da8f03a039dbb836c46863cb92d5b8e1e7c 100644 --- a/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/BatchAbstractTestBase.java +++ b/flink-table/flink-table-planner-blink/src/test/java/org/apache/flink/table/planner/runtime/utils/BatchAbstractTestBase.java @@ -47,7 +47,7 @@ public class BatchAbstractTestBase { private static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "100m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "100m"); return config; } diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/DeadlockBreakupTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/DeadlockBreakupTest.xml index 1b04b861f6c0b31d31de6b4e96ce4ade551da860..3bbf8bc763cce36512e139f012907b5285c70cd9 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/DeadlockBreakupTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/DeadlockBreakupTest.xml @@ -32,7 +32,7 @@ LogicalIntersect(all=[false]) ($3, 10)]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (price, 10)]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[]]]], fields=[name, id, amount, price]) ]]> @@ -58,13 +58,13 @@ Calc(select=[name, id, amount, price], where=[>(price, 10)]) ($2, 2), >($3, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (amount, 2), >(price, 10))]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[]]]], fields=[name, id, amount, price]) ]]> @@ -76,13 +76,13 @@ Calc(select=[name, id, amount, price], where=[OR(>(amount, 2), >(price, 10))]) ($2, 2), <($2, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (amount, 2), <(amount, 10))]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[]]]], fields=[name, id, amount, price]) ]]> @@ -94,12 +94,12 @@ Calc(select=[name, id, amount, price], where=[OR(>(amount, 2), <(amount, 10))]) ($2, 2), <($2, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> @@ -111,12 +111,12 @@ TableSourceScan(table=[[default_catalog, default_database, FilterableTable, sour ($2, 2)]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> @@ -128,13 +128,13 @@ TableSourceScan(table=[[default_catalog, default_database, FilterableTable, sour ($2, 2), >($3, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (price, 10)]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filter=[greaterThan(amount, 2)]]]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[greaterThan(amount, 2)]]]], fields=[name, id, amount, price]) ]]> @@ -149,13 +149,13 @@ SELECT * FROM FilterableTable WHERE ($2, 2), <($1, 100), >(CAST($2):BIGINT, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (CAST(amount), 10))]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filter=[greaterThan(amount, 2)]]]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[greaterThan(amount, 2)]]]], fields=[name, id, amount, price]) ]]> @@ -167,13 +167,13 @@ Calc(select=[name, id, amount, price], where=[AND(<(id, 100), >(CAST(amount), 10 ($2, 2), <(myUdf($2), 32))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> @@ -270,13 +270,13 @@ SELECT id FROM FilterableTable1 WHERE ($2, 14:25:02), >($1, 2017-02-03), >($3, 2017-02-03 14:25:02))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable1]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable1, source: [filterPushedDown=[false], filter=[]]]]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/AggregateReduceGroupingTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/AggregateReduceGroupingTest.xml index c8dc9301691afc58cec89204d1aa98e200da9d26..f9e8dde7ebf338d51aa1a4aa032edbc35845644d 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/AggregateReduceGroupingTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/AggregateReduceGroupingTest.xml @@ -706,7 +706,7 @@ HashAggregate(isMerge=[false], groupBy=[a1], auxGrouping=[d1], select=[a1, d1, C @@ -724,18 +724,18 @@ Calc(select=[a1, b1, b1 AS b10, c1]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/DistinctAggregateTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/DistinctAggregateTest.xml index 91e43f6051a991ec4b042526699dca62a0a672ec..e93942a7a7bf3e883c4f69ff53cec9318c343e95 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/DistinctAggregateTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/agg/DistinctAggregateTest.xml @@ -48,7 +48,7 @@ SortAggregate(isMerge=[true], select=[Final_COUNT(count$0) AS EXPR$0, Final_SUM( diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml index 5d81924213fb92994de4ea0d31da2ce3a0642471..9af0bbc6444ef858a11d8470aabfb0d63230ca88 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashJoinTest.xml @@ -22,10 +22,9 @@ limitations under the License. @@ -47,21 +46,18 @@ Calc(select=[c, g]) @@ -71,10 +67,9 @@ Calc(select=[d, e, f, g, h, a, b, c]) @@ -191,18 +186,17 @@ Calc(select=[a, d]) @@ -312,10 +305,9 @@ HashJoin(joinType=[RightOuterJoin], where=[AND(=(a, d), <(b, h))], select=[d, e, @@ -343,11 +335,11 @@ ON (src1.k = src2.k AND src2.k > 10) ($2, 10))], joinType=[left]) :- LogicalProject(k=[$0], v=[$1]) : +- LogicalFilter(condition=[=($0, 0)]) : +- LogicalTableScan(table=[[default_catalog, default_database, src, source: [TestTableSource(k, v)]]]) - +- LogicalProject(k=[$0], v=[$1], $f2=[>($0, 10)]) + +- LogicalProject(k=[$0], v=[$1]) +- LogicalFilter(condition=[=($0, 0)]) +- LogicalTableScan(table=[[default_catalog, default_database, src, source: [TestTableSource(k, v)]]]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashSemiAntiJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashSemiAntiJoinTest.xml index f6d5a7de6d822b6ee984c9be7cae6927106a8ad3..cd2c9eb7ea1c5c7f0bf3d11598eb80f178e03c21 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashSemiAntiJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/BroadcastHashSemiAntiJoinTest.xml @@ -1843,7 +1843,7 @@ LogicalProject(d=[$0]) (c0, 0), IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(<>(c1, 0), IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) + :- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) : +- HashJoin(joinType=[LeftOuterJoin], where=[=(a, EXPR$0)], select=[a, b, c, c0, ck, i0, c1, ck0, EXPR$0, i], isBroadcast=[true], build=[right]) : :- NestedLoopJoin(joinType=[InnerJoin], where=[true], select=[a, b, c, c0, ck, i0, c1, ck0], build=[right], singleRowJoin=[true]) : : :- Calc(select=[a, b, c, c0, ck, i0]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml index f93aa46929520aa896ae1a34f9d8e807596f23c5..73dbc0f76f87e8510085ce8f8dfcc5dc9dc32007 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopJoinTest.xml @@ -87,19 +87,18 @@ NestedLoopJoin(joinType=[FullOuterJoin], where=[true], select=[d, e, f, g, h, a, @@ -212,10 +208,9 @@ Calc(select=[CASE(IS NOT NULL(a), a, a0) AS $f0, b, c, b0, c0]) @@ -237,21 +232,18 @@ Calc(select=[c, g]) @@ -261,10 +253,9 @@ Calc(select=[d, e, f, g, h, a, b, c]) @@ -467,18 +458,17 @@ NestedLoopJoin(joinType=[LeftOuterJoin], where=[true], select=[d, e, f, g, h, a, @@ -653,10 +642,9 @@ NestedLoopJoin(joinType=[RightOuterJoin], where=[<>(a, d)], select=[d, e, f, g, @@ -682,11 +670,11 @@ ON (src1.k = src2.k AND src2.k > 10) ($2, 10))], joinType=[left]) :- LogicalProject(k=[$0], v=[$1]) : +- LogicalFilter(condition=[=($0, 0)]) : +- LogicalTableScan(table=[[default_catalog, default_database, src, source: [TestTableSource(k, v)]]]) - +- LogicalProject(k=[$0], v=[$1], $f2=[>($0, 10)]) + +- LogicalProject(k=[$0], v=[$1]) +- LogicalFilter(condition=[=($0, 0)]) +- LogicalTableScan(table=[[default_catalog, default_database, src, source: [TestTableSource(k, v)]]]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopSemiAntiJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopSemiAntiJoinTest.xml index 12a5449d0734752cf253b8bb0708a153f9b4d816..fa78f5d709a98c2619547a9c6677781b2ee99f78 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopSemiAntiJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/NestedLoopSemiAntiJoinTest.xml @@ -2239,7 +2239,7 @@ LogicalProject(d=[$0]) (c0, 0), IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(<>(c1, 0), IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) + :- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) : +- NestedLoopJoin(joinType=[LeftOuterJoin], where=[=(a, EXPR$0)], select=[a, b, c, c0, ck, i0, c1, ck0, EXPR$0, i], build=[right]) : :- NestedLoopJoin(joinType=[InnerJoin], where=[true], select=[a, b, c, c0, ck, i0, c1, ck0], build=[right], singleRowJoin=[true]) : : :- Calc(select=[a, b, c, c0, ck, i0]) @@ -2345,13 +2345,11 @@ LogicalProject(d=[$0]) (b, 10) AS $f3]) - : +- TableSourceScan(table=[[default_catalog, default_database, l, source: [TestTableSource(a, b, c)]]], fields=[a, b, c]) - +- Exchange(distribution=[broadcast]) - +- Calc(select=[d]) - +- TableSourceScan(table=[[default_catalog, default_database, r, source: [TestTableSource(d, e, f)]]], fields=[d, e, f]) +NestedLoopJoin(joinType=[LeftAntiJoin], where=[AND(OR(=(a, d), IS NULL(a), IS NULL(d)), >(b, 10))], select=[a, b, c], build=[right]) +:- TableSourceScan(table=[[default_catalog, default_database, l, source: [TestTableSource(a, b, c)]]], fields=[a, b, c]) ++- Exchange(distribution=[broadcast]) + +- Calc(select=[d]) + +- TableSourceScan(table=[[default_catalog, default_database, r, source: [TestTableSource(d, e, f)]]], fields=[d, e, f]) ]]> @@ -2479,7 +2477,7 @@ LogicalProject(e=[$1], d=[$0]) (c0, 0), IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c, 0), AND(<>(c, 0), IS NULL(i), >=(ck0, c), IS NOT NULL(a))), 2, 3) AS $f3]) + :- Calc(select=[b, CASE(OR(=(c0, 0), AND(IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c, 0), AND(IS NULL(i), >=(ck0, c), IS NOT NULL(a))), 2, 3) AS $f3]) : +- NestedLoopJoin(joinType=[LeftOuterJoin], where=[=(a, j)], select=[a, b, c0, ck, i0, c, ck0, j, i], build=[right]) : :- NestedLoopJoin(joinType=[InnerJoin], where=[true], select=[a, b, c0, ck, i0, c, ck0], build=[right], singleRowJoin=[true]) : : :- Calc(select=[a, b, c AS c0, ck, i0]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SemiAntiJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SemiAntiJoinTest.xml index 09176792b44e7b2aaf7674f1cf5c82d5f986c4ca..a9dfe9d028fa912a6eca3f0037a548fbaa080cf9 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SemiAntiJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SemiAntiJoinTest.xml @@ -2303,7 +2303,7 @@ LogicalProject(d=[$0]) Calc(select=[b]) +- HashJoin(joinType=[LeftAntiJoin], where=[AND(OR(=($f3, d), IS NULL(d)), =(c, f))], select=[b, c, $f3], build=[right]) :- Exchange(distribution=[hash[c]]) - : +- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(<>(c0, 0), IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(<>(c1, 0), IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) + : +- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) : +- HashJoin(joinType=[LeftOuterJoin], where=[=(a, EXPR$0)], select=[a, b, c, c0, ck, i0, c1, ck0, EXPR$0, i], build=[right]) : :- NestedLoopJoin(joinType=[InnerJoin], where=[true], select=[a, b, c, c0, ck, i0, c1, ck0], build=[right], singleRowJoin=[true]) : : :- Calc(select=[a, b, c, c0, ck, i0]) @@ -2409,13 +2409,11 @@ LogicalProject(d=[$0]) (b, 10) AS $f3]) - : +- TableSourceScan(table=[[default_catalog, default_database, l, source: [TestTableSource(a, b, c)]]], fields=[a, b, c]) - +- Exchange(distribution=[broadcast]) - +- Calc(select=[d]) - +- TableSourceScan(table=[[default_catalog, default_database, r, source: [TestTableSource(d, e, f)]]], fields=[d, e, f]) +NestedLoopJoin(joinType=[LeftAntiJoin], where=[AND(OR(=(a, d), IS NULL(a), IS NULL(d)), >(b, 10))], select=[a, b, c], build=[right]) +:- TableSourceScan(table=[[default_catalog, default_database, l, source: [TestTableSource(a, b, c)]]], fields=[a, b, c]) ++- Exchange(distribution=[broadcast]) + +- Calc(select=[d]) + +- TableSourceScan(table=[[default_catalog, default_database, r, source: [TestTableSource(d, e, f)]]], fields=[d, e, f]) ]]> @@ -2545,7 +2543,7 @@ LogicalProject(e=[$1], d=[$0]) (c0, 0), IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c, 0), AND(<>(c, 0), IS NULL(i), >=(ck0, c), IS NOT NULL(a))), 2, 3) AS $f3]) + :- Calc(select=[b, CASE(OR(=(c0, 0), AND(IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c, 0), AND(IS NULL(i), >=(ck0, c), IS NOT NULL(a))), 2, 3) AS $f3]) : +- HashJoin(joinType=[LeftOuterJoin], where=[=(a, j)], select=[a, b, c0, ck, i0, c, ck0, j, i], build=[right]) : :- NestedLoopJoin(joinType=[InnerJoin], where=[true], select=[a, b, c0, ck, i0, c, ck0], build=[right], singleRowJoin=[true]) : : :- Calc(select=[a, b, c AS c0, ck, i0]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml index 7afd0006354bae4188be7b2ade6e8c3d0b699436..8c279897d0a14b9f0475f365e6309459b5453f48 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashJoinTest.xml @@ -22,19 +22,18 @@ limitations under the License. @@ -122,13 +118,13 @@ Calc(select=[CASE(IS NOT NULL(a), a, a0) AS $f0, b, c, b0, c0]) + @@ -151,22 +147,19 @@ Calc(select=[c, g]) @@ -176,10 +169,9 @@ Calc(select=[d, e, f, g, h, a, b, c]) @@ -301,19 +293,18 @@ Calc(select=[a, d]) + @@ -427,10 +418,9 @@ HashJoin(joinType=[RightOuterJoin], where=[AND(=(a, d), <(b, h))], select=[d, e, diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashSemiAntiJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashSemiAntiJoinTest.xml index afcbaaa8cd569dc2e0889ac691156df62aa5585e..5100b1c62b255b90e4b913705edd1bf240ce0e53 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashSemiAntiJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/ShuffledHashSemiAntiJoinTest.xml @@ -1897,7 +1897,7 @@ LogicalProject(d=[$0]) Calc(select=[b]) +- HashJoin(joinType=[LeftAntiJoin], where=[AND(OR(=($f3, d), IS NULL(d)), =(c, f))], select=[b, c, $f3], build=[right]) :- Exchange(distribution=[hash[c]]) - : +- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(<>(c0, 0), IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(<>(c1, 0), IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) + : +- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) : +- HashJoin(joinType=[LeftOuterJoin], where=[=(a, EXPR$0)], select=[a, b, c, c0, ck, i0, c1, ck0, EXPR$0, i], build=[right]) : :- NestedLoopJoin(joinType=[InnerJoin], where=[true], select=[a, b, c, c0, ck, i0, c1, ck0], build=[right], singleRowJoin=[true]) : : :- Calc(select=[a, b, c, c0, ck, i0]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml index a1c52e2f712f1dab18faaf13d20c832cc6266601..f9ac1e6cfe9e30e348d88ef716b05a5347951cc5 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeJoinTest.xml @@ -22,19 +22,18 @@ limitations under the License. @@ -125,10 +121,9 @@ Calc(select=[CASE(IS NOT NULL(a), a, a0) AS $f0, b, c, b0, c0]) @@ -151,22 +146,19 @@ Calc(select=[c, g]) @@ -176,10 +168,9 @@ Calc(select=[d, e, f, g, h, a, b, c]) @@ -301,19 +292,18 @@ Calc(select=[a, d]) @@ -427,10 +416,9 @@ SortMergeJoin(joinType=[RightOuterJoin], where=[AND(=(a, d), <(b, h))], select=[ @@ -459,11 +447,11 @@ ON (src1.k = src2.k AND src2.k > 10) ($2, 10))], joinType=[left]) :- LogicalProject(k=[$0], v=[$1]) : +- LogicalFilter(condition=[=($0, 0)]) : +- LogicalTableScan(table=[[default_catalog, default_database, src, source: [TestTableSource(k, v)]]]) - +- LogicalProject(k=[$0], v=[$1], $f2=[>($0, 10)]) + +- LogicalProject(k=[$0], v=[$1]) +- LogicalFilter(condition=[=($0, 0)]) +- LogicalTableScan(table=[[default_catalog, default_database, src, source: [TestTableSource(k, v)]]]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeSemiAntiJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeSemiAntiJoinTest.xml index 669d4a64312576afef4e0779139d4bdd210c3139..54bb7209d93835fd4d68cbfec97b5984dbbe4294 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeSemiAntiJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/sql/join/SortMergeSemiAntiJoinTest.xml @@ -1982,7 +1982,7 @@ LogicalProject(d=[$0]) Calc(select=[b]) +- SortMergeJoin(joinType=[LeftAntiJoin], where=[AND(OR(=($f3, d), IS NULL(d)), =(c, f))], select=[b, c, $f3]) :- Exchange(distribution=[hash[c]]) - : +- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(<>(c0, 0), IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(<>(c1, 0), IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) + : +- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) : +- SortMergeJoin(joinType=[LeftOuterJoin], where=[=(a, EXPR$0)], select=[a, b, c, c0, ck, i0, c1, ck0, EXPR$0, i]) : :- NestedLoopJoin(joinType=[InnerJoin], where=[true], select=[a, b, c, c0, ck, i0, c1, ck0], build=[right], singleRowJoin=[true]) : : :- Calc(select=[a, b, c, c0, ck, i0]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/table/JoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/table/JoinTest.xml index 0bbbc5269760b8140b719a20a6e7c8a6d80bf7cc..22028912ea3f74bdd755eac9aa52c20af9734704 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/table/JoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/batch/table/JoinTest.xml @@ -51,9 +51,9 @@ LogicalProject(b=[$1], y=[$4]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/AggregateReduceGroupingRuleTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/AggregateReduceGroupingRuleTest.xml index b0170984bf91ab26ed9c4938132379de336ef713..14b9c6ed9b91f2cdba20d72814570629585d1484 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/AggregateReduceGroupingRuleTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/AggregateReduceGroupingRuleTest.xml @@ -646,17 +646,19 @@ FlinkLogicalAggregate(group=[{0}], d1=[AUXILIARY_GROUP($3)], EXPR$2=[COUNT($2) F @@ -666,17 +668,17 @@ FlinkLogicalAggregate(group=[{0}], EXPR$1=[SUM($1) FILTER $3], EXPR$2=[MAX($1) F diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.xml index 9e301578e5f553a700755828e599a43ce2f9e8f4..0433fbee4819760676dc26eb3d042cf6c63b3a0f 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.xml @@ -47,6 +47,23 @@ LogicalProject(EXPR$0=[myUdf(1)]) + + + + + + + + + + + diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateExpandDistinctAggregatesRuleTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateExpandDistinctAggregatesRuleTest.xml index ecf9d76b6815db0bdf381f2784cfa7a371abd2e4..b0744cf80375d3a33175c427751599a2014110e4 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateExpandDistinctAggregatesRuleTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateExpandDistinctAggregatesRuleTest.xml @@ -160,7 +160,7 @@ FlinkLogicalCalc(select=[EXPR$0, EXPR$1]) @@ -181,7 +181,7 @@ FlinkLogicalAggregate(group=[{}], EXPR$0=[COUNT($0)], EXPR$1=[SUM($0)], EXPR$2=[ diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateRemoveRuleTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateRemoveRuleTest.xml index 624bc190f429d7805e8e4d670ec4e20cef254aac..266f14beec26b5ff0e71a1cd1054c2560677a639 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateRemoveRuleTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateRemoveRuleTest.xml @@ -186,7 +186,7 @@ FlinkLogicalCalc(select=[a, EXPR$1, EXPR$2]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.xml index c65896b8243a4c9083991d76af7cb46c8b207c4c..8f253af5ea219bc6d46557e0ee56309a42144d63 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.xml @@ -255,13 +255,11 @@ LogicalProject(c=[$0]) ($1, 2)]) - : +- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) - +- LogicalProject(c=[$0]) - +- LogicalFilter(condition=[true]) - +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ++- LogicalJoin(condition=[AND(OR(=($0, $2), IS NULL($0), IS NULL($2)), >($1, 2))], joinType=[anti]) + :- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) + +- LogicalProject(c=[$0]) + +- LogicalFilter(condition=[true]) + +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ]]> @@ -310,13 +308,11 @@ LogicalProject(c=[$0]) ($1, 1)]) - : +- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) - +- LogicalProject(c=[$0], d=[$1]) - +- LogicalFilter(condition=[true]) - +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ++- LogicalJoin(condition=[AND(OR(=($0, $2), IS NULL($0), IS NULL($2)), =($1, $3), >($1, 1))], joinType=[anti]) + :- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) + +- LogicalProject(c=[$0], d=[$1]) + +- LogicalFilter(condition=[true]) + +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ]]> @@ -337,13 +333,11 @@ LogicalFilter(condition=[AND(=($cor0.a, $0), >($cor0.b, 2))]) ($1, 2)]) - : +- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) - +- LogicalProject(c=[$0]) - +- LogicalFilter(condition=[true]) - +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ++- LogicalJoin(condition=[AND(=($0, $2), >($1, 2))], joinType=[anti]) + :- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) + +- LogicalProject(c=[$0]) + +- LogicalFilter(condition=[true]) + +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ]]> @@ -365,14 +359,12 @@ LogicalProject(c=[$0]) ($1, 2)]) - : +- LogicalFilter(condition=[>($1, 2)]) - : +- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) - +- LogicalProject(c=[$0]) - +- LogicalFilter(condition=[true]) - +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ++- LogicalJoin(condition=[=($0, $2)], joinType=[semi]) + :- LogicalFilter(condition=[>($1, 2)]) + : +- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) + +- LogicalProject(c=[$0]) + +- LogicalFilter(condition=[true]) + +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ]]> @@ -421,14 +413,12 @@ LogicalFilter(condition=[AND(=($cor0.a, $0), >($cor0.b, 2))]) ($1, 2)]) - : +- LogicalFilter(condition=[>($1, 2)]) - : +- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) - +- LogicalProject(c=[$0]) - +- LogicalFilter(condition=[true]) - +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ++- LogicalJoin(condition=[=($0, $2)], joinType=[semi]) + :- LogicalFilter(condition=[>($1, 2)]) + : +- LogicalTableScan(table=[[default_catalog, default_database, leftT, source: [TestTableSource(a, b)]]]) + +- LogicalProject(c=[$0]) + +- LogicalFilter(condition=[true]) + +- LogicalTableScan(table=[[default_catalog, default_database, rightT, source: [TestTableSource(c, d)]]]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PushFilterIntoTableSourceScanRuleTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PushFilterIntoTableSourceScanRuleTest.xml index 5a490fc2edcbdb0bd5088483bfd3df3499fe518e..5ecf1310493ae0f92c1e651dbe4d17c13424224e 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PushFilterIntoTableSourceScanRuleTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/PushFilterIntoTableSourceScanRuleTest.xml @@ -24,14 +24,14 @@ limitations under the License. ($3, 10)]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> ($3, 10)]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[true], filter=[]]]]) ]]> @@ -43,14 +43,14 @@ LogicalProject(name=[$0], id=[$1], amount=[$2], price=[$3]) ($2, 2), <($2, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> ($2, 2), <($2, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[true], filter=[]]]]) ]]> @@ -62,13 +62,13 @@ LogicalProject(name=[$0], id=[$1], amount=[$2], price=[$3]) ($2, 2)]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> @@ -80,13 +80,13 @@ LogicalProject(name=[$0], id=[$1], amount=[$2], price=[$3]) ($2, 2), <($2, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> @@ -98,14 +98,14 @@ LogicalProject(name=[$0], id=[$1], amount=[$2], price=[$3]) ($2, 2), >($3, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> ($3, 10)]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filter=[greaterThan(amount, 2)]]]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[true], filter=[greaterThan(amount, 2)]]]]) ]]> @@ -117,14 +117,14 @@ LogicalProject(name=[$0], id=[$1], amount=[$2], price=[$3]) ($2, 2), >($3, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> ($2, 2), >($3, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[true], filter=[]]]]) ]]> @@ -139,14 +139,14 @@ SELECT * FROM MyTable WHERE ($2, 2), <($1, 100), >(CAST($2):BIGINT, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (CAST($2):BIGINT, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filter=[greaterThan(amount, 2)]]]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[true], filter=[greaterThan(amount, 2)]]]]) ]]> @@ -158,14 +158,14 @@ LogicalProject(name=[$0], id=[$1], amount=[$2], price=[$3]) ($2, 2), <(myUdf($2), 32))]) - +- LogicalTableScan(table=[[default_catalog, default_database, MyTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, MyTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.xml index b975d4e552f16d0c5fe641db92d6b53c335c1b76..7736deae1778182ee6905f43684754184f7a8dab 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQueryAntiJoinTest.xml @@ -435,13 +435,11 @@ LogicalFilter(condition=[AND(=($cor0.a, $0), >($cor0.b, 10))]) ($1, 10)]) - : +- LogicalTableScan(table=[[default_catalog, default_database, x, source: [TestTableSource(a, b)]]]) - +- LogicalProject(c=[$0]) - +- LogicalFilter(condition=[true]) - +- LogicalTableScan(table=[[default_catalog, default_database, y, source: [TestTableSource(c, d)]]]) ++- LogicalJoin(condition=[AND(=($0, $2), >($1, 10))], joinType=[anti]) + :- LogicalTableScan(table=[[default_catalog, default_database, x, source: [TestTableSource(a, b)]]]) + +- LogicalProject(c=[$0]) + +- LogicalFilter(condition=[true]) + +- LogicalTableScan(table=[[default_catalog, default_database, y, source: [TestTableSource(c, d)]]]) ]]> @@ -1006,13 +1004,11 @@ LogicalProject(c=[$0]) ($1, 10)]) - : +- LogicalTableScan(table=[[default_catalog, default_database, x, source: [TestTableSource(a, b)]]]) - +- LogicalProject(c=[$0]) - +- LogicalFilter(condition=[true]) - +- LogicalTableScan(table=[[default_catalog, default_database, y, source: [TestTableSource(c, d)]]]) ++- LogicalJoin(condition=[AND(OR(=($0, $2), IS NULL($0), IS NULL($2)), >($1, 10))], joinType=[anti]) + :- LogicalTableScan(table=[[default_catalog, default_database, x, source: [TestTableSource(a, b)]]]) + +- LogicalProject(c=[$0]) + +- LogicalFilter(condition=[true]) + +- LogicalTableScan(table=[[default_catalog, default_database, y, source: [TestTableSource(c, d)]]]) ]]> @@ -1106,7 +1102,7 @@ LogicalProject(d=[$0]) LogicalProject(b=[$1]) +- LogicalProject(a=[$0], b=[$1], c=[$2]) +- LogicalJoin(condition=[AND(OR(=($3, $4), IS NULL($4)), =($2, $5))], joinType=[anti]) - :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(<>($3, 0), IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, 2)]) + :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, 2)]) : +- LogicalJoin(condition=[=($0, $5)], joinType=[left]) : :- LogicalJoin(condition=[true], joinType=[inner]) : : :- LogicalTableScan(table=[[default_catalog, default_database, l, source: [TestTableSource(a, b, c)]]]) @@ -1148,7 +1144,7 @@ LogicalProject(d=[$0]) LogicalProject(b=[$1]) +- LogicalProject(a=[$0], b=[$1], c=[$2]) +- LogicalJoin(condition=[AND(OR(=($3, $4), IS NULL($4)), =($2, $5))], joinType=[anti]) - :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(<>($3, 0), IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, OR(=($7, 0), AND(<>($7, 0), IS NULL($10), >=($8, $7), IS NOT NULL($0))), 2, 3)]) + :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, OR(=($7, 0), AND(IS NULL($10), >=($8, $7), IS NOT NULL($0))), 2, 3)]) : +- LogicalJoin(condition=[=($0, $9)], joinType=[left]) : :- LogicalJoin(condition=[true], joinType=[inner]) : : :- LogicalJoin(condition=[=($0, $5)], joinType=[left]) @@ -1198,7 +1194,7 @@ LogicalProject(e=[$1], d=[$0]) LogicalProject(b=[$1]) +- LogicalProject(a=[$0], b=[$1], c=[$2]) +- LogicalJoin(condition=[AND(OR(=($1, $4), IS NULL($1), IS NULL($4)), OR(=($3, $5), IS NULL($5)), =($2, $6))], joinType=[anti]) - :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(<>($3, 0), IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, OR(=($7, 0), AND(<>($7, 0), IS NULL($10), >=($8, $7), IS NOT NULL($0))), 2, 3)]) + :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, OR(=($7, 0), AND(IS NULL($10), >=($8, $7), IS NOT NULL($0))), 2, 3)]) : +- LogicalJoin(condition=[=($0, $9)], joinType=[left]) : :- LogicalJoin(condition=[true], joinType=[inner]) : : :- LogicalJoin(condition=[=($0, $5)], joinType=[left]) @@ -1833,7 +1829,7 @@ LogicalProject(d=[$0]) LogicalProject(b=[$1]) +- LogicalProject(a=[$0], b=[$1], c=[$2]) +- LogicalJoin(condition=[OR(=($3, $4), IS NULL($4))], joinType=[anti]) - :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(<>($3, 0), IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, 2)]) + :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, 2)]) : +- LogicalJoin(condition=[=($0, $5)], joinType=[left]) : :- LogicalJoin(condition=[true], joinType=[inner]) : : :- LogicalTableScan(table=[[default_catalog, default_database, l, source: [TestTableSource(a, b, c)]]]) @@ -1870,7 +1866,7 @@ LogicalProject(d=[$0]) LogicalProject(b=[$1]) +- LogicalProject(a=[$0], b=[$1], c=[$2]) +- LogicalJoin(condition=[OR(=($3, $4), IS NULL($4))], joinType=[anti]) - :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(<>($3, 0), IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, 2)]) + :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, 2)]) : +- LogicalJoin(condition=[=($0, $5)], joinType=[left]) : :- LogicalJoin(condition=[true], joinType=[inner]) : : :- LogicalTableScan(table=[[default_catalog, default_database, l, source: [TestTableSource(a, b, c)]]]) @@ -1910,7 +1906,7 @@ LogicalProject(d=[$0]) LogicalProject(b=[$1]) +- LogicalProject(a=[$0], b=[$1], c=[$2]) +- LogicalJoin(condition=[OR(=($3, $4), IS NULL($4))], joinType=[anti]) - :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(<>($3, 0), IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, OR(=($7, 0), AND(<>($7, 0), IS NULL($10), >=($8, $7), IS NOT NULL($0))), 2, 3)]) + :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, OR(=($7, 0), AND(IS NULL($10), >=($8, $7), IS NOT NULL($0))), 2, 3)]) : +- LogicalJoin(condition=[=($0, $9)], joinType=[left]) : :- LogicalJoin(condition=[true], joinType=[inner]) : : :- LogicalJoin(condition=[=($0, $5)], joinType=[left]) @@ -1958,7 +1954,7 @@ LogicalProject(e=[$1], d=[$0]) LogicalProject(b=[$1]) +- LogicalProject(a=[$0], b=[$1], c=[$2]) +- LogicalJoin(condition=[AND(OR(=($1, $4), IS NULL($1), IS NULL($4)), OR(=($3, $5), IS NULL($5)))], joinType=[anti]) - :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(<>($3, 0), IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, OR(=($7, 0), AND(<>($7, 0), IS NULL($10), >=($8, $7), IS NOT NULL($0))), 2, 3)]) + :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, OR(=($7, 0), AND(IS NULL($10), >=($8, $7), IS NOT NULL($0))), 2, 3)]) : +- LogicalJoin(condition=[=($0, $9)], joinType=[left]) : :- LogicalJoin(condition=[true], joinType=[inner]) : : :- LogicalJoin(condition=[=($0, $5)], joinType=[left]) @@ -2006,7 +2002,7 @@ LogicalProject(d=[$0], e=[$1]) LogicalProject(c=[$2]) +- LogicalProject(a=[$0], b=[$1], c=[$2]) +- LogicalJoin(condition=[AND(OR(=($3, $5), IS NULL($5)), OR(=($4, $6), IS NULL($6)))], joinType=[anti]) - :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(<>($3, 0), IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, 2)], $f4=[CASE(OR(=($7, 0), AND(<>($7, 0), IS NULL($10), >=($8, $7), IS NOT NULL($1))), 3, 4)]) + :- LogicalProject(a=[$0], b=[$1], c=[$2], $f3=[CASE(OR(=($3, 0), AND(IS NULL($6), >=($4, $3), IS NOT NULL($0))), 1, 2)], $f4=[CASE(OR(=($7, 0), AND(IS NULL($10), >=($8, $7), IS NOT NULL($1))), 3, 4)]) : +- LogicalJoin(condition=[=($1, $9)], joinType=[left]) : :- LogicalJoin(condition=[true], joinType=[inner]) : : :- LogicalJoin(condition=[=($0, $5)], joinType=[left]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.xml index f0531e681d4658c2e393f792bc3e2f75d87ff820..97b419b754531dbd110abbda34a04219b2963b60 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/rules/logical/subquery/SubQuerySemiJoinTest.xml @@ -138,13 +138,11 @@ LogicalFilter(condition=[AND(=($cor0.a, $0), >($cor0.b, 10))]) ($1, 10)]) - : +- LogicalTableScan(table=[[default_catalog, default_database, x, source: [TestTableSource(a, b)]]]) - +- LogicalProject(c=[$0]) - +- LogicalFilter(condition=[true]) - +- LogicalTableScan(table=[[default_catalog, default_database, y, source: [TestTableSource(c, d)]]]) ++- LogicalJoin(condition=[AND(=($0, $2), >($1, 10))], joinType=[semi]) + :- LogicalTableScan(table=[[default_catalog, default_database, x, source: [TestTableSource(a, b)]]]) + +- LogicalProject(c=[$0]) + +- LogicalFilter(condition=[true]) + +- LogicalTableScan(table=[[default_catalog, default_database, y, source: [TestTableSource(c, d)]]]) ]]> @@ -1556,13 +1554,11 @@ LogicalProject(c=[$0]) ($1, 10)]) - : +- LogicalTableScan(table=[[default_catalog, default_database, x, source: [TestTableSource(a, b)]]]) - +- LogicalProject(c=[$0]) - +- LogicalFilter(condition=[true]) - +- LogicalTableScan(table=[[default_catalog, default_database, y, source: [TestTableSource(c, d)]]]) ++- LogicalJoin(condition=[AND(=($0, $2), >($1, 10))], joinType=[semi]) + :- LogicalTableScan(table=[[default_catalog, default_database, x, source: [TestTableSource(a, b)]]]) + +- LogicalProject(c=[$0]) + +- LogicalFilter(condition=[true]) + +- LogicalTableScan(table=[[default_catalog, default_database, y, source: [TestTableSource(c, d)]]]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SetOperatorsTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SetOperatorsTest.xml index b28a386e54a002d9f73903bfd9fa609df5a4f241..b93bc2c18da41d0cae5cfa907d9c6ba9ca9657a0 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SetOperatorsTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/SetOperatorsTest.xml @@ -226,8 +226,8 @@ LogicalMinus(all=[false]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableSourceTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableSourceTest.xml index 3d4f11f98c7604950c7d88443815110eb7c81173..837fa0829a399ea88efdd99cd60a73898ecb8e91 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableSourceTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/TableSourceTest.xml @@ -40,13 +40,13 @@ TableSourceScan(table=[[default_catalog, default_database, MyTable, source: [Tes ($3, 10)]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (price, 10)]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[]]]], fields=[name, id, amount, price]) ]]> @@ -58,13 +58,13 @@ Calc(select=[name, id, amount, price], where=[>(price, 10)]) ($2, 2), >($3, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (amount, 2), >(price, 10))]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[]]]], fields=[name, id, amount, price]) ]]> @@ -76,13 +76,13 @@ Calc(select=[name, id, amount, price], where=[OR(>(amount, 2), >(price, 10))]) ($2, 2), <($2, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (amount, 2), <(amount, 10))]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[]]]], fields=[name, id, amount, price]) ]]> @@ -94,12 +94,12 @@ Calc(select=[name, id, amount, price], where=[OR(>(amount, 2), <(amount, 10))]) ($2, 2)]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> @@ -140,12 +140,12 @@ Calc(select=[name, w$end AS EXPR$1, EXPR$2]) ($2, 2), <($2, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> @@ -157,13 +157,13 @@ TableSourceScan(table=[[default_catalog, default_database, FilterableTable, sour ($2, 2), >($3, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (price, 10)]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filter=[greaterThan(amount, 2)]]]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[greaterThan(amount, 2)]]]], fields=[name, id, amount, price]) ]]> @@ -178,13 +178,13 @@ SELECT * FROM FilterableTable WHERE ($2, 2), <($1, 100), >(CAST($2):BIGINT, 10))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> (CAST(amount), 10))]) -+- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filter=[greaterThan(amount, 2)]]]], fields=[name, id, amount, price]) ++- TableSourceScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[true], filter=[greaterThan(amount, 2)]]]], fields=[name, id, amount, price]) ]]> @@ -196,13 +196,13 @@ Calc(select=[name, id, amount, price], where=[AND(<(id, 100), >(CAST(amount), 10 ($2, 2), <(myUdf($2), 32))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable, source: [filterPushedDown=[false], filter=[]]]]) ]]> @@ -383,13 +383,13 @@ SELECT id FROM FilterableTable1 WHERE ($2, 14:25:02), >($1, 2017-02-03), >($3, 2017-02-03 14:25:02))]) - +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable1]]) + +- LogicalTableScan(table=[[default_catalog, default_database, FilterableTable1, source: [filterPushedDown=[false], filter=[]]]]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml index 5d63a64847b79976749273f06bdb31d3b8088bbb..49b6a4068352649ce9a137ef1abd604384806ef6 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/JoinTest.xml @@ -651,19 +651,18 @@ Calc(select=[a1, b1], updateAsRetraction=[false], accMode=[AccRetract]) @@ -1061,11 +1059,11 @@ ON (src1.key = src2.key AND src2.key > 10) ($2, 10))], joinType=[left]) :- LogicalProject(key=[$0], v=[$1]) : +- LogicalFilter(condition=[=($0, 0)]) : +- LogicalTableScan(table=[[default_catalog, default_database, src, source: [TestTableSource(key, v)]]]) - +- LogicalProject(key=[$0], v=[$1], $f2=[>($0, 10)]) + +- LogicalProject(key=[$0], v=[$1]) +- LogicalFilter(condition=[=($0, 0)]) +- LogicalTableScan(table=[[default_catalog, default_database, src, source: [TestTableSource(key, v)]]]) ]]> diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/SemiAntiJoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/SemiAntiJoinTest.xml index 879cfaa4a265a8b93b08e1134a6aa7819cce1890..7ea28313c0299dbfd9085e2058c56e4b38a3a2ca 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/SemiAntiJoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/sql/join/SemiAntiJoinTest.xml @@ -2273,7 +2273,7 @@ LogicalProject(d=[$0]) Calc(select=[b]) +- Join(joinType=[LeftAntiJoin], where=[AND(OR(=($f3, d), IS NULL(d)), =(c, f))], select=[b, c, $f3], leftInputSpec=[NoUniqueKey], rightInputSpec=[NoUniqueKey]) :- Exchange(distribution=[hash[c]]) - : +- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(<>(c0, 0), IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(<>(c1, 0), IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) + : +- Calc(select=[b, c, CASE(OR(=(c0, 0), AND(IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) : +- Join(joinType=[LeftOuterJoin], where=[=(a, EXPR$0)], select=[a, b, c, c0, ck, i0, c1, ck0, EXPR$0, i], leftInputSpec=[NoUniqueKey], rightInputSpec=[HasUniqueKey]) : :- Exchange(distribution=[hash[a]]) : : +- Join(joinType=[InnerJoin], where=[true], select=[a, b, c, c0, ck, i0, c1, ck0], leftInputSpec=[NoUniqueKey], rightInputSpec=[JoinKeyContainsUniqueKey]) @@ -2379,14 +2379,12 @@ LogicalProject(d=[$0]) (b, 10) AS $f3]) - : +- TableSourceScan(table=[[default_catalog, default_database, l, source: [TestTableSource(a, b, c)]]], fields=[a, b, c]) - +- Exchange(distribution=[single]) - +- Calc(select=[d]) - +- TableSourceScan(table=[[default_catalog, default_database, r, source: [TestTableSource(d, e, f)]]], fields=[d, e, f]) +Join(joinType=[LeftAntiJoin], where=[AND(OR(=(a, d), IS NULL(a), IS NULL(d)), >(b, 10))], select=[a, b, c], leftInputSpec=[NoUniqueKey], rightInputSpec=[NoUniqueKey]) +:- Exchange(distribution=[single]) +: +- TableSourceScan(table=[[default_catalog, default_database, l, source: [TestTableSource(a, b, c)]]], fields=[a, b, c]) ++- Exchange(distribution=[single]) + +- Calc(select=[d]) + +- TableSourceScan(table=[[default_catalog, default_database, r, source: [TestTableSource(d, e, f)]]], fields=[d, e, f]) ]]> @@ -2519,7 +2517,7 @@ LogicalProject(e=[$1], d=[$0]) Calc(select=[b]) +- Join(joinType=[LeftAntiJoin], where=[AND(OR(=(b, e), IS NULL(b), IS NULL(e)), OR(=($f3, d), IS NULL(d)))], select=[b, $f3], leftInputSpec=[NoUniqueKey], rightInputSpec=[NoUniqueKey]) :- Exchange(distribution=[single]) - : +- Calc(select=[b, CASE(OR(=(c0, 0), AND(<>(c0, 0), IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(<>(c1, 0), IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) + : +- Calc(select=[b, CASE(OR(=(c0, 0), AND(IS NULL(i0), >=(ck, c0), IS NOT NULL(a))), 1, OR(=(c1, 0), AND(IS NULL(i), >=(ck0, c1), IS NOT NULL(a))), 2, 3) AS $f3]) : +- Join(joinType=[LeftOuterJoin], where=[=(a, j)], select=[a, b, c0, ck, i0, c, ck0, j, i], leftInputSpec=[NoUniqueKey], rightInputSpec=[HasUniqueKey]) : :- Exchange(distribution=[hash[a]]) : : +- Join(joinType=[InnerJoin], where=[true], select=[a, b, c0, ck, i0, c, ck0], leftInputSpec=[NoUniqueKey], rightInputSpec=[JoinKeyContainsUniqueKey]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/table/AggregateTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/table/AggregateTest.xml index 95b1683c88dcf23eace5fd9efc58ef70c67e142a..0542c55c3306efc65229fc1141eec610ea2db44f 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/table/AggregateTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/table/AggregateTest.xml @@ -247,13 +247,13 @@ Calc(select=[EXPR$0, EXPR$1]) diff --git a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/table/JoinTest.xml b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/table/JoinTest.xml index 437cf16decefc57bbeb61ed14280ca42785030a4..952b2a8988b137e8c2934a4706529afbc5082074 100644 --- a/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/table/JoinTest.xml +++ b/flink-table/flink-table-planner-blink/src/test/resources/org/apache/flink/table/planner/plan/stream/table/JoinTest.xml @@ -28,9 +28,9 @@ LogicalProject(b=[$1], y=[$4]) diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/JobDispatcherResourceManagerComponent.java b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala similarity index 32% rename from flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/JobDispatcherResourceManagerComponent.java rename to flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala index c1df47fe98e230403a235831318339196e0d295a..953f0f311565b19d82253fa6c067541bab21dedd 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/component/JobDispatcherResourceManagerComponent.java +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala @@ -15,41 +15,47 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.flink.table.api.validation -package org.apache.flink.runtime.entrypoint.component; +import org.apache.flink.api.scala._ +import org.apache.flink.table.api.ValidationException +import org.apache.flink.table.api.scala._ +import org.apache.flink.table.planner.expressions.utils.Func0 +import org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions.OverAgg0 +import org.apache.flink.table.planner.utils.TableTestBase +import org.junit.Test -import org.apache.flink.runtime.clusterframework.ApplicationStatus; -import org.apache.flink.runtime.dispatcher.MiniDispatcher; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; -import org.apache.flink.runtime.resourcemanager.ResourceManager; -import org.apache.flink.runtime.webmonitor.WebMonitorEndpoint; +class UserDefinedFunctionValidationTest extends TableTestBase { -import java.util.concurrent.CompletableFuture; + @Test + def testScalarFunctionOperandTypeCheck(): Unit = { + thrown.expect(classOf[ValidationException]) + thrown.expectMessage( + "Given parameters of function 'func' do not match any signature. \n" + + "Actual: (java.lang.String) \n" + + "Expected: (int)") + val util = scalaStreamTestUtil() + util.addTableSource[(Int, String)]("t", 'a, 'b) + util.tableEnv.registerFunction("func", Func0) + util.verifyExplain("select func(b) from t") + } -/** - * {@link DispatcherResourceManagerComponent} for a job cluster. The dispatcher component starts - * a {@link MiniDispatcher}. - */ -class JobDispatcherResourceManagerComponent extends DispatcherResourceManagerComponent { - - JobDispatcherResourceManagerComponent( - MiniDispatcher dispatcher, - ResourceManager resourceManager, - LeaderRetrievalService dispatcherLeaderRetrievalService, - LeaderRetrievalService resourceManagerRetrievalService, - WebMonitorEndpoint webMonitorEndpoint, - JobManagerMetricGroup jobManagerMetricGroup) { - super(dispatcher, resourceManager, dispatcherLeaderRetrievalService, resourceManagerRetrievalService, webMonitorEndpoint, jobManagerMetricGroup); + @Test + def testAggregateFunctionOperandTypeCheck(): Unit = { + thrown.expect(classOf[ValidationException]) + thrown.expectMessage( + "Given parameters of function 'agg' do not match any signature. \n" + + "Actual: (org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions" + + ".Accumulator0, java.lang.String, java.lang.Integer) \n" + + "Expected: (org.apache.flink.table.planner.runtime.utils.JavaUserDefinedAggFunctions" + + ".Accumulator0, long, int)") - final CompletableFuture shutDownFuture = getShutDownFuture(); + val util = scalaStreamTestUtil() + val agg = new OverAgg0 + util.addTableSource[(Int, String)]("t", 'a, 'b) + util.tableEnv.registerFunction("agg", agg) + util.verifyExplain("select agg(b, a) from t") + } - dispatcher.getJobTerminationFuture().whenComplete((applicationStatus, throwable) -> { - if (throwable != null) { - shutDownFuture.completeExceptionally(throwable); - } else { - shutDownFuture.complete(applicationStatus); - } - }); - } } + diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/DecimalTypeTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/DecimalTypeTest.scala index e2439309efe9d6e6d03a3e4bf3ebbf6c723a3f27..0e4bbdfe98a6d52388a78d9c2115efba2a6135c2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/DecimalTypeTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/DecimalTypeTest.scala @@ -23,13 +23,25 @@ import org.apache.flink.table.api.scala._ import org.apache.flink.table.api.{DataTypes, Types} import org.apache.flink.table.expressions.utils.ApiExpressionUtils.valueLiteral import org.apache.flink.table.planner.expressions.utils.ExpressionTestBase -import org.apache.flink.table.runtime.typeutils.BigDecimalTypeInfo +import org.apache.flink.table.runtime.types.TypeInfoLogicalTypeConverter.fromLogicalTypeToTypeInfo +import org.apache.flink.table.types.logical.DecimalType import org.apache.flink.types.Row - import org.junit.{Ignore, Test} class DecimalTypeTest extends ExpressionTestBase { + private def DECIMAL = (p: Int, s: Int) => new DecimalType(p, s) + + private def BOOL = DataTypes.BOOLEAN.getLogicalType + + private def INT = DataTypes.INT.getLogicalType + + private def LONG = DataTypes.BIGINT.getLogicalType + + private def DOUBLE = DataTypes.DOUBLE.getLogicalType + + private def STRING = DataTypes.STRING.getLogicalType + @Test def testDecimalLiterals(): Unit = { // implicit double @@ -295,32 +307,1074 @@ class DecimalTypeTest extends ExpressionTestBase { "f4 < f0", "true") - // TODO add all tests if FLINK-4070 is fixed - testSqlApi( + testAllApis( + 12.toExpr < 'f1, "12 < f1", + "12 < f1", + "true") + + testAllApis( + 12.toExpr > 'f1, + "12 > f1", + "12 > f1", + "false") + + testAllApis( + 12.toExpr - 'f37, + "12 - f37", + "12 - f37", + "10") + + testAllApis( + 12.toExpr + 'f37, + "12 + f37", + "12 + f37", + "14") + + testAllApis( + 12.toExpr * 'f37, + "12 * f37", + "12 * f37", + "24") + + testAllApis( + 12.toExpr / 'f37, + "12 / f37", + "12 / f37", + "6") + } + + @Test + def testFieldAcess(): Unit = { + + // the most basic case + testAllApis( + 'f6, + "f6", + "f6", + "123") + + testAllApis( + 'f7, + "f7", + "f7", + "123.45") + + // data from source are rounded to their declared scale before entering next step + testAllApis( + 'f8, + "f8", + "f8", + "100.00") + + testAllApis( + 'f8 + 'f8, + "f8 + f8", + "f8 + f8", + "200.00") + + // trailing zeros are padded to the scale + testAllApis( + 'f9, + "f9", + "f9", + "100.10") + + testAllApis( + 'f9 + 'f9, + "f9 + f9", + "f9 + f9", + "200.20") + + // source data is within precision after rounding + testAllApis( + 'f10, + "f10", + "f10", + "100.00") + + testAllApis( + 'f10 + 'f10, + "f10 + f10", + "f10 + f10", + "200.00") + + // source data overflows over precision (after rounding) + testAllApis( + 'f11, + "f11", + "f11", + "null") + + testAllApis( + 'f12, + "f12", + "f12", + "null") + } + + @Test + def testUnaryPlusMinus(): Unit = { + + testAllApis( + + 'f6, + "+f6", + "+f6", + "123") + + testAllApis( + - 'f7, + "-f7", + "-f7", + "-123.45") + + testAllApis( + - (( + 'f6) - ( - 'f7)), + "- (( + f6) - ( - f7))", + "- (( + f6) - ( - f7))", + "-246.45") + } + + @Test + def testPlusMinus(): Unit = { + + // see calcite ReturnTypes.DECIMAL_SUM + // s = max(s1,s2), p-s = max(p1-s1, p2-s2) + 1 + // p then is capped at 38 + testAllApis( + 'f13 + 'f14, + "f13 + f14", + "f13 + f14", + "300.2434") + + testAllApis( + 'f13 - 'f14, + "f13 - f14", + "f13 - f14", + "-100.0034") + + // INT => DECIMAL(10,0) + // approximate + exact => approximate + testAllApis( + 'f7 + 'f2, + "f7 + f2", + "f7 + f2", + "165.45") + + testAllApis( + 'f2 + 'f7, + "f2 + f7", + "f2 + f7", + "165.45") + + testAllApis( + 'f7 + 'f3, + "f7 + f3", + "f7 + f3", + "127.65") + + testAllApis( + 'f3 + 'f7, + "f3 + f7", + "f3 + f7", + "127.65") + + // our result type precision is capped at 38 + // SQL2003 $6.26 -- result scale is dictated as max(s1,s2). no approximation allowed. + // calcite -- scale is not reduced; integral part may be reduced. overflow may occur + // (38,10)+(38,28)=>(57,28)=>(38,28) + // T-SQL -- scale may be reduced to keep the integral part. approximation may occur + // (38,10)+(38,28)=>(57,28)=>(38,9) + testAllApis( + 'f15 + 'f16, + "f15 + f16", + "f15 + f16", + "300.0246913578012345678901234567") + + testAllApis( + 'f15 - 'f16, + "f15 - f16", + "f15 - f16", + "-100.0000000000012345678901234567") + + // 10 digits integral part + testAllApis( + 'f17 + 'f18, + "f17 + f18", + "f17 + f18", + "null") + + testAllApis( + 'f17 - 'f18, + "f17 - f18", + "f17 - f18", + "null") + + // requires 39 digits + testAllApis( + 'f19 + 'f19, + "f19 + f19", + "f19 + f19", + "null") + + // overflows in subexpression + testAllApis( + 'f19 + 'f19 - 'f19, + "f19 + f19 - f19", + "f19 + f19 - f19", + "null") + } + + @Test + def testMultiply(): Unit = { + + // see calcite ReturnTypes.DECIMAL_PRODUCT + // s = s1+s2, p = p1+p2 + // both p&s are capped at 38 + // if s>38, result is rounded to s=38, and the integral part can only be zero + testAllApis( + 'f20 * 'f20, + "f20 * f20", + "f20 * f20", + "1.0000") + + testAllApis( + 'f20 * 'f21, + "f20 * f21", + "f20 * f21", + "2.000000") + + // INT => DECIMAL(10,0) + // approximate * exact => approximate + testAllApis( + 'f20 * 'f22, + "f20 * f22", + "f20 * f22", + "200.00") + + testAllApis( + 'f22 * 'f20, + "f22 * f20", + "f22 * f20", + "200.00") + + testAllApis( + 'f20 * 'f23, + "f20 * f23", + "f20 * f23", + "3.14") + + testAllApis( + 'f23 * 'f20, + "f23 * f20", + "f23 * f20", + "3.14") + + // precision is capped at 38; scale will not be reduced (unless over 38) + // similar to plus&minus, and calcite behavior is different from T-SQL. + testAllApis( + 'f24 * 'f24, + "f24 * f24", + "f24 * f24", + "1.000000000000") + + testAllApis( + 'f24 * 'f25, + "f24 * f25", + "f24 * f25", + "2.0000000000000000") + + testAllApis( + 'f26 * 'f26, + "f26 * f26", + "f26 * f26", + "0.00010000000000000000000000000000000000" + ) + + // scalastyle:off + // we don't have this ridiculous behavior: + // https://blogs.msdn.microsoft.com/sqlprogrammability/2006/03/29/multiplication-and-division-with-numerics/ + // scalastyle:on + + testAllApis( + 'f27 * 'f28, + "f27 * f28", + "f27 * f28", + "0.00000060000000000000" + ) + + // result overflow + testAllApis( + 'f29 * 'f29, + "f29 * f29", + "f29 * f29", + "null" + ) + + //(60,40)=>(38,38), no space for integral part + testAllApis( + 'f30 * 'f30, + "f30 * f30", + "f30 * f30", + "null" + ) + } + + @Test + def testDivide(): Unit = { + + // the default impl of Calcite apparently borrows from T-SQL, but differs in details. + // Flink overrides it to follow T-SQL exactly. See FlinkTypeFactory.createDecimalQuotient() + testAllApis( + 'f31 / 'f32, + "f31 / f32", + "f31 / f32", + "0.333333") + + testAllApis( + 'f31 / 'f33, + "f31 / f33", + "f31 / f33", + "0.3333333") + + testAllApis( + 'f31 / 'f34, + "f31 / f34", + "f31 / f34", + "0.3333333333") + + testAllApis( + 'f31 / 'f35, + "f31 / f35", + "f31 / f35", + "0.333333") + + // INT => DECIMAL(10,0) + // approximate / exact => approximate + testAllApis( + 'f36 / 'f37, + "f36 / f37", + "f36 / f37", + "0.5000000000000") + + + testAllApis( + 'f37 / 'f36, + "f37 / f36", + "f37 / f36", + "2.00000000000") + + + testAllApis( + 'f36 / 'f38, + "f36 / f38", + "f36 / f38", + (1.0/3.0).toString) + + testAllApis( + 'f38 / 'f36, + "f38 / f36", + "f38 / f36", + (3.0/1.0).toString) + + // result overflow, because result type integral part is reduced + testAllApis( + 'f39 / 'f40, + "f39 / f40", + "f39 / f40", + "null") + } + + @Test + def testMod(): Unit = { + // MOD(Exact1, Exact2) => Exact2 + testAllApis( + 'f41 % 'f42, + "f41 % f42", + "mod(f41, f42)", + "3.0000") + + testAllApis( + 'f42 % 'f41, + "f42 % f41", + "mod(f42, f41)", + "2.00") + + testAllApis( + 'f41 % 'f43, + "f41 % f43", + "mod(f41, f43)", + "3") + + testAllApis( + 'f43 % 'f41, + "f43 % f41", + "mod(f43, f41)", + "1.00") + + // signs. consistent with Java's % operator. + testAllApis( + 'f44 % 'f45, + "f44 % f45", + "mod(f44, f45)", + (3%5).toString) + + testAllApis( + -'f44 % 'f45, + "-f44 % f45", + "mod(-f44, f45)", + ((-3)%5).toString) + + testAllApis( + 'f44 % -'f45, + "f44 % -f45", + "mod(f44, -f45)", + (3%(-5)).toString) + + testAllApis( + -'f44 % -'f45, + "-f44 % -f45", + "mod(-f44, -f45)", + ((-3)%(-5)).toString) + + // rounding in case s1>s2. note that SQL2003 requires s1=s2=0. + // (In T-SQL, s2 is expanded to s1, so that there's no rounding.) + testAllApis( + 'f46 % 'f47, + "f46 % f47", + "mod(f46, f47)", + "3.12") + } + + @Test // functions that treat Decimal as exact value + def testExactionFunctions(): Unit = { + + testAllApis( + ifThenElse('f48 > 'f49, 'f48, 'f49), + "ifThenElse(greaterThan(f48, f49), f48, f49)", + "if(f48 > f49, f48, f49)", + "3.14") + + testAllApis( + 'f48.abs(), + "f48.abs()", + "abs(f48)", + "3.14" + ) + + testAllApis( + (-'f48).abs(), + "(-f48).abs()", + "abs(-f48)", + "3.14" + ) + + testAllApis( + 'f48.floor(), + "f48.floor()", + "floor(f48)", + "3" + ) + + testAllApis( + 'f48.ceil(), + "f48.ceil()", + "ceil(f48)", + "4" + ) + + // calcite: SIGN(Decimal(p,s))=>Decimal(p,s) + testAllApis( + 'f48.sign(), + "f48.sign()", + "sign(f48)", + "1.00" + ) + + testAllApis( + (-'f48).sign(), + "(-f48).sign()", + "sign(-f48)", + "-1.00" + ) + testAllApis( + ('f48 - 'f48).sign(), + "(f48 - f48).sign()", + "sign(f48 - f48)", + "0.00" + ) + + // ROUND(Decimal(p,s)[,INT]) + testAllApis( + 'f50.round(0), + "f50.round(0)", + "round(f50)", + "647") + + testAllApis( + 'f50.round(0), + "f50.round(0)", + "round(f50,0)", + "647") + + testAllApis( + 'f50.round(1), + "f50.round(1)", + "round(f50,1)", + "646.6") + + testAllApis( + 'f50.round(2), + "f50.round(2)", + "round(f50,2)", + "646.65") + + testAllApis( + 'f50.round(3), + "f50.round(3)", + "round(f50,3)", + "646.646") + + testAllApis( + 'f50.round(4), + "f50.round(4)", + "round(f50,4)", + "646.646") + + testAllApis( + 'f50.round(-1), + "f50.round(-1)", + "round(f50,-1)", + "650") + + testAllApis( + 'f50.round(-2), + "f50.round(-2)", + "round(f50,-2)", + "600") + + testAllApis( + 'f50.round(-3), + "f50.round(-3)", + "round(f50,-3)", + "1000") + + testAllApis( + 'f50.round(-4), + "f50.round(-4)", + "round(f50,-4)", + "0") + + testAllApis( + 'f51.round(1), + "f51.round(1)", + "round(f51,1)", + "100.0") + + testAllApis( + (-'f51).round(1), + "(-f51).round(1)", + "round(-f51,1)", + "-100.0") + + testAllApis( + ('f51).round(-1), + "(f51).round(-1)", + "round(f51,-1)", + "100") + + testAllApis( + (-'f51).round(-1), + "(-f51).round(-1)", + "round(-f51,-1)", + "-100") + + testAllApis( + ('f52).round(-1), + "(f52).round(-1)", + "round(f52,-1)", + "null") + } + + @Test // functions e.g. sin() that treat Decimal as double + def testApproximateFunctions(): Unit = { + // skip moving ApproximateFunctions tests from + // sql/DecimalITCase.scala and table/DecimalITcase.scala + // because these tests will run fail until FLINK-14036 is fixed + } + + @Test + def testCaseWhen(): Unit = { + + // result type: SQL2003 $9.23, calcite RelDataTypeFactory.leastRestrictive() + testSqlApi( + "case f53 when 0 then f53 else f54 end", + "0.0100") + + testSqlApi( + "case f53 when 0 then f53 else f2 end", + "42.0000") + + testSqlApi( + "case f53 when 0 then f23 else f53 end", + BigDecimal("0.0001").doubleValue().toString) + } + + @Test + def testCast(): Unit = { + + // String, numeric/Decimal => Decimal + testSqlApi( + "cast(f48 as Decimal(8,4))", + "3.1400") + + testSqlApi( + "cast(f2 as Decimal(8,4))", + "42.0000") + + testSqlApi( + "cast(f3 as Decimal(8,4))", + "4.2000") + + testSqlApi( + "cast(f55 as Decimal(8,4))", + "3.1400") + + // round up + testSqlApi( + "cast(f56 as Decimal(8,1))", + "3.2") + + testSqlApi( + "cast(f57 as Decimal(8,1))", + "3.2") + + testSqlApi( + "cast(f58 as Decimal(8,1))", + "3.2") + + testSqlApi( + "cast(f59 as Decimal(3,2))", + "null") + + // Decimal => String, numeric + testSqlApi( + "cast(f60 as VARCHAR(64))", + "1.99") + + testSqlApi( + "cast(f61 as DOUBLE)", + "1.99") + + testSqlApi( + "cast(f62 as INT)", + "1") + } + + @Test + def testEquality(): Unit = { + + // expressions that test equality. + // =, CASE, NULLIF, IN, IS DISTINCT FROM + testSqlApi( + "f63=f64", + "true") + + testSqlApi( + "f63=f65", + "true") + + testSqlApi( + "f63=f66", + "true") + + testSqlApi( + "f64=f63", + "true") + + testSqlApi( + "f65=f63", + "true") + + testSqlApi( + "f66=f63", + "true") + + testSqlApi( + "f63 IN(f64)", + "true") + + testSqlApi( + "f63 IN(f65)", + "true") + + testSqlApi( + "f63 IN(f66)", + "true") + + testSqlApi( + "f64 IN(f63)", + "true") + + testSqlApi( + "f65 IN(f63)", + "true") + + testSqlApi( + "f66 IN(f63)", + "true") + + testSqlApi( + "f63 IS DISTINCT FROM f64", + "false") + + testSqlApi( + "f64 IS DISTINCT FROM f63", + "false") + + testSqlApi( + "f63 IS DISTINCT FROM f65", + "false") + + testSqlApi( + "f65 IS DISTINCT FROM f63", + "false") + + testSqlApi( + "f63 IS DISTINCT FROM f66", + "false") + + testSqlApi( + "f66 IS DISTINCT FROM f63", + "false") + + testSqlApi( + "NULLIF(f63,f64)", + "null" + ) + + testSqlApi( + "NULLIF(f63,f65)", + "null" + ) + + testSqlApi( + "NULLIF(f63,f66)", + "null" + ) + + testSqlApi( + "NULLIF(f64,f63)", + "null" + ) + + testSqlApi( + "NULLIF(f65,f63)", + "null" + ) + + testSqlApi( + "NULLIF(f66,f63)", + "null" + ) + + testSqlApi( + "NULLIF(f63,f64)", + "null" + ) + + testSqlApi( + "case f63 when f64 then 1 else 0 end", + "1" + ) + + testSqlApi( + "case f63 when f65 then 1 else 0 end", + "1" + ) + + testSqlApi( + "case f63 when f66 then 1 else 0 end", + "1" + ) + + testSqlApi( + "case f64 when f63 then 1 else 0 end", + "1" + ) + + testSqlApi( + "case f65 when f64 then 1 else 0 end", + "1" + ) + + testSqlApi( + "case f66 when f65 then 1 else 0 end", + "1" + ) + } + + @Test + def testComparison(): Unit = { + testSqlApi( + "f63 < f64", + "false") + + testSqlApi( + "f63 < f65", + "false") + + testSqlApi( + "f63 < f66", + "false") + + testSqlApi( + "f64 < f63", + "false") + + testSqlApi( + "f65 < f63", + "false") + + testSqlApi( + "f66 < f63", + "false") + + // no overflow during type conversion. + // conceptually both operands are promoted to infinite precision before comparison. + testSqlApi( + "f67 < f68", + "true") + + testSqlApi( + "f67 < f69", + "true") + + testSqlApi( + "f67 < f70", + "true") + + testSqlApi( + "f68 < f67", + "false") + + testSqlApi( + "f69 < f67", + "false") + + testSqlApi( + "f70 < f67", + "false") + + testSqlApi( + "f63 between f64 and 1", + "true") + + testSqlApi( + "f64 between f63 and 1", + "true") + + testSqlApi( + "f63 between f65 and 1", + "true") + + testSqlApi( + "f65 between f63 and 1", + "true") + + testSqlApi( + "f63 between f66 and 1", + "true") + + testSqlApi( + "f66 between f63 and 1", + "true") + + testSqlApi( + "f63 between 0 and f64", + "true") + + testSqlApi( + "f64 between 0 and f63", + "true") + + testSqlApi( + "f63 between 0 and f65", + "true") + + testSqlApi( + "f65 between 0 and f63", + "true") + + testSqlApi( + "f63 between 0 and f66", + "true") + + testSqlApi( + "f66 between 0 and f63", "true") } // ---------------------------------------------------------------------------------------------- override def testData: Row = { - val testData = new Row(6) + val testData = new Row(71) testData.setField(0, BigDecimal("123456789.123456789123456789").bigDecimal) testData.setField(1, BigDecimal("123456789123456789123456789").bigDecimal) testData.setField(2, 42) testData.setField(3, 4.2) testData.setField(4, BigDecimal("123456789").bigDecimal) testData.setField(5, BigDecimal("0.000").bigDecimal) + + //convert ITCase to unit Test + testData.setField(6, BigDecimal("123").bigDecimal) + testData.setField(7, BigDecimal("123.45").bigDecimal) + testData.setField(8, BigDecimal("100.004").bigDecimal) + testData.setField(9, BigDecimal("100.1").bigDecimal) + testData.setField(10, BigDecimal("100.0040").bigDecimal) + testData.setField(11, BigDecimal("123").bigDecimal) + testData.setField(12, BigDecimal("123.0000").bigDecimal) + testData.setField(13, BigDecimal("100.12").bigDecimal) + testData.setField(14, BigDecimal("200.1234").bigDecimal) + testData.setField(15, BigDecimal("100.0123456789").bigDecimal) + testData.setField(16, BigDecimal("200.0123456789012345678901234567").bigDecimal) + testData.setField(17, BigDecimal("1e10").bigDecimal) + testData.setField(18, BigDecimal("0").bigDecimal) + testData.setField(19, BigDecimal("5e37").bigDecimal) + testData.setField(20, BigDecimal("1.00").bigDecimal) + testData.setField(21, BigDecimal("2.0000").bigDecimal) + testData.setField(22, 200) + testData.setField(23, 3.14) + testData.setField(24, BigDecimal("1").bigDecimal) + testData.setField(25, BigDecimal("2").bigDecimal) + testData.setField(26, BigDecimal("0.01").bigDecimal) + testData.setField(27, BigDecimal("0.0000006").bigDecimal) + testData.setField(28, BigDecimal("1.0").bigDecimal) + testData.setField(29, BigDecimal("1e19").bigDecimal) + testData.setField(30, BigDecimal("1.0").bigDecimal) + testData.setField(31, BigDecimal("1.00").bigDecimal) + testData.setField(32, BigDecimal("3").bigDecimal) + testData.setField(33, BigDecimal("3").bigDecimal) + testData.setField(34, BigDecimal("3").bigDecimal) + testData.setField(35, BigDecimal("3").bigDecimal) + testData.setField(36, BigDecimal("1.00").bigDecimal) + testData.setField(37, 2) + testData.setField(38, 3.0) + testData.setField(39, BigDecimal("1e20").bigDecimal) + testData.setField(40, BigDecimal("1e-15").bigDecimal) + testData.setField(41, BigDecimal("3.00").bigDecimal) + testData.setField(42, BigDecimal("5.00").bigDecimal) + testData.setField(43, 7) + testData.setField(44, BigDecimal("3").bigDecimal) + testData.setField(45, BigDecimal("5").bigDecimal) + testData.setField(46, BigDecimal("3.1234").bigDecimal) + testData.setField(47, BigDecimal("5").bigDecimal) + testData.setField(48, BigDecimal("3.14").bigDecimal) + testData.setField(49, BigDecimal("2.17").bigDecimal) + testData.setField(50, BigDecimal("646.646").bigDecimal) + testData.setField(51, BigDecimal("99.99").bigDecimal) + testData.setField(52, BigDecimal("1E38").bigDecimal.subtract(BigDecimal("1").bigDecimal)) + testData.setField(53, BigDecimal("0.0001").bigDecimal) + testData.setField(54, BigDecimal("0.01").bigDecimal) + testData.setField(55, "3.14") + testData.setField(56, BigDecimal("3.15").bigDecimal) + testData.setField(57, 3.15) + testData.setField(58, "3.15") + testData.setField(59, "13.14") + testData.setField(60, BigDecimal("1.99").bigDecimal) + testData.setField(61, "1.99") + testData.setField(62, 1) + testData.setField(63, BigDecimal("1").bigDecimal) + testData.setField(64, BigDecimal("1").bigDecimal) + testData.setField(65, 1) + testData.setField(66, 1.0) + + testData.setField(67, BigDecimal("1").bigDecimal) + testData.setField(68, BigDecimal("99").bigDecimal) + testData.setField(69, 99) + testData.setField(70, 99.0) + + + testData } override def typeInfo: RowTypeInfo = { new RowTypeInfo( - /* 1 */ BigDecimalTypeInfo.of(30, 18), - /* 2 */ BigDecimalTypeInfo.of(30, 0), - /* 3 */ Types.INT, - /* 4 */ Types.DOUBLE, - /* 5 */ BigDecimalTypeInfo.of(10, 0), - /* 6 */ BigDecimalTypeInfo.of(10, 3)) + /* 0 */ fromLogicalTypeToTypeInfo(DECIMAL(30, 18)), + /* 1 */ fromLogicalTypeToTypeInfo(DECIMAL(30, 0)), + /* 2 */ Types.INT(), + /* 3 */ Types.DOUBLE(), + /* 4 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 0)), + /* 5 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 3)), + + //convert ITCase to unit Test + /* 6 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 0)), + /* 7 */ fromLogicalTypeToTypeInfo(DECIMAL(7, 2)), + /* 8 */ fromLogicalTypeToTypeInfo(DECIMAL(7, 2)), + /* 9 */ fromLogicalTypeToTypeInfo(DECIMAL(7, 2)), + /* 10 */ fromLogicalTypeToTypeInfo(DECIMAL(5, 2)), + /* 11 */ fromLogicalTypeToTypeInfo(DECIMAL(2, 0)), + /* 12 */ fromLogicalTypeToTypeInfo(DECIMAL(4, 2)), + /* 13 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 2)), + /* 14 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 4)), + /* 15 */ fromLogicalTypeToTypeInfo(DECIMAL(38, 10)), + /* 16 */ fromLogicalTypeToTypeInfo(DECIMAL(38, 28)), + /* 17 */ fromLogicalTypeToTypeInfo(DECIMAL(38, 10)), + /* 18 */ fromLogicalTypeToTypeInfo(DECIMAL(38, 28)), + /* 19 */ fromLogicalTypeToTypeInfo(DECIMAL(38, 0)), + /* 20 */ fromLogicalTypeToTypeInfo(DECIMAL(5, 2)), + /* 21 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 4)), + /* 22 */ Types.INT(), + /* 23 */ Types.DOUBLE(), + /* 24 */ fromLogicalTypeToTypeInfo(DECIMAL(30, 6)), + /* 25 */ fromLogicalTypeToTypeInfo(DECIMAL(30, 10)), + /* 26 */ fromLogicalTypeToTypeInfo(DECIMAL(30, 20)), + /* 27 */ fromLogicalTypeToTypeInfo(DECIMAL(38, 10)), + /* 28 */ fromLogicalTypeToTypeInfo(DECIMAL(38, 10)), + /* 29 */ fromLogicalTypeToTypeInfo(DECIMAL(38, 0)), + /* 30 */ fromLogicalTypeToTypeInfo(DECIMAL(30, 20)), + /* 31 */ fromLogicalTypeToTypeInfo(DECIMAL(20, 2)), + /* 32 */ fromLogicalTypeToTypeInfo(DECIMAL(2, 1)), + /* 33 */ fromLogicalTypeToTypeInfo(DECIMAL(4, 3)), + /* 34 */ fromLogicalTypeToTypeInfo(DECIMAL(20, 10)), + /* 35 */ fromLogicalTypeToTypeInfo(DECIMAL(20, 16)), + /* 36 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 2)), + /* 37 */ Types.INT(), + /* 38 */ Types.DOUBLE(), + /* 39 */ fromLogicalTypeToTypeInfo(DECIMAL(30, 0)), + /* 40 */ fromLogicalTypeToTypeInfo(DECIMAL(30, 20)), + /* 41 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 2)), + /* 42 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 4)), + /* 43 */ Types.INT(), + /* 44 */ fromLogicalTypeToTypeInfo(DECIMAL(1, 0)), + /* 45 */ fromLogicalTypeToTypeInfo(DECIMAL(1, 0)), + /* 46 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 4)), + /* 47 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 2)), + /* 48 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 2)), + /* 49 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 2)), + /* 50 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 3)), + /* 51 */ fromLogicalTypeToTypeInfo(DECIMAL(4, 2)), + /* 52 */ fromLogicalTypeToTypeInfo(DECIMAL(38, 0)), + /* 53 */ fromLogicalTypeToTypeInfo(DECIMAL(8, 4)), + /* 54 */ fromLogicalTypeToTypeInfo(DECIMAL(10, 2)), + /* 55 */ Types.STRING(), + /* 56 */ fromLogicalTypeToTypeInfo(DECIMAL(8, 2)), + /* 57 */ Types.DOUBLE(), + /* 58 */ Types.STRING(), + /* 59 */ Types.STRING(), + /* 60 */ fromLogicalTypeToTypeInfo(DECIMAL(4, 2)), + /* 61 */ Types.STRING(), + /* 62 */ Types.INT(), + /* 63 */ fromLogicalTypeToTypeInfo(DECIMAL(8, 2)), + /* 64 */ fromLogicalTypeToTypeInfo(DECIMAL(8, 4)), + /* 65 */ Types.INT(), + /* 66 */ Types.DOUBLE(), + + /* 67 */ fromLogicalTypeToTypeInfo(DECIMAL(1, 0)), + /* 68 */ fromLogicalTypeToTypeInfo(DECIMAL(2, 0)), + /* 69 */ Types.INT(), + /* 70 */ Types.DOUBLE() + ) } } diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala index 4e1bb8d965e5d26b02e53ac95459e1a82317c236..064e8e0b3d7014b74a808c4c523d4c711f123b68 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/TemporalTypesTest.scala @@ -645,7 +645,7 @@ class TemporalTypesTest extends ExpressionTestBase { } private def timestampTz(str: String) = { - s"CAST(TIMESTAMP '$str' AS TIMESTAMP_WITH_LOCAL_TIME_ZONE)" + s"CAST(TIMESTAMP '$str' AS TIMESTAMP WITH LOCAL TIME ZONE)" } @Test @@ -803,7 +803,7 @@ class TemporalTypesTest extends ExpressionTestBase { cases.foreach { caseExpr => - testSqlNullable(caseExpr) + testSqlApi(caseExpr, "null") } } diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/utils/ExpressionTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/utils/ExpressionTestBase.scala index 02cc054e01c1a95d48b54b8c07fcddb45783d22c..909abcfc6c4d53bd95def10d971e8e8dfe16fc4b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/utils/ExpressionTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/expressions/utils/ExpressionTestBase.scala @@ -215,10 +215,6 @@ abstract class ExpressionTestBase { addTableApiTestExpr(expr, expected) addTableApiTestExpr(exprString, expected) addSqlTestExpr(sqlExpr, expected) - if (expected == nullable) { - testTableNullable(expr, exprString) - testSqlNullable(sqlExpr) - } } def testTableApi( @@ -227,9 +223,6 @@ abstract class ExpressionTestBase { expected: String): Unit = { addTableApiTestExpr(expr, expected) addTableApiTestExpr(exprString, expected) - if (expected == nullable) { - testTableNullable(expr, exprString) - } } private def addTableApiTestExpr(tableApiString: String, expected: String): Unit = { @@ -240,29 +233,14 @@ abstract class ExpressionTestBase { // create RelNode from Table API expression val relNode = relBuilder .queryOperation(tEnv.scan(tableName).select(tableApiExpr).getQueryOperation).build() - addTestExpr(relNode, expected, tableApiExpr.asSummaryString()) - } - def testSqlNullable(nullUdf: String): Unit = { - addSqlTestExpr( - s"CASE WHEN ($nullUdf) is null THEN '$nullable' ELSE '$notNullable' END", nullable) + addTestExpr(relNode, expected, tableApiExpr.asSummaryString()) } def testSqlApi( sqlExpr: String, expected: String): Unit = { addSqlTestExpr(sqlExpr, expected) - if (expected == nullable) { - testSqlNullable(sqlExpr) - } - } - - def testTableNullable(nullExpr: Expression, nullExprString: String): Unit = { - val retExpr = ExpressionBuilder.ifThenElse(nullExpr.isNull, nullable, notNullable) - addTableApiTestExpr(retExpr, nullable) - val retStrExpr = ifThenElse( - ExpressionParser.parseExpression(nullExprString).isNull, nullable, notNullable) - addTableApiTestExpr(retStrExpr, nullable) } def testData: Row diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.scala index 79cc13b7d355463a89881fd2a72e3b4405d4f147..54541ba3fad1c5d992b2e301f068e89df0c22ae3 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/ExpressionReductionRulesTest.scala @@ -20,6 +20,7 @@ package org.apache.flink.table.planner.plan.rules.logical import org.apache.flink.api.scala._ import org.apache.flink.table.api.scala._ +import org.apache.flink.table.functions.{FunctionLanguage, ScalarFunction} import org.apache.flink.table.planner.expressions.utils.{Func1, RichFunc1} import org.apache.flink.table.planner.utils.TableTestBase @@ -46,4 +47,19 @@ class ExpressionReductionRulesTest extends TableTestBase { util.verifyPlan("SELECT myUdf(1) FROM MyTable") } + @Test + def testExpressionReductionWithPythonUDF(): Unit = { + util.addFunction("PyUdf", DeterministicPythonFunc) + util.addFunction("MyUdf", Func1) + util.verifyPlan("SELECT PyUdf(), MyUdf(1) FROM MyTable") + } +} + +object DeterministicPythonFunc extends ScalarFunction { + + override def getLanguage: FunctionLanguage = FunctionLanguage.PYTHON + + def eval(): Long = 1 + + override def isDeterministic = true } diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRuleTest.scala index 0c3d30cc3097231e74a14d168946229f0f82068a..cbf90771899798cef49519a8cf8667d62971f3d3 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkAggregateJoinTransposeRuleTest.scala @@ -56,8 +56,8 @@ class FlinkAggregateJoinTransposeRuleTest extends TableTestBase { .setHepMatchOrder(HepMatchOrder.BOTTOM_UP) .add(RuleSets.ofList( AggregateReduceGroupingRule.INSTANCE, - FlinkFilterJoinRule.FILTER_ON_JOIN, - FlinkFilterJoinRule.JOIN, + FilterJoinRule.FILTER_ON_JOIN, + FilterJoinRule.JOIN, FilterAggregateTransposeRule.INSTANCE, FilterProjectTransposeRule.INSTANCE, FilterMergeRule.INSTANCE, diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.scala index 426740aab9a1835a7445aa8111b87c13a2bcb0c6..8b229100d179d96224caa918c31b7b2d6d48bbb3 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkFilterJoinRuleTest.scala @@ -23,12 +23,12 @@ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} import org.apache.calcite.plan.hep.HepMatchOrder -import org.apache.calcite.rel.rules.FilterProjectTransposeRule +import org.apache.calcite.rel.rules.{FilterJoinRule, FilterProjectTransposeRule} import org.apache.calcite.tools.RuleSets import org.junit.{Before, Test} /** - * Test for [[FlinkFilterJoinRule]]. + * Tests for [[org.apache.calcite.rel.rules.FilterJoinRule]]. */ class FlinkFilterJoinRuleTest extends TableTestBase { private val util = batchTestUtil() @@ -44,8 +44,8 @@ class FlinkFilterJoinRuleTest extends TableTestBase { .setHepMatchOrder(HepMatchOrder.BOTTOM_UP) .add(RuleSets.ofList( FilterProjectTransposeRule.INSTANCE, - FlinkFilterJoinRule.FILTER_ON_JOIN, - FlinkFilterJoinRule.JOIN)) + FilterJoinRule.FILTER_ON_JOIN, + FilterJoinRule.JOIN)) .build() ) diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRuleTest.scala index 43647045d0a7cca3ce5175943034be645ca9c36f..dca814ce33c6a793ea0e539d594aa3387ffdc445 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/FlinkJoinToMultiJoinRuleTest.scala @@ -23,12 +23,12 @@ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} import org.apache.calcite.plan.hep.HepMatchOrder -import org.apache.calcite.rel.rules.ProjectMultiJoinMergeRule +import org.apache.calcite.rel.rules.{JoinToMultiJoinRule, ProjectMultiJoinMergeRule} import org.apache.calcite.tools.RuleSets import org.junit.{Before, Test} /** - * Test for [[FlinkJoinToMultiJoinRule]]. + * Tests for [[org.apache.calcite.rel.rules.JoinToMultiJoinRule]]. */ class FlinkJoinToMultiJoinRuleTest extends TableTestBase { private val util = batchTestUtil() @@ -43,7 +43,7 @@ class FlinkJoinToMultiJoinRuleTest extends TableTestBase { .setHepRulesExecutionType(HEP_RULES_EXECUTION_TYPE.RULE_COLLECTION) .setHepMatchOrder(HepMatchOrder.BOTTOM_UP) .add(RuleSets.ofList( - FlinkJoinToMultiJoinRule.INSTANCE, + JoinToMultiJoinRule.INSTANCE, ProjectMultiJoinMergeRule.INSTANCE)) .build() ) diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala index efdd9b2516d98e7b35c0b238e5e7b861dc07dd75..0da08f036611f9588e2b8edcadb221cc98ad9dbb 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/JoinDependentConditionDerivationRuleTest.scala @@ -24,6 +24,7 @@ import org.apache.flink.table.planner.plan.optimize.program.{FlinkBatchProgram, import org.apache.flink.table.planner.utils.{TableConfigUtils, TableTestBase} import org.apache.calcite.plan.hep.HepMatchOrder +import org.apache.calcite.rel.rules.FilterJoinRule import org.apache.calcite.tools.RuleSets import org.junit.{Before, Test} @@ -45,8 +46,8 @@ class JoinDependentConditionDerivationRuleTest extends TableTestBase { .setHepRulesExecutionType(HEP_RULES_EXECUTION_TYPE.RULE_SEQUENCE) .setHepMatchOrder(HepMatchOrder.BOTTOM_UP) .add(RuleSets.ofList( - FlinkFilterJoinRule.FILTER_ON_JOIN, - FlinkFilterJoinRule.JOIN, + FilterJoinRule.FILTER_ON_JOIN, + FilterJoinRule.JOIN, JoinDependentConditionDerivationRule.INSTANCE)) .build() ) diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMultiJoinConditionRuleTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMultiJoinConditionRuleTest.scala index 5e8e8fef15e366d587870c035d91fcd99f43e49a..3e0e6380b6d6bba2d1a1cf5f3823868fac9c2d1b 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMultiJoinConditionRuleTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/rules/logical/RewriteMultiJoinConditionRuleTest.scala @@ -23,7 +23,7 @@ import org.apache.flink.table.planner.plan.optimize.program.{BatchOptimizeContex import org.apache.flink.table.planner.utils.TableTestBase import org.apache.calcite.plan.hep.HepMatchOrder -import org.apache.calcite.rel.rules.{FilterMultiJoinMergeRule, JoinToMultiJoinRule, ProjectMultiJoinMergeRule} +import org.apache.calcite.rel.rules.{FilterJoinRule, FilterMultiJoinMergeRule, JoinToMultiJoinRule, ProjectMultiJoinMergeRule} import org.apache.calcite.tools.RuleSets import org.junit.{Before, Test} @@ -43,8 +43,8 @@ class RewriteMultiJoinConditionRuleTest extends TableTestBase { .setHepRulesExecutionType(HEP_RULES_EXECUTION_TYPE.RULE_COLLECTION) .setHepMatchOrder(HepMatchOrder.BOTTOM_UP) .add(RuleSets.ofList( - FlinkFilterJoinRule.FILTER_ON_JOIN, - FlinkFilterJoinRule.JOIN)) + FilterJoinRule.FILTER_ON_JOIN, + FilterJoinRule.JOIN)) .build(), "push filter into join") .addProgram(FlinkHepRuleSetProgramBuilder.newBuilder .setHepRulesExecutionType(HEP_RULES_EXECUTION_TYPE.RULE_COLLECTION) diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractorTest.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractorTest.scala index a49f06fa6f35a5acc9833498aed0cca96a4ba230..f0289e4b1801dfcc6ae290fd1a7903592e3b0815 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractorTest.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/plan/utils/RexNodeExtractorTest.scala @@ -696,7 +696,7 @@ class RexNodeExtractorTest extends RexNodeTestBase { @Test def testExtractWithUdf(): Unit = { - functionCatalog.registerScalarFunction("myUdf", Func1) + functionCatalog.registerTempSystemScalarFunction("myUdf", Func1) // amount val t0 = rexBuilder.makeInputRef(allFieldTypes.get(2), 2) // my_udf(amount) diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/DecimalITCase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/DecimalITCase.scala index 88a4dd32547a51331ee3d0b05fd76bfe97a77721..c1c49be951a5f7f9d1510a4a68f1b75853c13fb6 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/DecimalITCase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/runtime/batch/sql/DecimalITCase.scala @@ -141,345 +141,6 @@ class DecimalITCase extends BatchTestBase { // // tests // - - @Test - def testDataSource(): Unit = { - - // the most basic case - checkQuery1( - Seq(DECIMAL(10, 0), DECIMAL(7, 2)), - s1r(d"123", d"123.45"), - "select * from Table1", - Seq(DECIMAL(10, 0), DECIMAL(7, 2)), - s1r(d"123", d"123.45")) - - // data from source are rounded to their declared scale before entering next step - checkQuery1( - Seq(DECIMAL(7, 2)), - s1r(d"100.004"), - "select f0, f0+f0 from Table1", // 100.00+100.00 - Seq(DECIMAL(7, 2), DECIMAL(8, 2)), - s1r(d"100.00", d"200.00")) // not 200.008=>200.01 - - // trailing zeros are padded to the scale - checkQuery1( - Seq(DECIMAL(7, 2)), - s1r(d"100.1"), - "select f0, f0+f0 from Table1", // 100.00+100.00 - Seq(DECIMAL(7, 2), DECIMAL(8, 2)), - s1r(d"100.10", d"200.20")) - - // source data is within precision after rounding - checkQuery1( - Seq(DECIMAL(5, 2)), - s1r(d"100.0040"), // p=7 => rounding => p=5 - "select f0, f0+f0 from Table1", - Seq(DECIMAL(5, 2), DECIMAL(6, 2)), - s1r(d"100.00", d"200.00")) - - // source data overflows over precision (after rounding) - checkQuery1( - Seq(DECIMAL(2, 0)), - s1r(d"123"), - "select * from Table1", - Seq(DECIMAL(2, 0)), - s1r(null)) - - checkQuery1( - Seq(DECIMAL(4, 2)), - s1r(d"123.0000"), - "select * from Table1", - Seq(DECIMAL(4, 2)), - s1r(null)) - } - - @Test - def testLiterals(): Unit = { - - checkQuery1( - Seq(DECIMAL(1,0)), - s1r(d"0"), - "select 12, 12.3, 12.34 from Table1", - Seq(INT, DECIMAL(3, 1), DECIMAL(4, 2)), - s1r(12, d"12.3", d"12.34")) - - checkQuery1( - Seq(DECIMAL(1,0)), - s1r(d"0"), - "select 123456789012345678901234567890.12345678 from Table1", - Seq(DECIMAL(38, 8)), - s1r(d"123456789012345678901234567890.12345678")) - - expectOverflow(()=> - checkQuery1( - Seq(DECIMAL(1,0)), - s1r(d"0"), - "select 123456789012345678901234567890.123456789 from Table1", - Seq(DECIMAL(38, 9)), - s1r(d"123456789012345678901234567890.123456789"))) - } - - @Test - def testUnaryPlusMinus(): Unit = { - - checkQuery1( - Seq(DECIMAL(10, 0), DECIMAL(7, 2)), - s1r(d"123", d"123.45"), - "select +f0, -f1, -((+f0)-(-f1)) from Table1", - Seq(DECIMAL(10, 0), DECIMAL(7, 2), DECIMAL(13,2)), - s1r(d"123", d"-123.45", d"-246.45")) - } - - @Test - def testPlusMinus(): Unit = { - - // see calcite ReturnTypes.DECIMAL_SUM - // s = max(s1,s2), p-s = max(p1-s1, p2-s2) + 1 - // p then is capped at 38 - checkQuery1( - Seq(DECIMAL(10, 2), DECIMAL(10, 4)), - s1r(d"100.12", d"200.1234"), - "select f0+f1, f0-f1 from Table1", - Seq(DECIMAL(13, 4), DECIMAL(13, 4)), - s1r(d"300.2434", d"-100.0034")) - - // INT => DECIMAL(10,0) - // approximate + exact => approximate - checkQuery1( - Seq(DECIMAL(10, 2), INT, DOUBLE), - s1r(d"100.00", 200, 3.14), - "select f0+f1, f1+f0, f0+f2, f2+f0 from Table1", - Seq(DECIMAL(13, 2), DECIMAL(13, 2), DOUBLE, DOUBLE), - s1r(d"300.00", d"300.00", d"103.14", d"103.14")) - - // our result type precision is capped at 38 - // SQL2003 $6.26 -- result scale is dictated as max(s1,s2). no approximation allowed. - // calcite -- scale is not reduced; integral part may be reduced. overflow may occur - // (38,10)+(38,28)=>(57,28)=>(38,28) - // T-SQL -- scale may be reduced to keep the integral part. approximation may occur - // (38,10)+(38,28)=>(57,28)=>(38,9) - checkQuery1( - Seq(DECIMAL(38, 10), DECIMAL(38, 28)), - s1r(d"100.0123456789", d"200.0123456789012345678901234567"), - "select f0+f1, f0-f1 from Table1", - Seq(DECIMAL(38, 28), DECIMAL(38, 28)), - s1r(d"300.0246913578012345678901234567", d"-100.0000000000012345678901234567")) - - checkQuery1( - Seq(DECIMAL(38, 10), DECIMAL(38, 28)), - s1r(d"1e10", d"0"), - "select f1+f0, f1-f0 from Table1", - Seq(DECIMAL(38, 28), DECIMAL(38, 28)), // 10 digits integral part - s1r(null, null)) - - checkQuery1( - Seq(DECIMAL(38, 0)), - s1r(d"5e37"), - "select f0+f0 from Table1", - Seq(DECIMAL(38, 0)), - s1r(null)) // requires 39 digits - - checkQuery1( - Seq(DECIMAL(38, 0), DECIMAL(38, 0)), - s1r(d"5e37", d"5e37"), - "select f0+f0-f1 from Table1", // overflows in subexpression - Seq(DECIMAL(38, 0)), - s1r(null)) - } - - @Test - def testMultiply(): Unit = { - - // see calcite ReturnTypes.DECIMAL_PRODUCT - // s = s1+s2, p = p1+p2 - // both p&s are capped at 38 - // if s>38, result is rounded to s=38, and the integral part can only be zero - checkQuery1( - Seq(DECIMAL(5, 2), DECIMAL(10, 4)), - s1r(d"1.00", d"2.0000"), - "select f0*f0, f0*f1 from Table1", - Seq(DECIMAL(10, 4), DECIMAL(15, 6)), - s1r(d"1.0000", d"2.000000")) - - // INT => DECIMAL(10,0) - // approximate * exact => approximate - checkQuery1( - Seq(DECIMAL(10, 2), INT, DOUBLE), - s1r(d"1.00", 200, 3.14), - "select f0*f1, f1*f0, f0*f2, f2*f0 from Table1", - Seq(DECIMAL(20, 2), DECIMAL(20, 2), DOUBLE, DOUBLE), - s1r(d"200.00", d"200.00", 3.14, 3.14)) - - // precision is capped at 38; scale will not be reduced (unless over 38) - // similar to plus&minus, and calcite behavior is different from T-SQL. - checkQuery1( - Seq(DECIMAL(30, 6), DECIMAL(30, 10)), - s1r(d"1", d"2"), - "select f0*f0, f0*f1 from Table1", - Seq(DECIMAL(38, 12), DECIMAL(38, 16)), - s1r(d"1${12}", d"2${16}")) - - checkQuery1( - Seq(DECIMAL(30, 20)), - s1r(d"0.1"), - "select f0*f0 from Table1", - Seq(DECIMAL(38, 38)), // (60,40)=>(38,38) - s1r(d"0.01${38}")) - - // scalastyle:off - // we don't have this ridiculous behavior: - // https://blogs.msdn.microsoft.com/sqlprogrammability/2006/03/29/multiplication-and-division-with-numerics/ - // scalastyle:on - checkQuery1( - Seq(DECIMAL(38, 10), DECIMAL(38, 10)), - s1r(d"0.0000006", d"1.0"), - "select f0*f1 from Table1", - Seq(DECIMAL(38, 20)), - s1r(d"0.0000006${20}")) - - // result overflow - checkQuery1( - Seq(DECIMAL(38, 0)), - s1r(d"1e19"), - "select f0*f0 from Table1", - Seq(DECIMAL(38, 0)), - s1r(null)) - - checkQuery1( - Seq(DECIMAL(30, 20)), - s1r(d"1.0"), - "select f0*f0 from Table1", - Seq(DECIMAL(38, 38)), // (60,40)=>(38,38), no space for integral part - s1r(null)) - } - - @Test - def testDivide(): Unit = { - - // the default impl of Calcite apparently borrows from T-SQL, but differs in details. - // Flink overrides it to follow T-SQL exactly. See FlinkTypeFactory.createDecimalQuotient() - checkQuery1( // test 1/3 in different scales - Seq(DECIMAL(20, 2), DECIMAL(2, 1), DECIMAL(4, 3), DECIMAL(20, 10), DECIMAL(20, 16)), - s1r(d"1.00", d"3", d"3", d"3", d"3"), - "select f0/f1, f0/f2, f0/f3, f0/f4 from Table1", - Seq(DECIMAL(25, 6), DECIMAL(28, 7), DECIMAL(38, 10), DECIMAL(38, 6)), - s1r(d"0.333333", d"0.3333333", d"0.3333333333", d"0.333333")) - - // INT => DECIMAL(10,0) - // approximate / exact => approximate - checkQuery1( - Seq(DECIMAL(10, 2), INT, DOUBLE), - s1r(d"1.00", 2, 3.0), - "select f0/f1, f1/f0, f0/f2, f2/f0 from Table1", - Seq(DECIMAL(21, 13), DECIMAL(23, 11), DOUBLE, DOUBLE), - s1r(d"0.5${13}", d"2${11}", 1.0/3.0, 3.0/1.0)) - - // result overflow, because result type integral part is reduced - checkQuery1( - Seq(DECIMAL(30, 0), DECIMAL(30, 20)), - s1r(d"1e20", d"1e-15"), - "select f0/f1 from Table1", - Seq(DECIMAL(38, 6)), - s1r(null)) - } - - @Test - def testMod(): Unit = { - - // MOD(Exact1, Exact2) => Exact2 - checkQuery1( - Seq(DECIMAL(10, 2), DECIMAL(10, 4), INT), - s1r(d"3.00", d"5.0000", 7), - "select mod(f0,f1), mod(f1,f0), mod(f0,f2), mod(f2,f0) from Table1", - Seq(DECIMAL(10, 4), DECIMAL(10, 2), INT, DECIMAL(10, 2)), - s1r(d"3.0000", d"2.00", 3, d"1.00")) - - // signs. consistent with Java's % operator. - checkQuery1( - Seq(DECIMAL(1, 0), DECIMAL(1, 0)), - s1r(d"3", d"5"), - "select mod(f0,f1), mod(-f0,f1), mod(f0,-f1), mod(-f0,-f1) from Table1", - Seq(DECIMAL(1, 0), DECIMAL(1, 0), DECIMAL(1, 0), DECIMAL(1, 0)), - s1r(3%5, (-3)%5, 3%(-5), (-3)%(-5))) - - // rounding in case s1>s2. note that SQL2003 requires s1=s2=0. - // (In T-SQL, s2 is expanded to s1, so that there's no rounding.) - checkQuery1( - Seq(DECIMAL(10, 4), DECIMAL(10, 2)), - s1r(d"3.1234", d"5"), - "select mod(f0,f1) from Table1", - Seq(DECIMAL(10, 2)), - s1r(d"3.12")) - } - - @Test // functions that treat Decimal as exact value - def testExactFunctions(): Unit = { - checkQuery1( - Seq(DECIMAL(10, 2), DECIMAL(10, 2)), - s1r(d"3.14", d"2.17"), - "select if(f0>f1, f0, f1) from Table1", - Seq(DECIMAL(10, 2)), - s1r(d"3.14")) - - checkQuery1( - Seq(DECIMAL(10, 2)), - s1r(d"3.14"), - "select abs(f0), abs(-f0) from Table1", - Seq(DECIMAL(10, 2), DECIMAL(10, 2)), - s1r(d"3.14", d"3.14")) - - checkQuery1( - Seq(DECIMAL(10, 2)), - s1r(d"3.14"), - "select floor(f0), ceil(f0) from Table1", - Seq(DECIMAL(10, 0), DECIMAL(10, 0)), - s1r(d"3", d"4")) - - // calcite: SIGN(Decimal(p,s))=>Decimal(p,s) - checkQuery1( - Seq(DECIMAL(10, 2)), - s1r(d"3.14"), - "select sign(f0), sign(-f0), sign(f0-f0) from Table1", - Seq(DECIMAL(10, 2), DECIMAL(10, 2), DECIMAL(11, 2)), - s1r(d"1.00", d"-1.00", d"0.00")) - - // ROUND(Decimal(p,s)[,INT]) - checkQuery1( - Seq(DECIMAL(10, 3)), - s1r(d"646.646"), - "select round(f0), round(f0, 0) from Table1", - Seq(DECIMAL(8, 0), DECIMAL(8, 0)), - s1r(d"647", d"647")) - - checkQuery1( - Seq(DECIMAL(10, 3)), - s1r(d"646.646"), - "select round(f0,1), round(f0,2), round(f0,3), round(f0,4) from Table1", - Seq(DECIMAL(9, 1), DECIMAL(10, 2), DECIMAL(10, 3), DECIMAL(10, 3)), - s1r(d"646.6", d"646.65", d"646.646", d"646.646")) - - checkQuery1( - Seq(DECIMAL(10, 3)), - s1r(d"646.646"), - "select round(f0,-1), round(f0,-2), round(f0,-3), round(f0,-4) from Table1", - Seq(DECIMAL(8, 0), DECIMAL(8, 0), DECIMAL(8, 0), DECIMAL(8, 0)), - s1r(d"650", d"600", d"1000", d"0")) - - checkQuery1( - Seq(DECIMAL(4, 2)), - s1r(d"99.99"), - "select round(f0,1), round(-f0,1), round(f0,-1), round(-f0,-1) from Table1", - Seq(DECIMAL(4, 1), DECIMAL(4, 1), DECIMAL(3, 0), DECIMAL(3, 0)), - s1r(d"100.0", d"-100.0", d"100", d"-100")) - - checkQuery1( - Seq(DECIMAL(38, 0)), - s1r(d"1E38".subtract(d"1")), - "select round(f0,-1) from Table1", - Seq(DECIMAL(38, 0)), - s1r(null)) - } - @Test // functions e.g. sin() that treat Decimal as double def testApproximateFunctions(): Unit = { @@ -585,173 +246,6 @@ class DecimalITCase extends BatchTestBase { s1r(d"10.000", d"90.000", 81L)) } - @Test - def testCaseWhen(): Unit = { - - // result type: SQL2003 $9.23, calcite RelDataTypeFactory.leastRestrictive() - checkQuery1( - Seq(DECIMAL(8, 4), DECIMAL(10, 2)), - s1r(d"0.0001", d"0.01"), - "select case f0 when 0 then f0 else f1 end from Table1", - Seq(DECIMAL(12, 4)), - s1r(d"0.0100")) - - checkQuery1( - Seq(DECIMAL(8, 4), INT), - s1r(d"0.0001", 1), - "select case f0 when 0 then f0 else f1 end from Table1", - Seq(DECIMAL(14, 4)), - s1r(d"1.0000")) - - checkQuery1( - Seq(DECIMAL(8, 4), DOUBLE), - s1r(d"0.0001", 3.14), - "select case f0 when 0 then f1 else f0 end from Table1", - Seq(DOUBLE), - s1r(d"0.0001".doubleValue())) - } - - @Test - def testCast(): Unit = { - - // String, numeric/Decimal => Decimal - checkQuery1( - Seq(DECIMAL(8, 2), INT, DOUBLE, STRING), - s1r(d"3.14", 3, 3.14, "3.14"), - "select cast(f0 as Decimal(8,4)), cast(f1 as Decimal(8,4)), " + - "cast(f2 as Decimal(8,4)), cast(f3 as Decimal(8,4)) from Table1", - Seq(DECIMAL(8, 4), DECIMAL(8, 4), DECIMAL(8, 4), DECIMAL(8, 4)), - s1r(d"3.1400", d"3.0000", d"3.1400", d"3.1400")) - - // round up - checkQuery1( - Seq(DECIMAL(8, 2), DOUBLE, STRING), - s1r(d"3.15", 3.15, "3.15"), - "select cast(f0 as Decimal(8,1)), cast(f1 as Decimal(8,1)), " + - "cast(f2 as Decimal(8,1)) from Table1", - Seq(DECIMAL(8, 1), DECIMAL(8, 1), DECIMAL(8, 1)), - s1r(d"3.2", d"3.2", d"3.2")) - - checkQuery1( - Seq(DECIMAL(4, 2)), - s1r(d"13.14"), - "select cast(f0 as Decimal(3,2)) from Table1", - Seq(DECIMAL(3, 2)), - s1r(null)) - - checkQuery1( - Seq(STRING), - s1r("13.14"), - "select cast(f0 as Decimal(3,2)) from Table1", - Seq(DECIMAL(3, 2)), - s1r(null)) - - // Decimal => String, numeric - checkQuery1( - Seq(DECIMAL(4, 2)), - s1r(d"1.99"), - "select cast(f0 as VARCHAR(64)), cast(f0 as DOUBLE), cast(f0 as INT) from Table1", - Seq(STRING, DOUBLE, INT), - s1r("1.99", 1.99, 1)) - - checkQuery1( - Seq(DECIMAL(10, 0), DECIMAL(10, 0)), - s1r(d"-2147483648", d"2147483647"), - "select cast(f0 as INT), cast(f1 as INT) from Table1", - Seq(INT, INT), - s1r(-2147483648, 2147483647)) - } - - @Test - def testEquality(): Unit = { - - // expressions that test equality. - // =, CASE, NULLIF, IN, IS DISTINCT FROM - - checkQuery1( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - "select f0=f1, f0=f2, f0=f3, f1=f0, f2=f0, f3=f0 from Table1", - Seq(BOOL, BOOL, BOOL, BOOL, BOOL, BOOL), - s1r(true, true, true, true, true, true)) - - checkQuery1( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - "select f0 IN(f1), f0 IN(f2), f0 IN(f3), " + - "f1 IN(f0), f2 IN(f0), f3 IN(f0) from Table1", - Seq(BOOL, BOOL, BOOL, BOOL, BOOL, BOOL), - s1r(true, true, true, true, true, true)) - - checkQuery1( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - "select " + - "f0 IS DISTINCT FROM f1, f1 IS DISTINCT FROM f0, " + - "f0 IS DISTINCT FROM f2, f2 IS DISTINCT FROM f0, " + - "f0 IS DISTINCT FROM f3, f3 IS DISTINCT FROM f0 from Table1", - Seq(BOOL, BOOL, BOOL, BOOL, BOOL, BOOL), - s1r(false, false, false, false, false, false)) - - checkQuery1( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - "select NULLIF(f0,f1), NULLIF(f0,f2), NULLIF(f0,f3)," + - "NULLIF(f1,f0), NULLIF(f2,f0), NULLIF(f3,f0) from Table1", - Seq(DECIMAL(8, 2), DECIMAL(8, 2), DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(null, null, null, null, null, null)) - - checkQuery1( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - "select " + - "case f0 when f1 then 1 else 0 end, case f1 when f0 then 1 else 0 end, " + - "case f0 when f2 then 1 else 0 end, case f2 when f0 then 1 else 0 end, " + - "case f0 when f3 then 1 else 0 end, case f3 when f0 then 1 else 0 end from Table1", - Seq(INT, INT, INT, INT, INT, INT), - s1r(1, 1, 1, 1, 1, 1)) - } - - @Test - def testComparison(): Unit = { - - checkQuery1( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - "select f0 table.select('*), - Seq(DECIMAL(10, 0), DECIMAL(7, 2)), - s1r(d"123", d"123.45")) - - // data from source are rounded to their declared scale before entering next step - checkQuery( - Seq(DECIMAL(7, 2)), - s1r(d"100.004"), - table => table.select('f0, 'f0 + 'f0), // 100.00+100.00 - Seq(DECIMAL(7, 2), DECIMAL(8, 2)), - s1r(d"100.00", d"200.00")) // not 200.008=>200.01 - - // trailing zeros are padded to the scale - checkQuery( - Seq(DECIMAL(7, 2)), - s1r(d"100.1"), - table => table.select('f0, 'f0 + 'f0), // 100.00+100.00 - Seq(DECIMAL(7, 2), DECIMAL(8, 2)), - s1r(d"100.10", d"200.20")) - - // source data is within precision after rounding - checkQuery( - Seq(DECIMAL(5, 2)), - s1r(d"100.0040"), // p=7 => rounding => p=5 - table => table.select('f0, 'f0 + 'f0), // 100.00+100.00 - Seq(DECIMAL(5, 2), DECIMAL(6, 2)), - s1r(d"100.00", d"200.00")) - - // source data overflows over precision (after rounding) - checkQuery( - Seq(DECIMAL(2, 0)), - s1r(d"123"), - table => table.select('*), - Seq(DECIMAL(2, 0)), - s1r(null)) - - checkQuery( - Seq(DECIMAL(4, 2)), - s1r(d"123.0000"), - table => table.select('*), - Seq(DECIMAL(4, 2)), - s1r(null)) - } - - @Test - def testUnaryPlusMinus(): Unit = { - - checkQuery( - Seq(DECIMAL(10, 0), DECIMAL(7, 2)), - s1r(d"123", d"123.45"), - table => table.select( + 'f0, - 'f1, - (( + 'f0) - ( - 'f1))), - Seq(DECIMAL(10, 0), DECIMAL(7, 2), DECIMAL(13, 2)), - s1r(d"123", d"-123.45", d"-246.45")) - } - - @Test - def testPlusMinus(): Unit = { - - // see calcite ReturnTypes.DECIMAL_SUM - // s = max(s1,s2), p-s = max(p1-s1, p2-s2) + 1 - // p then is capped at 38 - checkQuery( - Seq(DECIMAL(10, 2), DECIMAL(10, 4)), - s1r(d"100.12", d"200.1234"), - table => table.select('f0 + 'f1, 'f0 - 'f1), - Seq(DECIMAL(13, 4), DECIMAL(13, 4)), - s1r(d"300.2434", d"-100.0034")) - - // INT => DECIMAL(10,0) - // approximate + exact => approximate - checkQuery( - Seq(DECIMAL(10, 2), INT, DOUBLE), - s1r(d"100.00", 200, 3.14), - table => table.select('f0 + 'f1, 'f1 + 'f0, 'f0 + 'f2, 'f2 + 'f0), - Seq(DECIMAL(13, 2), DECIMAL(13, 2), DOUBLE, DOUBLE), - s1r(d"300.00", d"300.00", d"103.14", d"103.14")) - - // our result type precision is capped at 38 - // SQL2003 $6.26 -- result scale is dictated as max(s1,s2). no approximation allowed. - // calcite -- scale is not reduced; integral part may be reduced. overflow may occur - // (38,10)+(38,28)=>(57,28)=>(38,28) - // T-SQL -- scale may be reduced to keep the integral part. approximation may occur - // (38,10)+(38,28)=>(57,28)=>(38,9) - checkQuery( - Seq(DECIMAL(38, 10), DECIMAL(38, 28)), - s1r(d"100.0123456789", d"200.0123456789012345678901234567"), - table => table.select('f0 + 'f1, 'f0 - 'f1), - Seq(DECIMAL(38, 28), DECIMAL(38, 28)), - s1r(d"300.0246913578012345678901234567", d"-100.0000000000012345678901234567")) - - checkQuery( - Seq(DECIMAL(38, 10), DECIMAL(38, 28)), - s1r(d"1e10", d"0"), - table => table.select('f1 + 'f0, 'f1 - 'f0 ), - Seq(DECIMAL(38, 28), DECIMAL(38, 28)), // 10 digits integral part - s1r(null, null)) - - checkQuery( - Seq(DECIMAL(38, 0)), - s1r(d"5e37"), - table => table.select('f0 + 'f0 ), - Seq(DECIMAL(38, 0)), - s1r(null)) // requires 39 digits - - checkQuery( - Seq(DECIMAL(38, 0), DECIMAL(38, 0)), - s1r(d"5e37", d"5e37"), - table => table.select('f0 + 'f0 -'f1 ), // overflows in subexpression - Seq(DECIMAL(38, 0)), - s1r(null)) - } - - @Test - def testMultiply(): Unit = { - - // see calcite ReturnTypes.DECIMAL_PRODUCT - // s = s1+s2, p = p1+p2 - // both p&s are capped at 38 - // if s>38, result is rounded to s=38, and the integral part can only be zero - checkQuery( - Seq(DECIMAL(5, 2), DECIMAL(10, 4)), - s1r(d"1.00", d"2.0000"), - table => table.select('f0*'f0, 'f0*'f1 ), - Seq(DECIMAL(10, 4), DECIMAL(15, 6)), - s1r(d"1.0000", d"2.000000")) - - // INT => DECIMAL(10,0) - // approximate * exact => approximate - checkQuery( - Seq(DECIMAL(10, 2), INT, DOUBLE), - s1r(d"1.00", 200, 3.14), - table => table.select('f0*'f1, 'f1*'f0, 'f0*'f2, 'f2*'f0 ), - Seq(DECIMAL(20, 2), DECIMAL(20, 2), DOUBLE, DOUBLE), - s1r(d"200.00", d"200.00", 3.14, 3.14)) - - // precision is capped at 38; scale will not be reduced (unless over 38) - // similar to plus&minus, and calcite behavior is different from T-SQL. - checkQuery( - Seq(DECIMAL(30, 6), DECIMAL(30, 10)), - s1r(d"1", d"2"), - table => table.select('f0*'f0, 'f0*'f1 ), - Seq(DECIMAL(38, 12), DECIMAL(38, 16)), - s1r(d"1${12}", d"2${16}")) - - checkQuery( - Seq(DECIMAL(30, 20)), - s1r(d"0.1"), - table => table.select('f0*'f0 ), - Seq(DECIMAL(38, 38)), // (60,40)=>(38,38) - s1r(d"0.01${38}")) - - // scalastyle:off - // we don't have this ridiculous behavior: - // https://blogs.msdn.microsoft - // .com/sqlprogrammability/2006/03/29/multiplication-and-division-with-numerics/ - // scalastyle:on - checkQuery( - Seq(DECIMAL(38, 10), DECIMAL(38, 10)), - s1r(d"0.0000006", d"1.0"), - table => table.select('f0*'f1 ), - Seq(DECIMAL(38, 20)), - s1r(d"0.0000006${20}")) - - // result overflow - checkQuery( - Seq(DECIMAL(38, 0)), - s1r(d"1e19"), - table => table.select('f0*'f0 ), - Seq(DECIMAL(38, 0)), - s1r(null)) - - checkQuery( - Seq(DECIMAL(30, 20)), - s1r(d"1.0"), - table => table.select('f0*'f0 ), - Seq(DECIMAL(38, 38)), // (60,40)=>(38,38), no space for integral part - s1r(null)) - } - - @Test - def testDivide(): Unit = { - -// // the default impl of Calcite apparently borrows from T-SQL, but differs in details. -// // Flink overrides it to follow T-SQL exactly. See FlinkTypeFactory.createDecimalQuotient() - checkQuery( // test 1/3 in different scales - Seq(DECIMAL(20, 2), DECIMAL(2, 1), DECIMAL(4, 3), DECIMAL(20, 10), DECIMAL(20, 16)), - s1r(d"1.00", d"3", d"3", d"3", d"3"), - table => table.select('f0/'f1, 'f0/'f2, 'f0/'f3, 'f0/'f4 ), - Seq(DECIMAL(25, 6), DECIMAL(28, 7), DECIMAL(38, 10), DECIMAL(38, 6)), - s1r(d"0.333333", d"0.3333333", d"0.3333333333", d"0.333333")) - - // INT => DECIMAL(10,0) - // approximate / exact => approximate - checkQuery( - Seq(DECIMAL(10, 2), INT, DOUBLE), - s1r(d"1.00", 2, 3.0), - table => table.select('f0/'f1, 'f1/'f0, 'f0/'f2, 'f2/'f0 ), - Seq(DECIMAL(21, 13), DECIMAL(23, 11), DOUBLE, DOUBLE), - s1r(d"0.5${13}", d"2${11}", 1.0 / 3.0, 3.0 / 1.0)) - - // result overflow, because result type integral part is reduced - checkQuery( - Seq(DECIMAL(30, 0), DECIMAL(30, 20)), - s1r(d"1e20", d"1e-15"), - table => table.select('f0/'f1 ), - Seq(DECIMAL(38, 6)), - s1r(null)) - } - - @Test - def testMod(): Unit = { - // signs. consistent with Java's % operator. - checkQuery( - Seq(DECIMAL(1, 0), DECIMAL(1, 0)), - s1r(d"3", d"5"), - table => table.select('f0 % 'f1, (-'f0) % 'f1,'f0 % (-'f1), (-'f0) % (-'f1)), - Seq(DECIMAL(1, 0), DECIMAL(1, 0), DECIMAL(1, 0), DECIMAL(1, 0)), - s1r(3 % 5, (-3) % 5, 3 % (-5), (-3) % (-5))) - - // rounding in case s1>s2. note that SQL2003 requires s1=s2=0. - // (In T-SQL, s2 is expanded to s1, so that there's no rounding.) - checkQuery( - Seq(DECIMAL(10, 4), DECIMAL(10, 2)), - s1r(d"3.1234", d"5"), - table => table.select('f0 % 'f1), - Seq(DECIMAL(10, 2)), - s1r(d"3.12")) - } - - @Test // functions that treat Decimal as exact value - def testExactFunctions(): Unit = { - checkQuery( - Seq(DECIMAL(10, 2)), - s1r(d"3.14"), - table => table.select('f0.floor, 'f0.ceil), - Seq(DECIMAL(10, 0), DECIMAL(10, 0)), - s1r(d"3", d"4")) - - // calcite: SIGN(Decimal(p,s))=>Decimal(p,s) - checkQuery( - Seq(DECIMAL(10, 2)), - s1r(d"3.14"), - table => table.select('f0.sign, (-'f0).sign, ('f0 - 'f0).sign ), - Seq(DECIMAL(10, 2), DECIMAL(10, 2), DECIMAL(11, 2)), - s1r(d"1.00", d"-1.00", d"0.00")) - - // ROUND(Decimal(p,s)[,INT]) - checkQuery( - Seq(DECIMAL(10, 3)), - s1r(d"646.646"), - table => table.select('f0.round(0), 'f0.round(0)), - Seq(DECIMAL(8, 0), DECIMAL(8, 0)), - s1r(d"647", d"647")) - - checkQuery( - Seq(DECIMAL(10, 3)), - s1r(d"646.646"), - table => table.select('f0.round(1), 'f0.round(2), 'f0.round(3), 'f0.round(4) ), - Seq(DECIMAL(9, 1), DECIMAL(10, 2), DECIMAL(10, 3), DECIMAL(10, 3)), - s1r(d"646.6", d"646.65", d"646.646", d"646.646")) - - checkQuery( - Seq(DECIMAL(10, 3)), - s1r(d"646.646"), - table => table.select('f0.round(-1), 'f0.round(-2), 'f0.round(-3), 'f0.round(-4) ), - Seq(DECIMAL(8, 0), DECIMAL(8, 0), DECIMAL(8, 0), DECIMAL(8, 0)), - s1r(d"650", d"600", d"1000", d"0")) - - checkQuery( - Seq(DECIMAL(4, 2)), - s1r(d"99.99"), - table => table.select('f0.round(1), (-'f0).round(1), 'f0.round(-1), (-'f0).round(-1) ), - Seq(DECIMAL(4, 1), DECIMAL(4, 1), DECIMAL(3, 0), DECIMAL(3, 0)), - s1r(d"100.0", d"-100.0", d"100", d"-100")) - - checkQuery( - Seq(DECIMAL(38, 0)), - s1r(d"1E38".subtract(d"1")), - table => table.select('f0.round(-1) ), - Seq(DECIMAL(38, 0)), - s1r(null)) - } - @Test // functions e.g. sin() that treat Decimal as double def testApproximateFunctions(): Unit = { @@ -488,145 +196,6 @@ class DecimalITCase extends BatchTestBase { s1r(d"10.000", d"90.000", 81L)) } - @Test - def testCast(): Unit = { - - // String, numeric/Decimal => Decimal - checkQuery( - Seq(DECIMAL(8, 2), INT, DOUBLE, STRING), - s1r(d"3.14", 3, 3.14, "3.14"), - table => table.select('f0.cast(DataTypes.DECIMAL(8,4)), - 'f1.cast(DataTypes.DECIMAL(8,4)), - 'f2.cast(DataTypes.DECIMAL(8,4)), - 'f3.cast(DataTypes.DECIMAL(8,4)) ), - Seq(DECIMAL(8, 4), DECIMAL(8, 4), DECIMAL(8, 4), DECIMAL(8, 4)), - s1r(d"3.1400", d"3.0000", d"3.1400", d"3.1400")) - - // round up - checkQuery( - Seq(DECIMAL(8, 2), DOUBLE, STRING), - s1r(d"3.15", 3.15, "3.15"), - table => table.select( - 'f0.cast(DataTypes.DECIMAL(8,1)), - 'f1.cast(DataTypes.DECIMAL(8,1)), - 'f2.cast(DataTypes.DECIMAL(8,1))), - Seq(DECIMAL(8, 1), DECIMAL(8, 1), DECIMAL(8, 1)), - s1r(d"3.2", d"3.2", d"3.2")) - - checkQuery( - Seq(DECIMAL(4, 2)), - s1r(d"13.14"), - table => table.select('f0.cast(DataTypes.DECIMAL(3,2)) ), - Seq(DECIMAL(3, 2)), - s1r(null)) - - checkQuery( - Seq(STRING), - s1r("13.14"), - table => table.select('f0.cast(DataTypes.DECIMAL(3,2)) ), - Seq(DECIMAL(3, 2)), - s1r(null)) - - // Decimal => String, numeric - checkQuery( - Seq(DECIMAL(4, 2)), - s1r(d"1.99"), - table => table.select( - 'f0.cast(DataTypes.VARCHAR(64)), - 'f0.cast(DataTypes.DOUBLE), - 'f0.cast(DataTypes.INT)), - Seq(STRING, DOUBLE, INT), - s1r("1.99", 1.99, 1)) - - checkQuery( - Seq(DECIMAL(10, 0), DECIMAL(10, 0)), - s1r(d"-2147483648", d"2147483647"), - table => table.select('f0.cast(DataTypes.INT), 'f1.cast(DataTypes.INT)), - Seq(INT, INT), - s1r(-2147483648, 2147483647)) - } - - @Test - def testEquality(): Unit = { - - // expressions that test equality. - // =, CASE, NULLIF, IN, IS DISTINCT FROM - - checkQuery( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - table => table.select('f0==='f1, 'f0==='f2, 'f0==='f3, 'f1==='f0, 'f2==='f0, 'f3==='f0 ), - Seq(BOOL, BOOL, BOOL, BOOL, BOOL, BOOL), - s1r(true, true, true, true, true, true)) - - checkQuery( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - table => table.select('f0.in('f1), 'f0.in('f2), 'f0.in('f3), - 'f1.in('f0), 'f2.in('f0), 'f3.in('f0) ), - Seq(BOOL, BOOL, BOOL, BOOL, BOOL, BOOL), - s1r(true, true, true, true, true, true)) - - checkQuery( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - table => table.select( - ('f0 === 'f1) ? (1, 0), - ('f1 === 'f0) ?(1, 0), - ('f0 === 'f2) ? (1, 0), - ('f2 === 'f0) ? (1, 0), - ('f0 === 'f3) ? (1, 0), - ('f3 === 'f0) ? (1, 0)), - Seq(INT, INT, INT, INT, INT, INT), - s1r(1, 1, 1, 1, 1, 1)) - } - - @Test - def testComparison(): Unit = { - - checkQuery( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - table => table.select('f0<'f1, 'f0<'f2, 'f0<'f3, 'f1<'f0, 'f2<'f0, 'f3<'f0 ), - Seq(BOOL, BOOL, BOOL, BOOL, BOOL, BOOL), - s1r(false, false, false, false, false, false)) - - // no overflow during type conversion. - // conceptually both operands are promoted to infinite precision before comparison. - checkQuery( - Seq(DECIMAL(1, 0), DECIMAL(2, 0), INT, DOUBLE), - s1r(d"1", d"99", 99, 99.0), - table => table.select('f0<'f1, 'f0<'f2, 'f0<'f3, 'f1<'f0, 'f2<'f0, 'f3<'f0 ), - Seq(BOOL, BOOL, BOOL, BOOL, BOOL, BOOL), - s1r(true, true, true, false, false, false)) - - checkQuery( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - table => table.select( - 'f0.between('f1, 1), - 'f1.between('f0, 1), - 'f0.between('f2, 1), - 'f2.between('f0, 1), - 'f0.between('f3, 1), - 'f3.between('f0, 1)), - Seq(BOOL, BOOL, BOOL, BOOL, BOOL, BOOL), - s1r(true, true, true, true, true, true)) - - checkQuery( - Seq(DECIMAL(8, 2), DECIMAL(8, 4), INT, DOUBLE), - s1r(d"1", d"1", 1, 1.0), - table => table.select( - 'f0.between(0, 'f1), - 'f1.between(0, 'f0), - 'f0.between(0, 'f2), - 'f2.between(0, 'f0), - 'f0.between(0, 'f3), - 'f3.between(0, 'f0)), - Seq(BOOL, BOOL, BOOL, BOOL, BOOL, BOOL), - s1r(true, true, true, true, true, true)) - } - @Test def testJoin1(): Unit = { tEnv.getConfig.getConfiguration.setString( diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala index 0201d772b668a4c778c8d068f1650bbc2b9d53ef..a452717e0f64e950d9bd3b09f1c4343d628ee4b2 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/TableTestBase.scala @@ -912,7 +912,7 @@ class TestingTableEnvironment private( def registerFunction[T: TypeInformation](name: String, tf: TableFunction[T]): Unit = { val typeInfo = UserFunctionsTypeHelper .getReturnTypeOfTableFunction(tf, implicitly[TypeInformation[T]]) - functionCatalog.registerTableFunction( + functionCatalog.registerTempSystemTableFunction( name, tf, typeInfo @@ -944,7 +944,7 @@ class TestingTableEnvironment private( .getReturnTypeOfAggregateFunction(f, implicitly[TypeInformation[T]]) val accTypeInfo = UserFunctionsTypeHelper .getAccumulatorTypeOfAggregateFunction(f, implicitly[TypeInformation[ACC]]) - functionCatalog.registerAggregateFunction( + functionCatalog.registerTempSystemAggregateFunction( name, f, typeInfo, diff --git a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/testTableSources.scala b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/testTableSources.scala index 24fab4251f909e33640860e5e2ee9b677a950ba9..0416218eb837c9835b69f1389a1785aae393b673 100644 --- a/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/testTableSources.scala +++ b/flink-table/flink-table-planner-blink/src/test/scala/org/apache/flink/table/planner/utils/testTableSources.scala @@ -364,9 +364,10 @@ class TestFilterableTableSource( override def explainSource(): String = { if (filterPredicates.nonEmpty) { + s"filterPushedDown=[$filterPushedDown], " + s"filter=[${filterPredicates.reduce((l, r) => unresolvedCall(AND, l, r)).toString}]" } else { - "" + s"filterPushedDown=[$filterPushedDown], filter=[]" } } diff --git a/flink-table/flink-table-planner/pom.xml b/flink-table/flink-table-planner/pom.xml index 9ab8f19019f8ce54d496f19a8d357e5eeebebb9a..bb92960053faf2f3cfed68209b0ae692d7464dd7 100644 --- a/flink-table/flink-table-planner/pom.xml +++ b/flink-table/flink-table-planner/pom.xml @@ -148,15 +148,15 @@ under the License. org.apache.calcite calcite-core - 1.20.0 + ${calcite.version} + true + + + + diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/SqlFunction.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/SqlFunction.java new file mode 100644 index 0000000000000000000000000000000000000000..5bb2bf5f3823f044e0755eafad2f9d8c42e09868 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/SqlFunction.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.sql; + +import org.apache.calcite.linq4j.function.Functions; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.type.SqlOperandTypeChecker; +import org.apache.calcite.sql.type.SqlOperandTypeInference; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.validate.SqlValidator; +import org.apache.calcite.sql.validate.SqlValidatorScope; +import org.apache.calcite.sql.validate.implicit.TypeCoercion; +import org.apache.calcite.util.Util; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import javax.annotation.Nonnull; + +import static org.apache.calcite.util.Static.RESOURCE; + +/** + * A SqlFunction is a type of operator which has conventional + * function-call syntax. + * + *

    This file is copied from Apache Calcite and should be removed once CALCITE-3360 + * has been fixed. Changed lines: 279-293 + */ +public class SqlFunction extends SqlOperator { + + //~ Instance fields -------------------------------------------------------- + + private final SqlFunctionCategory category; + + private final SqlIdentifier sqlIdentifier; + + private final List paramTypes; + + //~ Constructors ----------------------------------------------------------- + + /** + * Creates a new SqlFunction for a call to a builtin function. + * + * @param name Name of builtin function + * @param kind kind of operator implemented by function + * @param returnTypeInference strategy to use for return type inference + * @param operandTypeInference strategy to use for parameter type inference + * @param operandTypeChecker strategy to use for parameter type checking + * @param category categorization for function + */ + public SqlFunction( + String name, + SqlKind kind, + SqlReturnTypeInference returnTypeInference, + SqlOperandTypeInference operandTypeInference, + SqlOperandTypeChecker operandTypeChecker, + SqlFunctionCategory category) { + // We leave sqlIdentifier as null to indicate + // that this is a builtin. Same for paramTypes. + this(name, null, kind, returnTypeInference, operandTypeInference, + operandTypeChecker, null, category); + + assert !((category == SqlFunctionCategory.USER_DEFINED_CONSTRUCTOR) + && (returnTypeInference == null)); + } + + /** + * Creates a placeholder SqlFunction for an invocation of a function with a + * possibly qualified name. This name must be resolved into either a builtin + * function or a user-defined function. + * + * @param sqlIdentifier possibly qualified identifier for function + * @param returnTypeInference strategy to use for return type inference + * @param operandTypeInference strategy to use for parameter type inference + * @param operandTypeChecker strategy to use for parameter type checking + * @param paramTypes array of parameter types + * @param funcType function category + */ + public SqlFunction( + SqlIdentifier sqlIdentifier, + SqlReturnTypeInference returnTypeInference, + SqlOperandTypeInference operandTypeInference, + SqlOperandTypeChecker operandTypeChecker, + List paramTypes, + SqlFunctionCategory funcType) { + this(Util.last(sqlIdentifier.names), sqlIdentifier, SqlKind.OTHER_FUNCTION, + returnTypeInference, operandTypeInference, operandTypeChecker, + paramTypes, funcType); + } + + /** + * Internal constructor. + */ + protected SqlFunction( + String name, + SqlIdentifier sqlIdentifier, + SqlKind kind, + SqlReturnTypeInference returnTypeInference, + SqlOperandTypeInference operandTypeInference, + SqlOperandTypeChecker operandTypeChecker, + List paramTypes, + SqlFunctionCategory category) { + super(name, kind, 100, 100, returnTypeInference, operandTypeInference, + operandTypeChecker); + + this.sqlIdentifier = sqlIdentifier; + this.category = Objects.requireNonNull(category); + this.paramTypes = + paramTypes == null ? null : ImmutableList.copyOf(paramTypes); + } + + //~ Methods ---------------------------------------------------------------- + + public SqlSyntax getSyntax() { + return SqlSyntax.FUNCTION; + } + + /** + * @return fully qualified name of function, or null for a builtin function + */ + public SqlIdentifier getSqlIdentifier() { + return sqlIdentifier; + } + + @Override public SqlIdentifier getNameAsId() { + if (sqlIdentifier != null) { + return sqlIdentifier; + } + return super.getNameAsId(); + } + + /** + * @return array of parameter types, or null for builtin function + */ + public List getParamTypes() { + return paramTypes; + } + + /** + * Returns a list of parameter names. + * + *

    The default implementation returns {@code [arg0, arg1, ..., argN]}. + */ + public List getParamNames() { + return Functions.generate(paramTypes.size(), i -> "arg" + i); + } + + public void unparse( + SqlWriter writer, + SqlCall call, + int leftPrec, + int rightPrec) { + getSyntax().unparse(writer, this, call, leftPrec, rightPrec); + } + + /** + * @return function category + */ + @Nonnull public SqlFunctionCategory getFunctionType() { + return this.category; + } + + /** + * Returns whether this function allows a DISTINCT or + * ALL quantifier. The default is false; some aggregate + * functions return true. + */ + public boolean isQuantifierAllowed() { + return false; + } + + public void validateCall( + SqlCall call, + SqlValidator validator, + SqlValidatorScope scope, + SqlValidatorScope operandScope) { + // This implementation looks for the quantifier keywords DISTINCT or + // ALL as the first operand in the list. If found then the literal is + // not called to validate itself. Further the function is checked to + // make sure that a quantifier is valid for that particular function. + // + // If the first operand does not appear to be a quantifier then the + // parent ValidateCall is invoked to do normal function validation. + + super.validateCall(call, validator, scope, operandScope); + validateQuantifier(validator, call); + } + + /** + * Throws a validation error if a DISTINCT or ALL quantifier is present but + * not allowed. + */ + protected void validateQuantifier(SqlValidator validator, SqlCall call) { + if ((null != call.getFunctionQuantifier()) && !isQuantifierAllowed()) { + throw validator.newValidationError(call.getFunctionQuantifier(), + RESOURCE.functionQuantifierNotAllowed(call.getOperator().getName())); + } + } + + public RelDataType deriveType( + SqlValidator validator, + SqlValidatorScope scope, + SqlCall call) { + return deriveType(validator, scope, call, true); + } + + private RelDataType deriveType( + SqlValidator validator, + SqlValidatorScope scope, + SqlCall call, + boolean convertRowArgToColumnList) { + // Scope for operands. Usually the same as 'scope'. + final SqlValidatorScope operandScope = scope.getOperandScope(call); + + // Indicate to the validator that we're validating a new function call + validator.pushFunctionCall(); + + final List argNames = constructArgNameList(call); + + final List args = constructOperandList(validator, call, argNames); + + final List argTypes = constructArgTypeList(validator, scope, + call, args, convertRowArgToColumnList); + + SqlFunction function = + (SqlFunction) SqlUtil.lookupRoutine(validator.getOperatorTable(), + getNameAsId(), argTypes, argNames, getFunctionType(), + SqlSyntax.FUNCTION, getKind(), + validator.getCatalogReader().nameMatcher(), + false); + try { + // if we have a match on function name and parameter count, but + // couldn't find a function with a COLUMN_LIST type, retry, but + // this time, don't convert the row argument to a COLUMN_LIST type; + // if we did find a match, go back and re-validate the row operands + // (corresponding to column references), now that we can set the + // scope to that of the source cursor referenced by that ColumnList + // type + if (convertRowArgToColumnList && containsRowArg(args)) { + if (function == null + && SqlUtil.matchRoutinesByParameterCount( + validator.getOperatorTable(), getNameAsId(), argTypes, + getFunctionType(), + validator.getCatalogReader().nameMatcher())) { + // remove the already validated node types corresponding to + // row arguments before re-validating + for (SqlNode operand : args) { + if (operand.getKind() == SqlKind.ROW) { + validator.removeValidatedNodeType(operand); + } + } + return deriveType(validator, scope, call, false); + } else if (function != null) { + validator.validateColumnListParams(function, argTypes, args); + } + } + + if (getFunctionType() == SqlFunctionCategory.USER_DEFINED_CONSTRUCTOR) { + return validator.deriveConstructorType(scope, call, this, function, + argTypes); + } + if (function == null) { + // try again if implicit type coercion is allowed. + boolean changed = false; + if (validator.isTypeCoercionEnabled()) { + function = (SqlFunction) SqlUtil.lookupRoutine(validator.getOperatorTable(), + getNameAsId(), argTypes, argNames, getFunctionType(), SqlSyntax.FUNCTION, getKind(), + validator.getCatalogReader().nameMatcher(), + true); + // try to coerce the function arguments to the declared sql type name. + // if we succeed, the arguments would be wrapped with CAST operator. + if (function != null) { + TypeCoercion typeCoercion = validator.getTypeCoercion(); + changed = typeCoercion.userDefinedFunctionCoercion(scope, call, function); + } + } + if (!changed) { + throw validator.handleUnresolvedFunction(call, this, argTypes, + argNames); + } + } + + // REVIEW jvs 25-Mar-2005: This is, in a sense, expanding + // identifiers, but we ignore shouldExpandIdentifiers() + // because otherwise later validation code will + // choke on the unresolved function. + ((SqlBasicCall) call).setOperator(function); + return function.validateOperands( + validator, + operandScope, + call); + } finally { + validator.popFunctionCall(); + } + } + + private boolean containsRowArg(List args) { + for (SqlNode operand : args) { + if (operand.getKind() == SqlKind.ROW) { + return true; + } + } + return false; + } +} + +// End SqlFunction.java diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java index bbcbb052494310f2cb2e1e951241cc6d82b3df64..c3ccc3f9b9b39e8d5ca8950e6d38fb03ee3a6d0f 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java @@ -90,6 +90,8 @@ import org.apache.calcite.sql.type.SqlTypeUtil; import org.apache.calcite.sql.util.SqlBasicVisitor; import org.apache.calcite.sql.util.SqlShuttle; import org.apache.calcite.sql.util.SqlVisitor; +import org.apache.calcite.sql.validate.implicit.TypeCoercion; +import org.apache.calcite.sql.validate.implicit.TypeCoercions; import org.apache.calcite.sql2rel.InitializerContext; import org.apache.calcite.util.BitString; import org.apache.calcite.util.Bug; @@ -290,6 +292,12 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { private final SqlValidatorImpl.ValidationErrorFunction validationErrorFunction = new SqlValidatorImpl.ValidationErrorFunction(); + // TypeCoercion instance used for implicit type coercion. + private TypeCoercion typeCoercion; + + // Flag saying if we enable the implicit type coercion. + private boolean enableTypeCoercion; + //~ Constructors ----------------------------------------------------------- /** @@ -321,6 +329,9 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { overFinder = new AggFinder(opTab, true, false, false, aggOrOverFinder, nameMatcher); groupFinder = new AggFinder(opTab, false, false, true, null, nameMatcher); aggOrOverOrGroupFinder = new AggFinder(opTab, true, true, true, null, nameMatcher); + this.enableTypeCoercion = catalogReader.getConfig() == null + || catalogReader.getConfig().typeCoercion(); + this.typeCoercion = TypeCoercions.getTypeCoercion(this, conformance); } //~ Methods ---------------------------------------------------------------- @@ -3818,6 +3829,29 @@ public class SqlValidatorImpl implements SqlValidatorWithHints { return scopes.get(withItem); } + @Override + public SqlValidator setEnableTypeCoercion(boolean enabled) { + this.enableTypeCoercion = enabled; + return this; + } + + @Override + public boolean isTypeCoercionEnabled() { + return this.enableTypeCoercion; + } + + @Override + public void setTypeCoercion(TypeCoercion typeCoercion) { + Objects.requireNonNull(typeCoercion); + this.typeCoercion = typeCoercion; + } + + @Override + public TypeCoercion getTypeCoercion() { + assert isTypeCoercionEnabled(); + return this.typeCoercion; + } + /** * Validates the ORDER BY clause of a SELECT statement. * diff --git a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/sqlexec/SqlToOperationConverter.java b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/sqlexec/SqlToOperationConverter.java index 427b89ab5f773a836c6f01ef53e7a352ce7bd20c..c7566602f737859183fd1e12020e3368a8b3c126 100644 --- a/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/sqlexec/SqlToOperationConverter.java +++ b/flink-table/flink-table-planner/src/main/java/org/apache/flink/table/sqlexec/SqlToOperationConverter.java @@ -27,7 +27,6 @@ import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.TableSchema; import org.apache.flink.table.calcite.FlinkPlannerImpl; import org.apache.flink.table.calcite.FlinkTypeFactory; -import org.apache.flink.table.calcite.FlinkTypeSystem; import org.apache.flink.table.catalog.CatalogTable; import org.apache.flink.table.catalog.CatalogTableImpl; import org.apache.flink.table.operations.CatalogSinkModifyOperation; @@ -44,7 +43,6 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.SqlNodeList; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -101,38 +99,27 @@ public class SqlToOperationConverter { */ private Operation convertCreateTable(SqlCreateTable sqlCreateTable) { // primary key and unique keys are not supported - if ((sqlCreateTable.getPrimaryKeyList() != null - && sqlCreateTable.getPrimaryKeyList().size() > 0) - || (sqlCreateTable.getUniqueKeysList() != null - && sqlCreateTable.getUniqueKeysList().size() > 0)) { + if ((sqlCreateTable.getPrimaryKeyList().size() > 0) + || (sqlCreateTable.getUniqueKeysList().size() > 0)) { throw new SqlConversionException("Primary key and unique key are not supported yet."); } // set with properties - SqlNodeList propertyList = sqlCreateTable.getPropertyList(); Map properties = new HashMap<>(); - if (propertyList != null) { - propertyList.getList().forEach(p -> - properties.put(((SqlTableOption) p).getKeyString().toLowerCase(), - ((SqlTableOption) p).getValueString())); - } + sqlCreateTable.getPropertyList().getList().forEach(p -> + properties.put(((SqlTableOption) p).getKeyString().toLowerCase(), + ((SqlTableOption) p).getValueString())); - TableSchema tableSchema = createTableSchema(sqlCreateTable, - new FlinkTypeFactory(new FlinkTypeSystem())); // need to make type factory singleton ? - String tableComment = ""; - if (sqlCreateTable.getComment() != null) { - tableComment = sqlCreateTable.getComment().getNlsString().getValue(); - } + TableSchema tableSchema = createTableSchema(sqlCreateTable); + String tableComment = sqlCreateTable.getComment().map(comment -> + comment.getNlsString().getValue()).orElse(null); // set partition key - List partitionKeys = new ArrayList<>(); - SqlNodeList partitionKey = sqlCreateTable.getPartitionKeyList(); - if (partitionKey != null) { - partitionKeys = partitionKey - .getList() - .stream() - .map(p -> ((SqlIdentifier) p).getSimple()) - .collect(Collectors.toList()); - } + List partitionKeys = sqlCreateTable.getPartitionKeyList() + .getList() + .stream() + .map(p -> ((SqlIdentifier) p).getSimple()) + .collect(Collectors.toList()); + CatalogTable catalogTable = new CatalogTableImpl(tableSchema, partitionKeys, properties, @@ -181,11 +168,9 @@ public class SqlToOperationConverter { *

    The returned table schema contains columns (a:int, b:varchar, c:timestamp). * * @param sqlCreateTable sql create table node. - * @param factory FlinkTypeFactory instance. * @return TableSchema */ - private TableSchema createTableSchema(SqlCreateTable sqlCreateTable, - FlinkTypeFactory factory) { + private TableSchema createTableSchema(SqlCreateTable sqlCreateTable) { // setup table columns SqlNodeList columnList = sqlCreateTable.getColumnList(); TableSchema physicalSchema = null; @@ -195,8 +180,10 @@ public class SqlToOperationConverter { .filter(n -> n instanceof SqlTableColumn).collect(Collectors.toList()); for (SqlNode node : physicalColumns) { SqlTableColumn column = (SqlTableColumn) node; - final RelDataType relType = column.getType().deriveType(factory, - column.getType().getNullable()); + final RelDataType relType = column.getType() + .deriveType( + flinkPlanner.getOrCreateSqlValidator(), + column.getType().getNullable()); builder.field(column.getName().getSimple(), TypeConversions.fromLegacyInfoToDataType(FlinkTypeFactory.toTypeInfo(relType))); physicalSchema = builder.build(); diff --git a/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE b/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE index 31d8cff0002bd1ce205f09a74d7341a528fdc1e8..d01508b3bbc9d1df74f96d8046a7b3195bc1ef06 100644 --- a/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE +++ b/flink-table/flink-table-planner/src/main/resources/META-INF/NOTICE @@ -12,8 +12,8 @@ This project bundles the following dependencies under the Apache Software Licens - com.fasterxml.jackson.core:jackson-databind:2.9.6 - com.jayway.jsonpath:json-path:2.4.0 - joda-time:joda-time:2.5 -- org.apache.calcite:calcite-core:1.20.0 -- org.apache.calcite:calcite-linq4j:1.20.0 +- org.apache.calcite:calcite-core:1.21.0 +- org.apache.calcite:calcite-linq4j:1.21.0 - org.apache.calcite.avatica:avatica-core:1.15.0 - commons-codec:commons-codec:1.10 diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/internal/TableEnvImpl.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/internal/TableEnvImpl.scala index 0e002688791a32788567038368ea5a05c481dedb..a00ea3526b22a41176a3411d5048f31cbc0ea7e3 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/internal/TableEnvImpl.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/internal/TableEnvImpl.scala @@ -99,7 +99,7 @@ abstract class TableEnvImpl( private def isBatchTable: Boolean = !isStreamingMode override def registerFunction(name: String, function: ScalarFunction): Unit = { - functionCatalog.registerScalarFunction( + functionCatalog.registerTempSystemScalarFunction( name, function) } @@ -117,7 +117,7 @@ abstract class TableEnvImpl( function, implicitly[TypeInformation[T]]) - functionCatalog.registerTableFunction( + functionCatalog.registerTempSystemTableFunction( name, function, resultTypeInfo) @@ -141,7 +141,7 @@ abstract class TableEnvImpl( function, implicitly[TypeInformation[ACC]]) - functionCatalog.registerAggregateFunction( + functionCatalog.registerTempSystemAggregateFunction( name, function, resultTypeInfo, diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/CalciteConfig.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/CalciteConfig.scala index 8064fa9b9aa43f017a6eed3e1c736f7e1b3d3a4f..4ba17131ddbdf750e42cbcdbf1c2e2874297e80d 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/CalciteConfig.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/CalciteConfig.scala @@ -52,6 +52,12 @@ class CalciteConfigBuilder { private var replaceLogicalOptRules: Boolean = false private var logicalOptRuleSets: List[RuleSet] = Nil + /** + * Defines the logical rewrite rule set. + */ + private var replaceLogicalRewriteRules: Boolean = false + private var logicalRewriteRuleSets: List[RuleSet] = Nil + /** * Defines the physical optimization rule set. */ @@ -119,6 +125,25 @@ class CalciteConfigBuilder { this } + /** + * Replaces the built-in logical rewrite rule set with the given rule set. + */ + def replaceLogicalRewriteRuleSet(replaceRuleSet: RuleSet): CalciteConfigBuilder = { + Preconditions.checkNotNull(replaceRuleSet) + logicalRewriteRuleSets = List(replaceRuleSet) + replaceLogicalRewriteRules = true + this + } + + /** + * Appends the given logical rewrite rule set to the built-in rule set. + */ + def addLogicalRewriteRuleSet(addedRuleSet: RuleSet): CalciteConfigBuilder = { + Preconditions.checkNotNull(addedRuleSet) + logicalRewriteRuleSets = addedRuleSet :: logicalRewriteRuleSets + this + } + /** * Replaces the built-in optimization rule set with the given rule set. */ @@ -225,6 +250,8 @@ class CalciteConfigBuilder { replaceNormRules, getRuleSet(logicalOptRuleSets), replaceLogicalOptRules, + getRuleSet(logicalRewriteRuleSets), + replaceLogicalRewriteRules, getRuleSet(physicalOptRuleSets), replacePhysicalOptRules, getRuleSet(decoRuleSets), @@ -254,6 +281,10 @@ class CalciteConfig( val logicalOptRuleSet: Option[RuleSet], /** Whether this configuration replaces the built-in logical optimization rule set. */ val replacesLogicalOptRuleSet: Boolean, + /** A custom logical rewrite rule set. */ + val logicalRewriteRuleSet: Option[RuleSet], + /** Whether this configuration replaces the built-in logical rewrite rule set. */ + val replacesLogicalRewriteRuleSet: Boolean, /** A custom physical optimization rule set. */ val physicalOptRuleSet: Option[RuleSet], /** Whether this configuration replaces the built-in physical optimization rule set. */ diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala index e3b7e1479c31008136fe123f3c105de0adcef95d..285631645be103ee731e6a3996902831a9b0da27 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/FlinkPlannerImpl.scala @@ -88,6 +88,29 @@ class FlinkPlannerImpl( hints.toArray } + /** + * Get the [[FlinkCalciteSqlValidator]] instance from this planner, create a new instance + * if current validator has not been initialized, or returns the validator + * instance directly. + * + *

    The validator instance creation is not thread safe. + * + * @return a new validator instance or current existed one + */ + def getOrCreateSqlValidator(): FlinkCalciteSqlValidator = { + if (validator == null) { + val catalogReader = catalogReaderSupplier.apply(false) + validator = new FlinkCalciteSqlValidator( + operatorTable, + catalogReader, + typeFactory) + validator.setIdentifierExpansion(true) + // Disable implicit type coercion for now. + validator.setEnableTypeCoercion(false) + } + validator + } + def parse(sql: String): SqlNode = { try { ready() @@ -115,13 +138,8 @@ class FlinkPlannerImpl( || sqlNode.getKind == SqlKind.INSERT) { return sqlNode } - validator = new FlinkCalciteSqlValidator( - operatorTable, - catalogReader, - typeFactory) - validator.setIdentifierExpansion(true) try { - validator.validate(sqlNode) + getOrCreateSqlValidator().validate(sqlNode) } catch { case e: RuntimeException => @@ -137,7 +155,7 @@ class FlinkPlannerImpl( val catalogReader: CatalogReader = catalogReaderSupplier.apply(false) val sqlToRelConverter: SqlToRelConverter = new SqlToRelConverter( new ViewExpanderImpl, - validator, + getOrCreateSqlValidator(), catalogReader, cluster, convertletTable, diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala index 836866786d2464109e42ea6118dcefafb963ca15..a64d61dcbcced2d2d1b21f7dd075e98ccd07e77e 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/FlinkRelBuilder.scala @@ -50,11 +50,11 @@ class FlinkRelBuilder( private val toRelNodeConverter = new QueryOperationConverter(this, expressionBridge) - def getRelOptSchema: RelOptSchema = relOptSchema + override def getRelOptSchema: RelOptSchema = relOptSchema def getPlanner: RelOptPlanner = cluster.getPlanner - def getCluster: RelOptCluster = relOptCluster + override def getCluster: RelOptCluster = relOptCluster override def shouldMergeProject(): Boolean = false diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala index 1a9613c72cdd8c09426e8812d6c345b5b12eac1e..d3aaf7d8e69ee3e60270c97f50789519074a3a81 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/calcite/RelTimeIndicatorConverter.scala @@ -109,9 +109,6 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle { val measures = matchRel.getMeasures .mapValues(_.accept(materializer)) - val partitionKeys = matchRel.getPartitionKeys - .map(_.accept(materializer)) - .map(materializerUtils.materialize) val interval = if (matchRel.getInterval != null) { matchRel.getInterval.accept(materializer) } else { @@ -137,7 +134,7 @@ class RelTimeIndicatorConverter(rexBuilder: RexBuilder) extends RelShuttle { matchRel.getAfter, matchRel.getSubsets.asInstanceOf[java.util.Map[String, java.util.TreeSet[String]]], matchRel.isAllRows, - partitionKeys, + matchRel.getPartitionKeys, matchRel.getOrderKeys, interval) } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/ExpressionReducer.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/ExpressionReducer.scala index dfed70be101895bdb2fef16933c4ef15dfc4c189..e3e44ae8a1d47f2b16d774faacf96311ebf2c51a 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/ExpressionReducer.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/ExpressionReducer.scala @@ -28,9 +28,12 @@ import org.apache.flink.api.common.typeinfo.BasicTypeInfo import org.apache.flink.api.java.typeutils.RowTypeInfo import org.apache.flink.table.api.TableConfig import org.apache.flink.table.calcite.FlinkTypeFactory +import org.apache.flink.table.functions.FunctionLanguage +import org.apache.flink.table.plan.util.PythonUtil import org.apache.flink.types.Row import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer /** * Evaluates constant expressions using Flink's [[FunctionCodeGenerator]]. @@ -48,8 +51,16 @@ class ExpressionReducer(config: TableConfig) val typeFactory = rexBuilder.getTypeFactory.asInstanceOf[FlinkTypeFactory] + val pythonUDFExprs = ListBuffer[RexNode]() + val literals = constExprs.asScala.map(e => (e.getType.getSqlTypeName, e)).flatMap { + // Skip expressions that contain python functions because it's quite expensive to + // call Python UDFs during optimization phase. They will be optimized during the runtime. + case (_, e) if PythonUtil.containsFunctionOf(e, FunctionLanguage.PYTHON) => + pythonUDFExprs += e + None + // we need to cast here for RexBuilder.makeLiteral case (SqlTypeName.DATE, e) => Some( @@ -114,34 +125,40 @@ class ExpressionReducer(config: TableConfig) var reducedIdx = 0 while (i < constExprs.size()) { val unreduced = constExprs.get(i) - unreduced.getType.getSqlTypeName match { - // we insert the original expression for object literals - case SqlTypeName.ANY | - SqlTypeName.ROW | - SqlTypeName.ARRAY | - SqlTypeName.MAP | - SqlTypeName.MULTISET => - reducedValues.add(unreduced) - - case _ => - val reducedValue = reduced.getField(reducedIdx) - // RexBuilder handle double literal incorrectly, convert it into BigDecimal manually - val value = if (unreduced.getType.getSqlTypeName == SqlTypeName.DOUBLE) { - if (reducedValue == null) { - reducedValue + // use eq to compare reference + if (pythonUDFExprs.exists(_ eq unreduced)) { + // if contains python function then just insert the original expression. + reducedValues.add(unreduced) + } else { + unreduced.getType.getSqlTypeName match { + // we insert the original expression for object literals + case SqlTypeName.ANY | + SqlTypeName.ROW | + SqlTypeName.ARRAY | + SqlTypeName.MAP | + SqlTypeName.MULTISET => + reducedValues.add(unreduced) + + case _ => + val reducedValue = reduced.getField(reducedIdx) + // RexBuilder handle double literal incorrectly, convert it into BigDecimal manually + val value = if (unreduced.getType.getSqlTypeName == SqlTypeName.DOUBLE) { + if (reducedValue == null) { + reducedValue + } else { + new java.math.BigDecimal(reducedValue.asInstanceOf[Number].doubleValue()) + } } else { - new java.math.BigDecimal(reducedValue.asInstanceOf[Number].doubleValue()) + reducedValue } - } else { - reducedValue - } - - val literal = rexBuilder.makeLiteral( - value, - unreduced.getType, - true) - reducedValues.add(literal) - reducedIdx += 1 + + val literal = rexBuilder.makeLiteral( + value, + unreduced.getType, + true) + reducedValues.add(literal) + reducedIdx += 1 + } } i += 1 } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/MatchCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/MatchCodeGenerator.scala index e2c956e57996ebaec906ec24ad7869e29d9819b2..903521ff8dff55f02010f0692505bcdce039ce48 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/MatchCodeGenerator.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/MatchCodeGenerator.scala @@ -20,7 +20,6 @@ package org.apache.flink.table.codegen import java.lang.{Long => JLong} import java.util - import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rex._ import org.apache.calcite.sql.SqlAggFunction @@ -48,6 +47,8 @@ import org.apache.flink.types.Row import org.apache.flink.util.Collector import org.apache.flink.util.MathUtils.checkedDownCast +import org.apache.calcite.util.ImmutableBitSet + import scala.collection.JavaConverters._ import scala.collection.mutable @@ -267,7 +268,7 @@ class MatchCodeGenerator( */ def generateOneRowPerMatchExpression( returnType: RowSchema, - partitionKeys: util.List[RexNode], + partitionKeys: ImmutableBitSet, measures: util.Map[String, RexNode]) : PatternProcessFunctionRunner = { val resultExpression = generateOneRowPerMatchExpression( @@ -404,15 +405,15 @@ class MatchCodeGenerator( * @return generated code for the given key */ private def generatePartitionKeyAccess( - partitionKey: RexInputRef) + partitionIdx: Int) : GeneratedExpression = { val keyRow = generateKeyRow() - generateFieldAccess(keyRow, partitionKey.getIndex) + generateFieldAccess(keyRow, partitionIdx) } private def generateOneRowPerMatchExpression( - partitionKeys: util.List[RexNode], + partitionKeys: ImmutableBitSet, measures: util.Map[String, RexNode], returnType: RowSchema) : GeneratedExpression = { @@ -420,9 +421,10 @@ class MatchCodeGenerator( // 1) the partition columns; // 2) the columns defined in the measures clause. val resultExprs = - partitionKeys.asScala.map { case inputRef: RexInputRef => - generatePartitionKeyAccess(inputRef) - } ++ returnType.fieldNames.filter(measures.containsKey(_)).map { fieldName => + partitionKeys.toList.asScala + .map(generatePartitionKeyAccess(_)) ++ + returnType.fieldNames + .filter(measures.containsKey(_)).map { fieldName => generateExpression(measures.get(fieldName)) } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/PythonFunctionCodeGenerator.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/PythonFunctionCodeGenerator.scala new file mode 100644 index 0000000000000000000000000000000000000000..e97e7b79eeb797b7daa229b7f27d042daa551607 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/codegen/PythonFunctionCodeGenerator.scala @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.table.codegen + +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.table.codegen.CodeGenUtils.{primitiveDefaultValue, primitiveTypeTermForTypeInfo, newName} +import org.apache.flink.table.codegen.Indenter.toISC +import org.apache.flink.table.functions.{UserDefinedFunction, FunctionLanguage, ScalarFunction} +import org.apache.flink.table.functions.python.{PythonEnv, PythonFunction} +import org.apache.flink.table.utils.EncodingUtils + +/** + * A code generator for generating Python [[UserDefinedFunction]]s. + */ +object PythonFunctionCodeGenerator extends Compiler[UserDefinedFunction] { + + private val PYTHON_SCALAR_FUNCTION_NAME = "PythonScalarFunction" + + /** + * Generates a [[ScalarFunction]] for the specified Python user-defined function. + * + * @param name class name of the user-defined function. Must be a valid Java class identifier + * @param serializedScalarFunction serialized Python scalar function + * @param inputTypes input data types + * @param resultType expected result type + * @param deterministic the determinism of the function's results + * @param pythonEnv the Python execution environment + * @return instance of generated ScalarFunction + */ + def generateScalarFunction( + name: String, + serializedScalarFunction: Array[Byte], + inputTypes: Array[TypeInformation[_]], + resultType: TypeInformation[_], + deterministic: Boolean, + pythonEnv: PythonEnv): ScalarFunction = { + val funcName = newName(PYTHON_SCALAR_FUNCTION_NAME) + val resultTypeTerm = primitiveTypeTermForTypeInfo(resultType) + val defaultResultValue = primitiveDefaultValue(resultType) + val inputParamCode = inputTypes.zipWithIndex.map { case (inputType, index) => + s"${primitiveTypeTermForTypeInfo(inputType)} in$index" + }.mkString(", ") + + val encodingUtilsTypeTerm = classOf[EncodingUtils].getCanonicalName + val typeInfoTypeTerm = classOf[TypeInformation[_]].getCanonicalName + val inputTypesCode = inputTypes.map(EncodingUtils.encodeObjectToString).map { inputType => + s""" + |($typeInfoTypeTerm) $encodingUtilsTypeTerm.decodeStringToObject( + | "$inputType", $typeInfoTypeTerm.class) + |""".stripMargin + }.mkString(", ") + + val encodedResultType = EncodingUtils.encodeObjectToString(resultType) + val encodedScalarFunction = EncodingUtils.encodeBytesToBase64(serializedScalarFunction) + val encodedPythonEnv = EncodingUtils.encodeObjectToString(pythonEnv) + val pythonEnvTypeTerm = classOf[PythonEnv].getCanonicalName + + val funcCode = j""" + |public class $funcName extends ${classOf[ScalarFunction].getCanonicalName} + | implements ${classOf[PythonFunction].getCanonicalName} { + | + | private static final long serialVersionUID = 1L; + | + | public $resultTypeTerm eval($inputParamCode) { + | return $defaultResultValue; + | } + | + | @Override + | public $typeInfoTypeTerm[] getParameterTypes(Class[] signature) { + | return new $typeInfoTypeTerm[]{$inputTypesCode}; + | } + | + | @Override + | public $typeInfoTypeTerm getResultType(Class[] signature) { + | return ($typeInfoTypeTerm) $encodingUtilsTypeTerm.decodeStringToObject( + | "$encodedResultType", $typeInfoTypeTerm.class); + | } + | + | @Override + | public ${classOf[FunctionLanguage].getCanonicalName} getLanguage() { + | return ${classOf[FunctionLanguage].getCanonicalName}.PYTHON; + | } + | + | @Override + | public byte[] getSerializedPythonFunction() { + | return $encodingUtilsTypeTerm.decodeBase64ToBytes("$encodedScalarFunction"); + | } + | + | @Override + | public $pythonEnvTypeTerm getPythonEnv() { + | return ($pythonEnvTypeTerm) $encodingUtilsTypeTerm.decodeStringToObject( + | "$encodedPythonEnv", $pythonEnvTypeTerm.class); + | } + | + | @Override + | public boolean isDeterministic() { + | return $deterministic; + | } + | + | @Override + | public String toString() { + | return "$name"; + | } + |} + |""".stripMargin + + val clazz = compile( + Thread.currentThread().getContextClassLoader, + funcName, + funcCode) + clazz.newInstance().asInstanceOf[ScalarFunction] + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/functions/utils/AggSqlFunction.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/functions/utils/AggSqlFunction.scala index 4859c2a3d28969a9e101a3b6c42b8127e3c52573..de71825ddac8f50cad9a07ff4fb10cd8a1aef4fd 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/functions/utils/AggSqlFunction.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/functions/utils/AggSqlFunction.scala @@ -56,8 +56,8 @@ class AggSqlFunction( extends SqlUserDefinedAggFunction( new SqlIdentifier(name, SqlParserPos.ZERO), createReturnTypeInference(returnType, typeFactory), - createOperandTypeInference(aggregateFunction, typeFactory), - createOperandTypeChecker(aggregateFunction), + createOperandTypeInference(aggregateFunction, typeFactory, accType), + createOperandTypeChecker(aggregateFunction, accType), // Do not need to provide a calcite aggregateFunction here. Flink aggregation function // will be generated when translating the calcite relnode to flink runtime execution plan null, @@ -104,7 +104,8 @@ object AggSqlFunction { private[flink] def createOperandTypeInference( aggregateFunction: UserDefinedAggregateFunction[_, _], - typeFactory: FlinkTypeFactory) + typeFactory: FlinkTypeFactory, + accType: TypeInformation[_]) : SqlOperandTypeInference = { /** * Operand type inference based on [[AggregateFunction]] given information. @@ -117,11 +118,13 @@ object AggSqlFunction { val operandTypeInfo = getOperandTypeInfo(callBinding) + val actualSignature = accType +: operandTypeInfo + val foundSignature = getAccumulateMethodSignature(aggregateFunction, operandTypeInfo) .getOrElse( throw new ValidationException( s"Given parameters of function do not match any signature. \n" + - s"Actual: ${signatureToString(operandTypeInfo)} \n" + + s"Actual: ${signatureToString(actualSignature)} \n" + s"Expected: ${signaturesToString(aggregateFunction, "accumulate")}")) val inferredTypes = getParameterTypes(aggregateFunction, foundSignature.drop(1)) @@ -153,8 +156,10 @@ object AggSqlFunction { } } - private[flink] def createOperandTypeChecker(aggregateFunction: UserDefinedAggregateFunction[_, _]) - : SqlOperandTypeChecker = { + private[flink] def createOperandTypeChecker( + aggregateFunction: UserDefinedAggregateFunction[_, _], + accType: TypeInformation[_]) + : SqlOperandTypeChecker = { val methods = checkAndExtractMethods(aggregateFunction, "accumulate") @@ -195,13 +200,15 @@ object AggSqlFunction { : Boolean = { val operandTypeInfo = getOperandTypeInfo(callBinding) + val actualSignature = accType +: operandTypeInfo + val foundSignature = getAccumulateMethodSignature(aggregateFunction, operandTypeInfo) if (foundSignature.isEmpty) { if (throwOnFailure) { throw new ValidationException( s"Given parameters of function do not match any signature. \n" + - s"Actual: ${signatureToString(operandTypeInfo)} \n" + + s"Actual: ${signatureToString(actualSignature)} \n" + s"Expected: ${signaturesToString(aggregateFunction, "accumulate")}") } else { false diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/Optimizer.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/Optimizer.scala index 8506749dbe751da9d972bb5d0fd3a96ef9d352c9..4336c074c3c94808cb0c492147db29a3c93b7374 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/Optimizer.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/Optimizer.scala @@ -86,6 +86,25 @@ abstract class Optimizer( } } + /** + * Returns the logical rewrite rule set for this optimizer + * including a custom RuleSet configuration. + */ + protected def getLogicalRewriteRuleSet: RuleSet = { + materializedConfig.logicalRewriteRuleSet match { + + case None => + getBuiltInLogicalRewriteRuleSet + + case Some(ruleSet) => + if (materializedConfig.replacesLogicalRewriteRuleSet) { + ruleSet + } else { + RuleSets.ofList((getBuiltInLogicalRewriteRuleSet.asScala ++ ruleSet.asScala).asJava) + } + } + } + /** * Returns the physical optimization rule set for this optimizer * including a custom RuleSet configuration. @@ -117,6 +136,13 @@ abstract class Optimizer( FlinkRuleSets.LOGICAL_OPT_RULES } + /** + * Returns the built-in logical rewrite rules that are defined by the optimizer. + */ + protected def getBuiltInLogicalRewriteRuleSet: RuleSet = { + FlinkRuleSets.LOGICAL_REWRITE_RULES + } + /** * Returns the built-in physical optimization rules that are defined by the optimizer. */ @@ -153,6 +179,19 @@ abstract class Optimizer( } } + protected def optimizeLogicalRewritePlan(relNode: RelNode): RelNode = { + val logicalRewriteRuleSet = getLogicalRewriteRuleSet + if (logicalRewriteRuleSet.iterator().hasNext) { + runHepPlannerSimultaneously( + HepMatchOrder.TOP_DOWN, + logicalRewriteRuleSet, + relNode, + relNode.getTraitSet) + } else { + relNode + } + } + protected def optimizeLogicalPlan(relNode: RelNode): RelNode = { val logicalOptRuleSet = getLogicalOptRuleSet val logicalOutputProps = relNode.getTraitSet.replace(FlinkConventions.LOGICAL).simplify() diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/StreamOptimizer.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/StreamOptimizer.scala index 30ca4861ca69552dbcc940caa76534845f778fb5..1ef6d7017d2606a80168abcaedc3711432d3603f 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/StreamOptimizer.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/StreamOptimizer.scala @@ -64,8 +64,8 @@ class StreamOptimizer( RelTimeIndicatorConverter.convert(decorPlan, relBuilder.getRexBuilder) val normalizedPlan = optimizeNormalizeLogicalPlan(planWithMaterializedTimeAttributes) val logicalPlan = optimizeLogicalPlan(normalizedPlan) - - val physicalPlan = optimizePhysicalPlan(logicalPlan, FlinkConventions.DATASTREAM) + val logicalRewritePlan = optimizeLogicalRewritePlan(logicalPlan) + val physicalPlan = optimizePhysicalPlan(logicalRewritePlan, FlinkConventions.DATASTREAM) optimizeDecoratePlan(physicalPlan, updatesAsRetraction) } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/logical/MatchRecognize.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/logical/MatchRecognize.scala index 7a1e81d0811b37893c44b2c9bb0db1d4bb9077cc..a51eb20661603fc41b2eda9ac4e24553ce95cec0 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/logical/MatchRecognize.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/logical/MatchRecognize.scala @@ -19,11 +19,11 @@ package org.apache.flink.table.plan.logical import java.util - import com.google.common.collect.ImmutableMap import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rel.{RelCollation, RelNode} import org.apache.calcite.rex.RexNode +import org.apache.calcite.util.ImmutableBitSet /** * Describes MATCH RECOGNIZE clause. @@ -37,6 +37,6 @@ case class MatchRecognize( after: RexNode, subsets: ImmutableMap[String, util.SortedSet[String]], allRows: Boolean, - partitionKeys: util.List[RexNode], + partitionKeys: ImmutableBitSet, orderKeys: RelCollation, interval: RexNode) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala index 36df67a0721a8cc98071eea399bf019a099ad467..34f4ba84bda28903ffb3c192765d2a161719e7d3 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonCalc.scala @@ -34,7 +34,6 @@ trait CommonCalc { private[flink] def generateFunction[T <: Function]( generator: FunctionCodeGenerator, ruleDescription: String, - inputSchema: RowSchema, returnSchema: RowSchema, calcProjection: Seq[RexNode], calcCondition: Option[RexNode], diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonMatchRecognize.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonMatchRecognize.scala index 8be00bba53e77e292148c2a1c9391795a3753265..c052a42e8720bc6e39b2d18b72b8f51b446352d5 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonMatchRecognize.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonMatchRecognize.scala @@ -18,8 +18,7 @@ package org.apache.flink.table.plan.nodes -import java.util.{List => JList, SortedSet => JSortedSet} - +import java.util.{SortedSet => JSortedSet} import com.google.common.collect.ImmutableMap import org.apache.calcite.rel.{RelCollation, RelWriter} import org.apache.calcite.rex.{RexCall, RexLiteral, RexNode} @@ -28,16 +27,17 @@ import org.apache.calcite.sql.SqlMatchRecognize.AfterOption import org.apache.flink.table.plan.logical.MatchRecognize import org.apache.flink.table.runtime.aggregate.SortUtil.directionToOrder +import org.apache.calcite.util.ImmutableBitSet + import scala.collection.JavaConverters._ trait CommonMatchRecognize { private def partitionKeysToString( - keys: JList[RexNode], - fieldNames: Seq[String], - expression: (RexNode, Seq[String], Option[Seq[RexNode]]) => String) + keys: ImmutableBitSet, + fieldNames: Seq[String]) : String = - keys.asScala.map(k => expression(k, fieldNames, None)).mkString(", ") + keys.toArray.map(k => fieldNames(k)).mkString(", ") private def orderingToString(orders: RelCollation, fieldNames: Seq[String]): String = orders.getFieldCollations.asScala.map { @@ -88,7 +88,7 @@ trait CommonMatchRecognize { : String = { val partitionBy = if (!logicalMatch.partitionKeys.isEmpty) { s"PARTITION BY: ${ - partitionKeysToString(logicalMatch.partitionKeys, fieldNames, expression) + partitionKeysToString(logicalMatch.partitionKeys, fieldNames) }" } else { "" @@ -136,7 +136,7 @@ trait CommonMatchRecognize { expression: (RexNode, Seq[String], Option[Seq[RexNode]]) => String) : RelWriter = { pw.itemIf("partitionBy", - partitionKeysToString(logicalMatch.partitionKeys, fieldNames, expression), + partitionKeysToString(logicalMatch.partitionKeys, fieldNames), !logicalMatch.partitionKeys.isEmpty) .itemIf("orderBy", orderingToString(logicalMatch.orderKeys, fieldNames), diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonCalc.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonCalc.scala new file mode 100644 index 0000000000000000000000000000000000000000..aca06762ed1d5f738fb4292699d7f6bae8ee4d09 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/CommonPythonCalc.scala @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.table.plan.nodes + +import org.apache.calcite.rex.{RexCall, RexInputRef, RexLiteral, RexNode} +import org.apache.calcite.sql.`type`.SqlTypeName +import org.apache.flink.table.functions.FunctionLanguage +import org.apache.flink.table.functions.python.{PythonFunction, PythonFunctionInfo, SimplePythonFunction} +import org.apache.flink.table.functions.utils.ScalarSqlFunction + +import scala.collection.JavaConversions._ +import scala.collection.mutable + +trait CommonPythonCalc { + + private lazy val convertLiteralToPython = { + val clazz = Class.forName("org.apache.flink.api.common.python.PythonBridgeUtils") + clazz.getMethod("convertLiteralToPython", classOf[RexLiteral], classOf[SqlTypeName]) + } + + private[flink] def extractPythonScalarFunctionInfos( + rexCalls: Array[RexCall]): (Array[Int], Array[PythonFunctionInfo]) = { + // using LinkedHashMap to keep the insert order + val inputNodes = new mutable.LinkedHashMap[RexNode, Integer]() + val pythonFunctionInfos = rexCalls.map(createPythonScalarFunctionInfo(_, inputNodes)) + + val udfInputOffsets = inputNodes.toArray + .map(_._1) + .filter(_.isInstanceOf[RexInputRef]) + .map(_.asInstanceOf[RexInputRef].getIndex) + (udfInputOffsets, pythonFunctionInfos) + } + + private[flink] def createPythonScalarFunctionInfo( + rexCall: RexCall, + inputNodes: mutable.Map[RexNode, Integer]): PythonFunctionInfo = rexCall.getOperator match { + case sfc: ScalarSqlFunction if sfc.getScalarFunction.getLanguage == FunctionLanguage.PYTHON => + val inputs = new mutable.ArrayBuffer[AnyRef]() + rexCall.getOperands.foreach { + case pythonRexCall: RexCall if pythonRexCall.getOperator.asInstanceOf[ScalarSqlFunction] + .getScalarFunction.getLanguage == FunctionLanguage.PYTHON => + // Continuous Python UDFs can be chained together + val argPythonInfo = createPythonScalarFunctionInfo(pythonRexCall, inputNodes) + inputs.append(argPythonInfo) + + case literal: RexLiteral => + inputs.append( + convertLiteralToPython.invoke(null, literal, literal.getType.getSqlTypeName)) + + case argNode: RexNode => + // For input arguments of RexInputRef, it's replaced with an offset into the input row + inputNodes.get(argNode) match { + case Some(existing) => inputs.append(existing) + case None => + val inputOffset = Integer.valueOf(inputNodes.size) + inputs.append(inputOffset) + inputNodes.put(argNode, inputOffset) + } + } + + // Extracts the necessary information for Python function execution, such as + // the serialized Python function, the Python env, etc + val pythonFunction = new SimplePythonFunction( + sfc.getScalarFunction.asInstanceOf[PythonFunction].getSerializedPythonFunction, + sfc.getScalarFunction.asInstanceOf[PythonFunction].getPythonEnv) + new PythonFunctionInfo(pythonFunction, inputs.toArray) + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala index fd60bfe99cfc9345880c6e3df112bb63833348ad..7d2aa58a6353ee09d324f3836b69b6db6c317805 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/dataset/DataSetCalc.scala @@ -105,7 +105,6 @@ class DataSetCalc( val genFunction = generateFunction( generator, ruleDescription, - new RowSchema(getInput.getRowType), new RowSchema(getRowType), projection, condition, diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala index 07b53eb9cb49d1d77e95da187f90524ccd36f916..f3e6afa7e4791353381a80aa03afef80e48c3f43 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalc.scala @@ -18,18 +18,15 @@ package org.apache.flink.table.plan.nodes.datastream -import org.apache.calcite.plan.{RelOptCluster, RelOptCost, RelOptPlanner, RelTraitSet} -import org.apache.calcite.rel.`type`.RelDataType +import org.apache.calcite.plan.{RelOptCluster, RelTraitSet} import org.apache.calcite.rel.core.Calc -import org.apache.calcite.rel.metadata.RelMetadataQuery -import org.apache.calcite.rel.{RelNode, RelWriter} +import org.apache.calcite.rel.RelNode import org.apache.calcite.rex.RexProgram import org.apache.flink.streaming.api.datastream.DataStream import org.apache.flink.streaming.api.functions.ProcessFunction import org.apache.flink.table.api.StreamQueryConfig import org.apache.flink.table.calcite.RelTimeIndicatorConverter import org.apache.flink.table.codegen.FunctionCodeGenerator -import org.apache.flink.table.plan.nodes.CommonCalc import org.apache.flink.table.plan.schema.RowSchema import org.apache.flink.table.planner.StreamPlanner import org.apache.flink.table.runtime.CRowProcessRunner @@ -49,11 +46,14 @@ class DataStreamCalc( schema: RowSchema, calcProgram: RexProgram, ruleDescription: String) - extends Calc(cluster, traitSet, input, calcProgram) - with CommonCalc - with DataStreamRel { - - override def deriveRowType(): RelDataType = schema.relDataType + extends DataStreamCalcBase( + cluster, + traitSet, + input, + inputSchema, + schema, + calcProgram, + ruleDescription) { override def copy(traitSet: RelTraitSet, child: RelNode, program: RexProgram): Calc = { new DataStreamCalc( @@ -66,28 +66,6 @@ class DataStreamCalc( ruleDescription) } - override def toString: String = calcToString(calcProgram, getExpressionString) - - override def explainTerms(pw: RelWriter): RelWriter = { - pw.input("input", getInput) - .item("select", selectionToString(calcProgram, getExpressionString)) - .itemIf("where", - conditionToString(calcProgram, getExpressionString), - calcProgram.getCondition != null) - } - - override def computeSelfCost(planner: RelOptPlanner, metadata: RelMetadataQuery): RelOptCost = { - val child = this.getInput - val rowCnt = metadata.getRowCount(child) - computeSelfCost(calcProgram, planner, rowCnt) - } - - override def estimateRowCount(metadata: RelMetadataQuery): Double = { - val child = this.getInput - val rowCnt = metadata.getRowCount(child) - estimateRowCount(calcProgram, rowCnt) - } - override def translateToPlan( planner: StreamPlanner, queryConfig: StreamQueryConfig): DataStream[CRow] = { @@ -117,7 +95,6 @@ class DataStreamCalc( val genFunction = generateFunction( generator, ruleDescription, - inputSchema, schema, projection, condition, diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalcBase.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalcBase.scala new file mode 100644 index 0000000000000000000000000000000000000000..5e62a9f2ddfceb12e33e45022c70563c454b23c2 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamCalcBase.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.plan.nodes.datastream + +import org.apache.calcite.plan.{RelOptCluster, RelOptCost, RelOptPlanner, RelTraitSet} +import org.apache.calcite.rel.`type`.RelDataType +import org.apache.calcite.rel.core.Calc +import org.apache.calcite.rel.metadata.RelMetadataQuery +import org.apache.calcite.rel.{RelNode, RelWriter} +import org.apache.calcite.rex.RexProgram +import org.apache.flink.table.plan.nodes.CommonCalc +import org.apache.flink.table.plan.schema.RowSchema + +/** + * Base RelNode for data stream calc. + */ +abstract class DataStreamCalcBase( + cluster: RelOptCluster, + traitSet: RelTraitSet, + input: RelNode, + inputSchema: RowSchema, + schema: RowSchema, + calcProgram: RexProgram, + ruleDescription: String) + extends Calc(cluster, traitSet, input, calcProgram) + with CommonCalc + with DataStreamRel { + + override def deriveRowType(): RelDataType = schema.relDataType + + override def toString: String = calcToString(calcProgram, getExpressionString) + + override def explainTerms(pw: RelWriter): RelWriter = { + pw.input("input", getInput) + .item("select", selectionToString(calcProgram, getExpressionString)) + .itemIf("where", + conditionToString(calcProgram, getExpressionString), + calcProgram.getCondition != null) + } + + override def computeSelfCost(planner: RelOptPlanner, metadata: RelMetadataQuery): RelOptCost = { + val child = this.getInput + val rowCnt = metadata.getRowCount(child) + computeSelfCost(calcProgram, planner, rowCnt) + } + + override def estimateRowCount(metadata: RelMetadataQuery): Double = { + val child = this.getInput + val rowCnt = metadata.getRowCount(child) + estimateRowCount(calcProgram, rowCnt) + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala index 9d54666a696a70d2c8ef230ad2e43d6ffeb05501..f775303306a5eb51ac504fb65f441e2a03ec0f92 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamMatch.scala @@ -20,7 +20,6 @@ package org.apache.flink.table.plan.nodes.datastream import java.lang.{Boolean => JBoolean, Long => JLong} import java.util - import org.apache.calcite.plan.{RelOptCluster, RelTraitSet} import org.apache.calcite.rel.RelFieldCollation.Direction import org.apache.calcite.rel._ @@ -60,6 +59,8 @@ import org.apache.flink.table.runtime.{RowKeySelector, RowtimeProcessFunction} import org.apache.flink.types.Row import org.apache.flink.util.MathUtils +import org.apache.calcite.util.ImmutableBitSet + /** * Flink RelNode which matches along with LogicalMatch. */ @@ -236,12 +237,10 @@ class DataStreamMatch( } } - private def applyPartitioning(partitionKeys: util.List[RexNode], inputDs: DataStream[Row]) + private def applyPartitioning(partitionKeys: ImmutableBitSet, inputDs: DataStream[Row]) : DataStream[Row] = { if (partitionKeys.size() > 0) { - val keys = partitionKeys.asScala.map { - case ref: RexInputRef => ref.getIndex - }.toArray + val keys = partitionKeys.toArray val keySelector = new RowKeySelector(keys, inputSchema.projectedTypeInfo(keys)) inputDs.keyBy(keySelector) } else { diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala new file mode 100644 index 0000000000000000000000000000000000000000..2a8dce24ef9c9fe51d7b5ee418f21441c6be4c75 --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/datastream/DataStreamPythonCalc.scala @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.plan.nodes.datastream + +import org.apache.calcite.plan.{RelOptCluster, RelTraitSet} +import org.apache.calcite.rel.RelNode +import org.apache.calcite.rel.core.Calc +import org.apache.calcite.rex.{RexCall, RexInputRef, RexLocalRef, RexNode, RexProgram} +import org.apache.flink.api.java.typeutils.RowTypeInfo +import org.apache.flink.streaming.api.datastream.DataStream +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.apache.flink.streaming.api.operators.OneInputStreamOperator +import org.apache.flink.table.api.StreamQueryConfig +import org.apache.flink.table.calcite.FlinkTypeFactory +import org.apache.flink.table.codegen.FunctionCodeGenerator +import org.apache.flink.table.functions.python.PythonFunctionInfo +import org.apache.flink.table.plan.nodes.CommonPythonCalc +import org.apache.flink.table.plan.nodes.datastream.DataStreamPythonCalc.PYTHON_SCALAR_FUNCTION_OPERATOR_NAME +import org.apache.flink.table.plan.schema.RowSchema +import org.apache.flink.table.planner.StreamPlanner +import org.apache.flink.table.runtime.CRowProcessRunner +import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo} +import org.apache.flink.table.types.logical.RowType +import org.apache.flink.table.types.utils.TypeConversions + +import scala.collection.JavaConversions._ + +/** + * RelNode for Python ScalarFunctions. + */ +class DataStreamPythonCalc( + cluster: RelOptCluster, + traitSet: RelTraitSet, + input: RelNode, + inputSchema: RowSchema, + schema: RowSchema, + calcProgram: RexProgram, + ruleDescription: String) + extends DataStreamCalcBase( + cluster, + traitSet, + input, + inputSchema, + schema, + calcProgram, + ruleDescription) + with CommonPythonCalc { + + override def copy(traitSet: RelTraitSet, child: RelNode, program: RexProgram): Calc = { + new DataStreamPythonCalc( + cluster, + traitSet, + child, + inputSchema, + schema, + program, + ruleDescription) + } + + private lazy val pythonRexCalls = calcProgram.getProjectList + .map(calcProgram.expandLocalRef) + .filter(_.isInstanceOf[RexCall]) + .map(_.asInstanceOf[RexCall]) + .toArray + + private lazy val forwardedFields: Array[Int] = calcProgram.getProjectList + .map(calcProgram.expandLocalRef) + .filter(_.isInstanceOf[RexInputRef]) + .map(_.asInstanceOf[RexInputRef].getIndex) + .toArray + + private lazy val (pythonUdfInputOffsets, pythonFunctionInfos) = + extractPythonScalarFunctionInfos(pythonRexCalls) + + private lazy val resultProjectList: Array[RexNode] = { + var idx = 0 + calcProgram.getProjectList + .map(calcProgram.expandLocalRef) + .map { + case pythonCall: RexCall => + val inputRef = new RexInputRef(forwardedFields.length + idx, pythonCall.getType) + idx += 1 + inputRef + case node => node + } + .toArray + } + + override def translateToPlan( + planner: StreamPlanner, + queryConfig: StreamQueryConfig): DataStream[CRow] = { + val config = planner.getConfig + + val inputDataStream = + getInput.asInstanceOf[DataStreamRel].translateToPlan(planner, queryConfig) + + val inputParallelism = inputDataStream.getParallelism + + val pythonOperatorResultTypeInfo = new RowTypeInfo( + forwardedFields.map(inputSchema.fieldTypeInfos.get(_)) ++ + pythonRexCalls.map(node => FlinkTypeFactory.toTypeInfo(node.getType)): _*) + + // Constructs the Python operator + val pythonOperatorInputRowType = TypeConversions.fromLegacyInfoToDataType( + inputSchema.typeInfo).getLogicalType.asInstanceOf[RowType] + val pythonOperatorOutputRowType = TypeConversions.fromLegacyInfoToDataType( + pythonOperatorResultTypeInfo).getLogicalType.asInstanceOf[RowType] + val pythonOperator = getPythonScalarFunctionOperator( + pythonOperatorInputRowType, pythonOperatorOutputRowType, pythonUdfInputOffsets) + + val pythonDataStream = inputDataStream + .transform( + calcOpName(calcProgram, getExpressionString), + CRowTypeInfo(pythonOperatorResultTypeInfo), + pythonOperator) + // keep parallelism to ensure order of accumulate and retract messages + .setParallelism(inputParallelism) + + val generator = new FunctionCodeGenerator( + config, false, pythonOperatorResultTypeInfo) + + val genFunction = generateFunction( + generator, + ruleDescription, + schema, + resultProjectList, + None, + config, + classOf[ProcessFunction[CRow, CRow]]) + + val processFunc = new CRowProcessRunner( + genFunction.name, + genFunction.code, + CRowTypeInfo(schema.typeInfo)) + + pythonDataStream + .process(processFunc) + .name(calcOpName(calcProgram, getExpressionString)) + // keep parallelism to ensure order of accumulate and retract messages + .setParallelism(inputParallelism) + } + + private[flink] def getPythonScalarFunctionOperator( + inputRowType: RowType, + outputRowType: RowType, + udfInputOffsets: Array[Int]) = { + val clazz = Class.forName(PYTHON_SCALAR_FUNCTION_OPERATOR_NAME) + val ctor = clazz.getConstructor( + classOf[Array[PythonFunctionInfo]], + classOf[RowType], + classOf[RowType], + classOf[Array[Int]], + classOf[Array[Int]]) + ctor.newInstance( + pythonFunctionInfos, + inputRowType, + outputRowType, + udfInputOffsets, + forwardedFields) + .asInstanceOf[OneInputStreamOperator[CRow, CRow]] + } +} + +object DataStreamPythonCalc { + val PYTHON_SCALAR_FUNCTION_OPERATOR_NAME = + "org.apache.flink.table.runtime.operators.python.PythonScalarFunctionOperator" +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalMatch.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalMatch.scala index 9e29b105dc83ee88ae6b2f21cb0227516ec722fb..ceef79fd6ca3814e043fad7c4dccb417f466ec2b 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalMatch.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/nodes/logical/FlinkLogicalMatch.scala @@ -19,7 +19,6 @@ package org.apache.flink.table.plan.nodes.logical import java.util - import org.apache.calcite.plan._ import org.apache.calcite.rel.{RelCollation, RelNode} import org.apache.calcite.rel.`type`.RelDataType @@ -29,6 +28,8 @@ import org.apache.calcite.rel.logical.LogicalMatch import org.apache.calcite.rex.RexNode import org.apache.flink.table.plan.nodes.FlinkConventions +import org.apache.calcite.util.ImmutableBitSet + class FlinkLogicalMatch( cluster: RelOptCluster, traitSet: RelTraitSet, @@ -42,7 +43,7 @@ class FlinkLogicalMatch( after: RexNode, subsets: util.Map[String, _ <: util.SortedSet[String]], allRows: Boolean, - partitionKeys: util.List[RexNode], + partitionKeys: ImmutableBitSet, orderKeys: RelCollation, interval: RexNode) extends Match( @@ -63,36 +64,23 @@ class FlinkLogicalMatch( interval) with FlinkLogicalRel { - override def copy( - input: RelNode, - rowType: RelDataType, - pattern: RexNode, - strictStart: Boolean, - strictEnd: Boolean, - patternDefinitions: util.Map[String, RexNode], - measures: util.Map[String, RexNode], - after: RexNode, - subsets: util.Map[String, _ <: util.SortedSet[String]], - allRows: Boolean, - partitionKeys: util.List[RexNode], - orderKeys: RelCollation, - interval: RexNode): Match = { + override def copy(traitSet: RelTraitSet, inputs: util.List[RelNode]): RelNode = { new FlinkLogicalMatch( cluster, traitSet, - input, + inputs.get(0), rowType, pattern, strictStart, strictEnd, - patternDefinitions, - measures, - after, - subsets, + getPatternDefinitions, + getMeasures, + getAfter, + getSubsets, allRows, - partitionKeys, - orderKeys, - interval) + getPartitionKeys, + getOrderKeys, + getInterval) } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala index b7701cdde07517ee2f9b8cad5bcdf51f899395df..c13ed65ea465c8fdc27747818d1336882a428d02 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/FlinkRuleSets.scala @@ -143,6 +143,13 @@ object FlinkRuleSets { FlinkLogicalWindowTableAggregate.CONVERTER ) + /** + * RuleSet to do rewrite on FlinkLogicalRel + */ + val LOGICAL_REWRITE_RULES: RuleSet = RuleSets.ofList( + PythonScalarFunctionSplitRule.INSTANCE + ) + /** * RuleSet to normalize plans for batch / DataSet execution */ @@ -233,7 +240,8 @@ object FlinkRuleSets { StreamTableSourceScanRule.INSTANCE, DataStreamMatchRule.INSTANCE, DataStreamTableAggregateRule.INSTANCE, - DataStreamGroupWindowTableAggregateRule.INSTANCE + DataStreamGroupWindowTableAggregateRule.INSTANCE, + DataStreamPythonCalcRule.INSTANCE ) /** diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetCalcRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetCalcRule.scala index a29889b4d41f56b81813bef0f8888c075c14dc63..c008b22409cc5ff92cb43179572fd785bef252c9 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetCalcRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetCalcRule.scala @@ -43,7 +43,7 @@ class DataSetCalcRule convInput, rel.getRowType, calc.getProgram, - description) + "DataSetCalcRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetCorrelateRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetCorrelateRule.scala index af9b516c1d68333b88532eb6b174fc6cb0fe0726..fc218026e463e5ba29bdd8660d202ff30c18c251 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetCorrelateRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetCorrelateRule.scala @@ -75,7 +75,7 @@ class DataSetCorrelateRule rel.getRowType, join.getRowType, join.getJoinType, - description) + "DataSetCorrelateRule") } } convertToCorrelate(right, None) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetDistinctRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetDistinctRule.scala index a899c6efd590367ad5e850cc036dc40ece6da8e8..4af93c5f2aa2df9ef79ba747747fc0e91e15032d 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetDistinctRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetDistinctRule.scala @@ -52,7 +52,7 @@ class DataSetDistinctRule traitSet, convInput, agg.getRowType, - description) + "DataSetDistinctRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetJoinRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetJoinRule.scala index eded45f7dbc9d813d849f1bec21e68d4c1f9ce9d..d686c9eaa28d793021a70d8119031ac56fbcec23 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetJoinRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetJoinRule.scala @@ -64,7 +64,7 @@ class DataSetJoinRule joinInfo.pairs.toList, join.getJoinType, null, - description) + "DataSetJoinRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetSingleRowJoinRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetSingleRowJoinRule.scala index f964a95f3d52146cb52bc1d9336e684f83a4a571..9461483e6c987048e819770ba9c83b4d297b30d3 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetSingleRowJoinRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetSingleRowJoinRule.scala @@ -81,7 +81,7 @@ class DataSetSingleRowJoinRule join.getCondition, join.getRowType, join.getJoinType, - description) + "DataSetSingleRowJoinRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetValuesRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetValuesRule.scala index 281a875f5ec7bdbd794f8ee4740c01633060e0ca..af0a6dcdaf289d337fa74b7e551a91c9bff3ff7f 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetValuesRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/dataSet/DataSetValuesRule.scala @@ -42,7 +42,7 @@ class DataSetValuesRule traitSet, rel.getRowType, values.getTuples, - description) + "DataSetValuesRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCalcRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCalcRule.scala index 0a1a31a7a5f3e84ff63878734a5d1144786630f6..4ca8913e862cbcfe773c6de5c7a36d848e8333e8 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCalcRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCalcRule.scala @@ -18,21 +18,30 @@ package org.apache.flink.table.plan.rules.datastream -import org.apache.calcite.plan.{RelOptRule, RelTraitSet} +import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet} import org.apache.calcite.rel.RelNode import org.apache.calcite.rel.convert.ConverterRule +import org.apache.flink.table.functions.FunctionLanguage import org.apache.flink.table.plan.nodes.FlinkConventions import org.apache.flink.table.plan.nodes.datastream.DataStreamCalc import org.apache.flink.table.plan.nodes.logical.FlinkLogicalCalc import org.apache.flink.table.plan.schema.RowSchema +import org.apache.flink.table.plan.util.PythonUtil.containsFunctionOf + +import scala.collection.JavaConverters._ class DataStreamCalcRule extends ConverterRule( classOf[FlinkLogicalCalc], FlinkConventions.LOGICAL, FlinkConventions.DATASTREAM, - "DataStreamCalcRule") -{ + "DataStreamCalcRule") { + + override def matches(call: RelOptRuleCall): Boolean = { + val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc] + val program = calc.getProgram + !program.getExprList.asScala.exists(containsFunctionOf(_, FunctionLanguage.PYTHON)) + } def convert(rel: RelNode): RelNode = { val calc: FlinkLogicalCalc = rel.asInstanceOf[FlinkLogicalCalc] @@ -46,7 +55,7 @@ class DataStreamCalcRule new RowSchema(convInput.getRowType), new RowSchema(rel.getRowType), calc.getProgram, - description) + "DataStreamCalcRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCorrelateRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCorrelateRule.scala index ae0370d039aed7b71b23eb1e3c34dec8eea7ea93..954d9ae3792ab0c4260c76fe1455e4bf011f66c9 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCorrelateRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamCorrelateRule.scala @@ -77,7 +77,7 @@ class DataStreamCorrelateRule new RowSchema(rel.getRowType), new RowSchema(join.getRowType), join.getJoinType, - description) + "DataStreamCorrelateRule") } } convertToCorrelate(right, None) diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamJoinRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamJoinRule.scala index 817c95f28a6bca8db957f0af65e8290785ace6eb..56e43d1666c248da0da8782b82891079ea87941c 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamJoinRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamJoinRule.scala @@ -87,7 +87,7 @@ class DataStreamJoinRule leftRowSchema, rightRowSchema, new RowSchema(rel.getRowType), - description) + "DataStreamJoinRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala index 046a49aefdfe66989acbb690d934348ae151ad9d..52d046976f6ef39b4af87ad927801f654e547754 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamMatchRule.scala @@ -18,13 +18,11 @@ package org.apache.flink.table.plan.rules.datastream -import java.util.{List => JList} - import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet} import org.apache.calcite.rel.RelNode import org.apache.calcite.rel.`type`.RelDataType import org.apache.calcite.rel.convert.ConverterRule -import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode} +import org.apache.calcite.rex.{RexCall, RexNode} import org.apache.calcite.sql.SqlAggFunction import org.apache.flink.table.api.{TableException, ValidationException} import org.apache.flink.table.plan.logical.MatchRecognize @@ -35,7 +33,10 @@ import org.apache.flink.table.plan.schema.RowSchema import org.apache.flink.table.plan.util.RexDefaultVisitor import org.apache.flink.table.util.MatchUtil +import org.apache.calcite.util.ImmutableBitSet + import scala.collection.JavaConverters._ +import scala.collection.JavaConversions._ import scala.collection.mutable class DataStreamMatchRule @@ -101,43 +102,32 @@ class DataStreamMatchRule if (logicalMatch.isAllRows) { throw new TableException("All rows per match mode is not supported yet.") } else { - val refNameFinder = new RefNameFinder(logicalMatch.getInput.getRowType) validateAmbiguousColumnsOnRowPerMatch( logicalMatch.getPartitionKeys, logicalMatch.getMeasures.keySet().asScala, - logicalMatch.getRowType, - refNameFinder) + logicalMatch.getInput.getRowType, + logicalMatch.getRowType) } } private def validateAmbiguousColumnsOnRowPerMatch( - partitionKeys: JList[RexNode], + partitionKeys: ImmutableBitSet, measuresNames: mutable.Set[String], - expectedSchema: RelDataType, - refNameFinder: RefNameFinder) + inputSchema: RelDataType, + expectedSchema: RelDataType) : Unit = { - val actualSize = partitionKeys.size() + measuresNames.size + val actualSize = partitionKeys.toArray.length + measuresNames.size val expectedSize = expectedSchema.getFieldCount if (actualSize != expectedSize) { //try to find ambiguous column - val ambiguousColumns = partitionKeys.asScala.map(_.accept(refNameFinder)) + val ambiguousColumns = partitionKeys.toArray.map(inputSchema.getFieldList.get(_).getName) .filter(measuresNames.contains).mkString("{", ", ", "}") throw new ValidationException(s"Columns ambiguously defined: $ambiguousColumns") } } - private class RefNameFinder(inputSchema: RelDataType) extends RexDefaultVisitor[String] { - - override def visitInputRef(inputRef: RexInputRef): String = { - inputSchema.getFieldList.get(inputRef.getIndex).getName - } - - override def visitNode(rexNode: RexNode): String = - throw new TableException(s"PARTITION BY clause accepts only input reference. Found $rexNode") - } - private class AggregationsValidator extends RexDefaultVisitor[Object] { override def visitCall(call: RexCall): AnyRef = { diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamPythonCalcRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamPythonCalcRule.scala new file mode 100644 index 0000000000000000000000000000000000000000..7c618dceb70ec344955f26d1a7b08c7981cd364f --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamPythonCalcRule.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.plan.rules.datastream + +import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall, RelTraitSet} +import org.apache.calcite.rel.RelNode +import org.apache.calcite.rel.convert.ConverterRule +import org.apache.flink.table.functions.FunctionLanguage +import org.apache.flink.table.plan.nodes.FlinkConventions +import org.apache.flink.table.plan.nodes.datastream.DataStreamPythonCalc +import org.apache.flink.table.plan.nodes.logical.FlinkLogicalCalc +import org.apache.flink.table.plan.schema.RowSchema +import org.apache.flink.table.plan.util.PythonUtil.containsFunctionOf + +import scala.collection.JavaConverters._ + +class DataStreamPythonCalcRule + extends ConverterRule( + classOf[FlinkLogicalCalc], + FlinkConventions.LOGICAL, + FlinkConventions.DATASTREAM, + "DataStreamPythonCalcRule") { + + override def matches(call: RelOptRuleCall): Boolean = { + val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc] + val program = calc.getProgram + program.getExprList.asScala.exists(containsFunctionOf(_, FunctionLanguage.PYTHON)) + } + + def convert(rel: RelNode): RelNode = { + val calc: FlinkLogicalCalc = rel.asInstanceOf[FlinkLogicalCalc] + val traitSet: RelTraitSet = rel.getTraitSet.replace(FlinkConventions.DATASTREAM) + val convInput: RelNode = RelOptRule.convert(calc.getInput, FlinkConventions.DATASTREAM) + + new DataStreamPythonCalc( + rel.getCluster, + traitSet, + convInput, + new RowSchema(convInput.getRowType), + new RowSchema(rel.getRowType), + calc.getProgram, + "DataStreamPythonCalcRule") + } +} + +object DataStreamPythonCalcRule { + val INSTANCE: RelOptRule = new DataStreamPythonCalcRule +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamSortRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamSortRule.scala index 0186347231c0b60282aee5e27ea39793d454e4ee..aad1f0f2313beb90e77512afd67506f933732e57 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamSortRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamSortRule.scala @@ -61,7 +61,7 @@ class DataStreamSortRule sort.collation, sort.offset, sort.fetch, - description) + "DataStreamSortRule") } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamTemporalTableJoinRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamTemporalTableJoinRule.scala index 5bdea53b58a27a2eebe32769c80fcd6b893e859c..d57f24674c89968cd4dc3d0c0a1ec059111377e4 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamTemporalTableJoinRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamTemporalTableJoinRule.scala @@ -58,7 +58,7 @@ class DataStreamTemporalTableJoinRule leftRowSchema, rightRowSchema, new RowSchema(rel.getRowType), - description) + "DataStreamTemporalTableJoinRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamValuesRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamValuesRule.scala index a1453a8d1b52a947a168cdc349c42da203c4ba3f..229c0a92c1713181029c06a70dbbe7eb554fdf35 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamValuesRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamValuesRule.scala @@ -43,7 +43,7 @@ class DataStreamValuesRule traitSet, new RowSchema(rel.getRowType), values.getTuples, - description) + "DataStreamValuesRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamWindowJoinRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamWindowJoinRule.scala index 3abaf3f0b76ba9d8ac17050b73c20e6ebfb23568..e50e037f84401d06f3bc458b2f58d05140db272c 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamWindowJoinRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/datastream/DataStreamWindowJoinRule.scala @@ -98,7 +98,7 @@ class DataStreamWindowJoinRule windowBounds.get.leftTimeIdx, windowBounds.get.rightTimeIdx, remainCondition, - description) + "DataStreamWindowJoinRule") } } diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PushFilterIntoTableSourceScanRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PushFilterIntoTableSourceScanRule.scala index 9c16135065be234a94038eedd46d6a31169e6cc3..de1d0e00dd04f16999cfecd48486681acc0e4d93 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PushFilterIntoTableSourceScanRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PushFilterIntoTableSourceScanRule.scala @@ -23,6 +23,7 @@ import java.util import org.apache.calcite.plan.RelOptRule.{none, operand} import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall} import org.apache.calcite.rex.RexProgram +import org.apache.flink.table.api.TableException import org.apache.flink.table.catalog.{CatalogManager, FunctionCatalog, GenericInMemoryCatalog} import org.apache.flink.table.expressions.{Expression, PlannerExpression} import org.apache.flink.table.plan.nodes.logical.{FlinkLogicalCalc, FlinkLogicalTableSourceScan} @@ -83,6 +84,13 @@ class PushFilterIntoTableSourceScanRule extends RelOptRule( val newTableSource = filterableSource.applyPredicate(remainingPredicates) + if (newTableSource.asInstanceOf[FilterableTableSource[_]].isFilterPushedDown + && newTableSource.explainSource().equals(scan.tableSource.explainSource())) { + throw new TableException("Failed to push filter into table source! " + + "table source with pushdown capability must override and change " + + "explainSource() API to explain the pushdown applied!") + } + // check whether framework still need to do a filter val relBuilder = call.builder() val remainingCondition = { diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PushProjectIntoTableSourceScanRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PushProjectIntoTableSourceScanRule.scala index 3ea97abdce40ec5c6d8ab2d0cfdde18c7f60a666..9864cd09298238f92edb347fc414a83d345c1239 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PushProjectIntoTableSourceScanRule.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PushProjectIntoTableSourceScanRule.scala @@ -50,16 +50,23 @@ class PushProjectIntoTableSourceScanRule extends RelOptRule( if (!(0 until scan.getRowType.getFieldCount).toArray.sameElements(accessedLogicalFields)) { // try to push projection of physical fields to TableSource - val newTableSource = source match { + val (newTableSource, isProjectSuccess) = source match { case nested: NestedFieldsProjectableTableSource[_] => val nestedFields = RexProgramExtractor .extractRefNestedInputFields(calc.getProgram, accessedPhysicalFields) - nested.projectNestedFields(accessedPhysicalFields, nestedFields) + (nested.projectNestedFields(accessedPhysicalFields, nestedFields), true) case projecting: ProjectableTableSource[_] => - projecting.projectFields(accessedPhysicalFields) + (projecting.projectFields(accessedPhysicalFields), true) case nonProjecting: TableSource[_] => // projection cannot be pushed to TableSource - nonProjecting + (nonProjecting, false) + } + + if (isProjectSuccess + && newTableSource.explainSource().equals(scan.tableSource.explainSource())) { + throw new TableException("Failed to push project into table source! " + + "table source with pushdown capability must override and change " + + "explainSource() API to explain the pushdown applied!") } // check that table schema of the new table source is identical to original diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PythonScalarFunctionSplitRule.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PythonScalarFunctionSplitRule.scala new file mode 100644 index 0000000000000000000000000000000000000000..deb56c55de8a004ef1f4e0c843269b26e128ed7e --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/rules/logical/PythonScalarFunctionSplitRule.scala @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.plan.rules.logical + +import org.apache.calcite.plan.RelOptRule.{any, operand} +import org.apache.calcite.plan.{RelOptRule, RelOptRuleCall} +import org.apache.calcite.rex.{RexCall, RexInputRef, RexNode, RexProgram} +import org.apache.calcite.sql.validate.SqlValidatorUtil +import org.apache.flink.table.functions.FunctionLanguage +import org.apache.flink.table.functions.ScalarFunction +import org.apache.flink.table.functions.utils.ScalarSqlFunction +import org.apache.flink.table.plan.nodes.logical.FlinkLogicalCalc +import org.apache.flink.table.plan.util.PythonUtil.containsFunctionOf +import org.apache.flink.table.plan.util.{InputRefVisitor, RexDefaultVisitor} + +import scala.collection.JavaConverters._ +import scala.collection.JavaConversions._ +import scala.collection.mutable + +/** + * Rule that splits [[FlinkLogicalCalc]] into multiple [[FlinkLogicalCalc]]s. After this rule + * is applied, each [[FlinkLogicalCalc]] will only contain Python [[ScalarFunction]]s or Java + * [[ScalarFunction]]s. This is to ensure that the Python [[ScalarFunction]]s which could be + * executed in a batch are grouped into the same [[FlinkLogicalCalc]] node. + */ +class PythonScalarFunctionSplitRule extends RelOptRule( + operand(classOf[FlinkLogicalCalc], any), + "PythonScalarFunctionSplitRule") { + + override def matches(call: RelOptRuleCall): Boolean = { + val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc] + val program = calc.getProgram + + // This rule matches if one of the following cases is met: + // 1. There are Python functions and Java functions mixed in the Calc + // 2. There are Python functions in the condition of the Calc + (program.getExprList.exists(containsFunctionOf(_, FunctionLanguage.PYTHON)) && + program.getExprList.exists(containsFunctionOf(_, FunctionLanguage.JVM))) || + Option(program.getCondition) + .map(program.expandLocalRef) + .exists(containsFunctionOf(_, FunctionLanguage.PYTHON)) + } + + override def onMatch(call: RelOptRuleCall): Unit = { + val calc: FlinkLogicalCalc = call.rel(0).asInstanceOf[FlinkLogicalCalc] + val input = calc.getInput + val rexBuilder = call.builder().getRexBuilder + val program = calc.getProgram + val extractedRexCalls = new mutable.ArrayBuffer[RexCall]() + + val convertPythonFunction = + program.getProjectList + .map(program.expandLocalRef) + .exists(containsFunctionOf(_, FunctionLanguage.JVM, recursive = false)) || + Option(program.getCondition) + .map(program.expandLocalRef) + .exists(expr => + containsFunctionOf(expr, FunctionLanguage.JVM, recursive = false) || + containsFunctionOf(expr, FunctionLanguage.PYTHON)) + + val extractedFunctionOffset = input.getRowType.getFieldCount + val splitter = new ScalarFunctionSplitter( + extractedFunctionOffset, + extractedRexCalls, + convertPythonFunction) + + val newProjects = program.getProjectList.map(program.expandLocalRef(_).accept(splitter)) + val newCondition = Option(program.getCondition).map(program.expandLocalRef(_).accept(splitter)) + val accessedFields = extractRefInputFields(newProjects, newCondition, extractedFunctionOffset) + + val bottomCalcProjects = + accessedFields.map(RexInputRef.of(_, input.getRowType)) ++ extractedRexCalls + val bottomCalcFieldNames = SqlValidatorUtil.uniquify( + accessedFields.map(i => input.getRowType.getFieldNames.get(i)).toSeq ++ + extractedRexCalls.indices.map("f" + _), + rexBuilder.getTypeFactory.getTypeSystem.isSchemaCaseSensitive) + + val bottomCalc = new FlinkLogicalCalc( + calc.getCluster, + calc.getTraitSet, + input, + RexProgram.create( + input.getRowType, + bottomCalcProjects.toList, + null, + bottomCalcFieldNames, + rexBuilder)) + + val inputRewriter = new ExtractedFunctionInputRewriter(extractedFunctionOffset, accessedFields) + val topCalc = new FlinkLogicalCalc( + calc.getCluster, + calc.getTraitSet, + bottomCalc, + RexProgram.create( + bottomCalc.getRowType, + newProjects.map(_.accept(inputRewriter)), + newCondition.map(_.accept(inputRewriter)).orNull, + calc.getRowType, + rexBuilder)) + + call.transformTo(topCalc) + } + + /** + * Extracts the indices of the input fields referred by the specified projects and condition. + */ + private def extractRefInputFields( + projects: Seq[RexNode], + condition: Option[RexNode], + inputFieldsCount: Int): Array[Int] = { + val visitor = new InputRefVisitor + + // extract referenced input fields from projections + projects.foreach(exp => exp.accept(visitor)) + + // extract referenced input fields from condition + condition.foreach(_.accept(visitor)) + + // fields of indexes greater than inputFieldsCount is the extracted functions and + // should be filtered as they are not from the original input + visitor.getFields.filter(_ < inputFieldsCount) + } +} + +private class ScalarFunctionSplitter( + extractedFunctionOffset: Int, + extractedRexCalls: mutable.ArrayBuffer[RexCall], + convertPythonFunction: Boolean) + extends RexDefaultVisitor[RexNode] { + + override def visitCall(call: RexCall): RexNode = { + call.getOperator match { + case sfc: ScalarSqlFunction if sfc.getScalarFunction.getLanguage == + FunctionLanguage.PYTHON => + visit(convertPythonFunction, call) + + case _ => + visit(!convertPythonFunction, call) + } + } + + override def visitNode(rexNode: RexNode): RexNode = rexNode + + private def visit(needConvert: Boolean, call: RexCall): RexNode = { + if (needConvert) { + val newNode = new RexInputRef( + extractedFunctionOffset + extractedRexCalls.length, call.getType) + extractedRexCalls.append(call) + newNode + } else { + call.clone(call.getType, call.getOperands.asScala.map(_.accept(this))) + } + } +} + +/** + * Rewrite field accesses of a RexNode as not all the fields from the original input are forwarded: + * 1) Fields of index greater than or equal to extractedFunctionOffset refer to the + * extracted function. + * 2) Fields of index less than extractedFunctionOffset refer to the original input field. + * + * @param extractedFunctionOffset the original start offset of the extracted functions + * @param accessedFields the accessed fields which will be forwarded + */ +private class ExtractedFunctionInputRewriter( + extractedFunctionOffset: Int, + accessedFields: Array[Int]) + extends RexDefaultVisitor[RexNode] { + + /** old input fields ref index -> new input fields ref index mappings */ + private val fieldMap: Map[Int, Int] = accessedFields.zipWithIndex.toMap + + override def visitInputRef(inputRef: RexInputRef): RexNode = { + if (inputRef.getIndex >= extractedFunctionOffset) { + new RexInputRef( + inputRef.getIndex - extractedFunctionOffset + accessedFields.length, + inputRef.getType) + } else { + new RexInputRef( + fieldMap.getOrElse(inputRef.getIndex, + throw new IllegalArgumentException("input field contains invalid index")), + inputRef.getType) + } + } + + override def visitCall(call: RexCall): RexNode = { + call.clone(call.getType, call.getOperands.asScala.map(_.accept(this))) + } + + override def visitNode(rexNode: RexNode): RexNode = rexNode +} + +object PythonScalarFunctionSplitRule { + val INSTANCE: RelOptRule = new PythonScalarFunctionSplitRule +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/util/PythonUtil.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/util/PythonUtil.scala new file mode 100644 index 0000000000000000000000000000000000000000..2e0073cc12ff54d39bb5708c39579720633433ec --- /dev/null +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/util/PythonUtil.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.flink.table.plan.util + +import org.apache.calcite.rex.{RexCall, RexNode} +import org.apache.flink.table.functions.FunctionLanguage +import org.apache.flink.table.functions.utils.ScalarSqlFunction + +import scala.collection.JavaConversions._ + +object PythonUtil { + + /** + * Checks whether it contains the specified kind of function in the specified node. + * + * @param node the RexNode to check + * @param language the expected kind of function to find + * @param recursive whether check the inputs of the specified node + * @return true if it contains the specified kind of function in the specified node. + */ + def containsFunctionOf( + node: RexNode, + language: FunctionLanguage, + recursive: Boolean = true): Boolean = { + node.accept(new FunctionFinder(language, recursive)) + } + + /** + * Checks whether it contains the specified kind of function in a RexNode. + * + * @param expectedLanguage the expected kind of function to find + * @param recursive whether check the inputs + */ + private class FunctionFinder(expectedLanguage: FunctionLanguage, recursive: Boolean) + extends RexDefaultVisitor[Boolean] { + + override def visitCall(call: RexCall): Boolean = { + call.getOperator match { + case sfc: ScalarSqlFunction if sfc.getScalarFunction.getLanguage == + FunctionLanguage.PYTHON => + findInternal(FunctionLanguage.PYTHON, call) + case _ => + findInternal(FunctionLanguage.JVM, call) + } + } + + override def visitNode(rexNode: RexNode): Boolean = false + + private def findInternal(actualLanguage: FunctionLanguage, call: RexCall): Boolean = + actualLanguage == expectedLanguage || + (recursive && call.getOperands.exists(_.accept(this))) + } +} diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala index b771c5e3b9315dccb48fe38033187daff221e343..f2b506fa6d6f5b522f93eb476f680ff5edf96fba 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/runtime/aggregate/GeneratedAggregations.scala @@ -149,7 +149,7 @@ class SingleElementIterable[T] extends java.lang.Iterable[T] { } } - override def remove(): Unit = new java.lang.UnsupportedOperationException + override def remove(): Unit = throw new java.lang.UnsupportedOperationException } val it = new SingleElementIterator diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/util/python/PythonTableUtils.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/util/python/PythonTableUtils.scala index 094945c344bbfc13e8a2ac8f50706831d1d86d7f..a84dad81599bce711f09bba978bf698064139d8d 100644 --- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/util/python/PythonTableUtils.scala +++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/util/python/PythonTableUtils.scala @@ -31,6 +31,9 @@ import org.apache.flink.api.java.io.CollectionInputFormat import org.apache.flink.api.java.typeutils.{MapTypeInfo, ObjectArrayTypeInfo, RowTypeInfo} import org.apache.flink.core.io.InputSplit import org.apache.flink.table.api.{TableSchema, Types} +import org.apache.flink.table.codegen.PythonFunctionCodeGenerator +import org.apache.flink.table.functions.ScalarFunction +import org.apache.flink.table.functions.python.PythonEnv import org.apache.flink.table.sources.InputFormatTableSource import org.apache.flink.types.Row @@ -38,6 +41,32 @@ import scala.collection.JavaConversions._ object PythonTableUtils { + /** + * Creates a [[ScalarFunction]] for the specified Python ScalarFunction. + * + * @param funcName class name of the user-defined function. Must be a valid Java class identifier + * @param serializedScalarFunction serialized Python scalar function + * @param inputTypes input data types + * @param resultType expected result type + * @param deterministic the determinism of the function's results + * @param pythonEnv the Python execution environment + * @return A generated Java ScalarFunction representation for the specified Python ScalarFunction + */ + def createPythonScalarFunction( + funcName: String, + serializedScalarFunction: Array[Byte], + inputTypes: Array[TypeInformation[_]], + resultType: TypeInformation[_], + deterministic: Boolean, + pythonEnv: PythonEnv): ScalarFunction = + PythonFunctionCodeGenerator.generateScalarFunction( + funcName, + serializedScalarFunction, + inputTypes, + resultType, + deterministic, + pythonEnv) + /** * Wrap the unpickled python data with an InputFormat. It will be passed to * PythonInputFormatTableSource later. diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableSourceTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableSourceTest.scala index 5ae51ab5a31bddc36c2bff295b611657af594f14..5b7736442e266f31737bf64c11b6b56b4dfb2e87 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableSourceTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/TableSourceTest.scala @@ -55,10 +55,12 @@ class TableSourceTest extends TableTestBase { batchFilterableSourceTableNode( "table1", Array("name", "id", "amount", "price"), + isPushedDown = true, "'amount > 2"), batchFilterableSourceTableNode( "table2", Array("name", "id", "amount", "price"), + isPushedDown = true, "'amount > 2"), term("all", "true"), term("union", "name, id, amount, price") @@ -161,8 +163,11 @@ class TableSourceTest extends TableTestBase { val expected = unaryNode( "DataSetCalc", - "BatchTableSourceScan(table=[[default_catalog, default_database, filterableTable]], " + - "fields=[price, id, amount])", + batchFilterableSourceTableNode( + tableName, + Array("price", "id", "amount"), + isPushedDown = true, + ""), term("select", "price", "id", "amount"), term("where", "<(*(price, 2), 32)") ) @@ -188,6 +193,7 @@ class TableSourceTest extends TableTestBase { batchFilterableSourceTableNode( tableName, Array("price", "name", "amount"), + isPushedDown = true, "'amount > 2"), term("select", "price", "LOWER(name) AS _c1", "amount"), term("where", "<(*(price, 2), 32)") @@ -211,6 +217,7 @@ class TableSourceTest extends TableTestBase { val expected = batchFilterableSourceTableNode( tableName, Array("price", "id", "amount"), + isPushedDown = true, "'amount > 2 && 'amount < 32") util.verifyTable(result, expected) } @@ -234,6 +241,7 @@ class TableSourceTest extends TableTestBase { batchFilterableSourceTableNode( tableName, Array("price", "id", "amount"), + isPushedDown = true, "'amount > 2"), term("select", "price", "id", "amount"), term("where", "AND(<(id, 1.2E0:DOUBLE), OR(<(amount, 32), >(CAST(amount), 10)))") @@ -261,6 +269,7 @@ class TableSourceTest extends TableTestBase { batchFilterableSourceTableNode( tableName, Array("price", "id", "amount"), + isPushedDown = true, "'amount > 2"), term("select", "price", "id", "amount"), term("where", s"<(${Func0.getClass.getSimpleName}(amount), 32)") @@ -344,6 +353,7 @@ class TableSourceTest extends TableTestBase { streamFilterableSourceTableNode( tableName, Array("price", "id", "amount"), + isPushedDown = true, "'amount > 2"), term("select", "price", "id", "amount"), term("where", "<(*(price, 2), 32)") @@ -446,6 +456,7 @@ class TableSourceTest extends TableTestBase { val expected = batchFilterableSourceTableNode( tableName, Array("id"), + isPushedDown = true, expectedFilter ) util.verifyTable(result, expected) @@ -501,25 +512,27 @@ class TableSourceTest extends TableTestBase { def batchFilterableSourceTableNode( sourceName: String, fields: Array[String], + isPushedDown: Boolean, exp: String) : String = { "BatchTableSourceScan(" + s"table=[[default_catalog, default_database, $sourceName]], fields=[${ fields .mkString(", ") - }], source=[filter=[$exp]])" + }], source=[filterPushedDown=[$isPushedDown], filter=[$exp]])" } def streamFilterableSourceTableNode( sourceName: String, fields: Array[String], + isPushedDown: Boolean, exp: String) : String = { "StreamTableSourceScan(" + s"table=[[default_catalog, default_database, $sourceName]], fields=[${ fields .mkString(", ") - }], source=[filter=[$exp]])" + }], source=[filterPushedDown=[$isPushedDown], filter=[$exp]])" } } diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/DistinctAggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/DistinctAggregateTest.scala index ff5e560b5a68c9424bc13b5c0937107c4c017ae8..d3c279562f2b3636dd3ae753d547d3894693241d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/DistinctAggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/DistinctAggregateTest.scala @@ -57,7 +57,13 @@ class DistinctAggregateTest extends TableTestBase { val sqlQuery = "SELECT COUNT(DISTINCT a), SUM(DISTINCT a), MAX(DISTINCT a) FROM MyTable" - val expected = unaryNode( + val left = unaryNode("DataSetAggregate", + unaryNode("DataSetCalc", + batchTableNode(table), + term("select", "a")), + term("select", "MAX(a) AS EXPR$2")) + + val right = unaryNode( "DataSetAggregate", unaryNode( "DataSetDistinct", @@ -68,9 +74,18 @@ class DistinctAggregateTest extends TableTestBase { ), term("distinct", "a") ), - term("select", "COUNT(a) AS EXPR$0", "SUM(a) AS EXPR$1", "MAX(a) AS EXPR$2") + term("select", "COUNT(a) AS EXPR$0", "SUM(a) AS EXPR$1") ) + val expected = unaryNode("DataSetCalc", + binaryNode("DataSetSingleRowJoin", + left, + right, + term("where", "true"), + term("join", "EXPR$2", "EXPR$0", "EXPR$1"), + term("joinType", "NestedLoopInnerJoin")), + term("select", "EXPR$0", "EXPR$1", "EXPR$2")) + util.verifySql(sqlQuery, expected) } diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/JoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/JoinTest.scala index 8574fcfcc8b52e1153e68dd3e6977a6122d69c5c..5709b256d97217eaf3c5dd5c2f597c7a33e352c8 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/JoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/batch/sql/JoinTest.scala @@ -75,15 +75,15 @@ class JoinTest extends TableTestBase { unaryNode( "DataSetCalc", batchTableNode(table), - term("select", "a", "b", "<(b, 2) AS $f3") + term("select", "a", "b") ), unaryNode( "DataSetCalc", batchTableNode(table1), term("select", "y", "z") ), - term("where", "AND(=(a, z), $f3)"), - term("join", "a", "b", "$f3", "y", "z"), + term("where", "AND(=(a, z), <(b, 2))"), + term("join", "a", "b", "y", "z"), term("joinType", "LeftOuterJoin") ), term("select", "b", "y") @@ -175,10 +175,10 @@ class JoinTest extends TableTestBase { unaryNode( "DataSetCalc", batchTableNode(table1), - term("select", "x", "z", "<(x, 2) AS $f3") + term("select", "x", "z") ), - term("where", "AND(=(a, z), $f3)"), - term("join", "a", "b", "x", "z", "$f3"), + term("where", "AND(=(a, z), <(x, 2))"), + term("join", "a", "b", "x", "z"), term("joinType", "RightOuterJoin") ), term("select", "b", "x") @@ -265,15 +265,15 @@ class JoinTest extends TableTestBase { unaryNode( "DataSetCalc", batchTableNode(table), - term("select", "a", "b", "<(b, 2) AS $f3") + term("select", "a", "b") ), unaryNode( "DataSetCalc", batchTableNode(table1), - term("select", "y", "z", ">(z, 5) AS $f3") + term("select", "y", "z") ), - term("where", "AND(=(a, z), $f3, $f30)"), - term("join", "a", "b", "$f3", "y", "z", "$f30"), + term("where", "AND(=(a, z), <(b, 2), >(z, 5))"), + term("join", "a", "b", "y", "z"), term("joinType", "FullOuterJoin") ), term("select", "b", "y") diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/DistinctAggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/DistinctAggregateTest.scala index 03335319121b97d29ef1787a0649f6683c925bcb..348c408a71d295129c326eda4935b70edf0c4cee 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/DistinctAggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/DistinctAggregateTest.scala @@ -130,7 +130,7 @@ class DistinctAggregateTest extends TableTestBase { ), term("window", "SlidingGroupWindow('w$, 'rowtime, 3600000.millis, 900000.millis)"), term("select", "COUNT(DISTINCT a) AS EXPR$0", "SUM(DISTINCT a) AS EXPR$1", - "MAX(DISTINCT a) AS EXPR$2") + "MAX(a) AS EXPR$2") ) streamUtil.verifySql(sqlQuery, expected) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala index 483b42e04b8c5dede36fc4f513afc19447d175da..89ad5992335547be63bf46f71e7429194c7ae99b 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/sql/JoinTest.scala @@ -848,15 +848,15 @@ class JoinTest extends TableTestBase { unaryNode( "DataStreamCalc", streamTableNode(left), - term("select", "a", "b", "<(b, 2) AS $f3") + term("select", "a", "b") ), unaryNode( "DataStreamCalc", streamTableNode(right), term("select", "y", "z") ), - term("where", "AND(=(a, z), $f3)"), - term("join", "a", "b", "$f3", "y", "z"), + term("where", "AND(=(a, z), <(b, 2))"), + term("join", "a", "b", "y", "z"), term("joinType", "LeftOuterJoin") ), term("select", "b", "y") @@ -948,10 +948,10 @@ class JoinTest extends TableTestBase { unaryNode( "DataStreamCalc", streamTableNode(right), - term("select", "x", "z", "<(x, 2) AS $f3") + term("select", "x", "z") ), - term("where", "AND(=(a, z), $f3)"), - term("join", "a", "b", "x", "z", "$f3"), + term("where", "AND(=(a, z), <(x, 2))"), + term("join", "a", "b", "x", "z"), term("joinType", "RightOuterJoin") ), term("select", "b", "x") diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala index 2501e6321d7f7ed20fedd468ef0d3d12ceddad6b..1250f0d512dc768a45e15c07ab49b492ec9cfa67 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/AggregateTest.scala @@ -283,7 +283,7 @@ class AggregateTest extends TableTestBase { ), term("window", "SlidingGroupWindow('w, 'rowtime, 3600000.millis, 900000.millis)"), term("select", "COUNT(DISTINCT a) AS EXPR$0", "SUM(DISTINCT a) AS EXPR$1", - "MAX(DISTINCT a) AS EXPR$2") + "MAX(a) AS EXPR$2") ) util.verifyTable(result, expected) diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSourceValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSourceValidationTest.scala new file mode 100644 index 0000000000000000000000000000000000000000..11bdf6ebb8eb70858682e452b94722b591505887 --- /dev/null +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/stream/table/validation/TableSourceValidationTest.scala @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.api.stream.table.validation + +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.RowTypeInfo +import org.apache.flink.table.api.scala._ +import org.apache.flink.table.api.{TableException, TableSchema, Types} +import org.apache.flink.table.utils.{TableTestBase, TestFilterableTableSourceWithoutExplainSourceOverride, TestProjectableTableSourceWithoutExplainSourceOverride} +import org.hamcrest.Matchers +import org.junit.Test + +class TableSourceValidationTest extends TableTestBase { + + @Test + def testPushProjectTableSourceWithoutExplainSource(): Unit = { + expectedException.expectCause(Matchers.isA(classOf[TableException])) + + val tableSchema = new TableSchema( + Array("id", "rtime", "val", "ptime", "name"), + Array(Types.INT, Types.SQL_TIMESTAMP, Types.LONG, Types.SQL_TIMESTAMP, Types.STRING)) + val returnType = new RowTypeInfo( + Array(Types.INT, Types.STRING, Types.LONG, Types.LONG) + .asInstanceOf[Array[TypeInformation[_]]], + Array("id", "name", "val", "rtime")) + + val util = streamTestUtil() + util.tableEnv.registerTableSource( + "T", + new TestProjectableTableSourceWithoutExplainSourceOverride( + tableSchema, returnType, Seq(), "rtime", "ptime")) + + val t = util.tableEnv.scan("T").select('name, 'val, 'id) + + // must fail since pushed projection is not explained in source + util.explain(t) + } + + @Test + def testPushFilterableTableSourceWithoutExplainSource(): Unit = { + expectedException.expectCause(Matchers.isA(classOf[TableException])) + + val tableSource = TestFilterableTableSourceWithoutExplainSourceOverride() + val util = batchTestUtil() + + util.tableEnv.registerTableSource("T", tableSource) + + val t = util.tableEnv + .scan("T") + .select('price, 'id, 'amount) + .where("price * 2 < 32") + + // must fail since pushed filter is not explained in source + util.explain(t) + } +} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala index aeb226eb35a80ac62fdb3c82e744e02551741e71..4ba2d5227f55047f0541b3483d6fca9ea635b36d 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/UserDefinedFunctionValidationTest.scala @@ -45,7 +45,8 @@ class UserDefinedFunctionValidationTest extends TableTestBase { thrown.expect(classOf[ValidationException]) thrown.expectMessage( "Given parameters of function do not match any signature. \n" + - "Actual: (java.lang.String, java.lang.Integer) \n" + + "Actual: (org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions" + + ".Accumulator0, java.lang.String, java.lang.Integer) \n" + "Expected: (org.apache.flink.table.runtime.utils.JavaUserDefinedAggFunctions" + ".Accumulator0, long, int)") diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/calcite/CalciteConfigBuilderTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/calcite/CalciteConfigBuilderTest.scala index a7ffa594a989341cb0d0a5f5598c8d6638d7a31c..9dc36065cada59d1aa3df83ebfe633c11aed5d4c 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/calcite/CalciteConfigBuilderTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/calcite/CalciteConfigBuilderTest.scala @@ -47,6 +47,9 @@ class CalciteConfigBuilderTest { assertFalse(cc.replacesDecoRuleSet) assertFalse(cc.decoRuleSet.isDefined) + + assertFalse(cc.replacesLogicalRewriteRuleSet) + assertFalse(cc.logicalRewriteRuleSet.isDefined) } @Test @@ -56,6 +59,7 @@ class CalciteConfigBuilderTest { .addNormRuleSet(RuleSets.ofList(ReduceExpressionsRule.FILTER_INSTANCE)) .replaceLogicalOptRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE)) .replacePhysicalOptRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE)) + .replaceLogicalRewriteRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE)) .replaceDecoRuleSet(RuleSets.ofList(DataStreamRetractionRules.DEFAULT_RETRACTION_INSTANCE)) .build() @@ -65,6 +69,9 @@ class CalciteConfigBuilderTest { assertTrue(cc.replacesLogicalOptRuleSet) assertTrue(cc.logicalOptRuleSet.isDefined) + assertTrue(cc.replacesLogicalOptRuleSet) + assertTrue(cc.logicalRewriteRuleSet.isDefined) + assertTrue(cc.replacesPhysicalOptRuleSet) assertTrue(cc.physicalOptRuleSet.isDefined) @@ -182,6 +189,54 @@ class CalciteConfigBuilderTest { assertTrue(cSet.contains(CalcSplitRule.INSTANCE)) } + @Test + def testReplaceLogicalRewriteRules(): Unit = { + + val cc: CalciteConfig = new CalciteConfigBuilder() + .replaceLogicalRewriteRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE)) + .build() + + assertEquals(true, cc.replacesLogicalRewriteRuleSet) + assertTrue(cc.logicalRewriteRuleSet.isDefined) + val cSet = cc.logicalRewriteRuleSet.get.iterator().asScala.toSet + assertEquals(1, cSet.size) + assertTrue(cSet.contains(FilterMergeRule.INSTANCE)) + } + + @Test + def testReplaceLogicalRewriteAddRules(): Unit = { + + val cc: CalciteConfig = new CalciteConfigBuilder() + .replaceLogicalRewriteRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE)) + .addLogicalRewriteRuleSet(RuleSets.ofList(CalcMergeRule.INSTANCE, CalcSplitRule.INSTANCE)) + .build() + + assertEquals(true, cc.replacesLogicalRewriteRuleSet) + assertTrue(cc.logicalRewriteRuleSet.isDefined) + val cSet = cc.logicalRewriteRuleSet.get.iterator().asScala.toSet + assertEquals(3, cSet.size) + assertTrue(cSet.contains(FilterMergeRule.INSTANCE)) + assertTrue(cSet.contains(CalcMergeRule.INSTANCE)) + assertTrue(cSet.contains(CalcSplitRule.INSTANCE)) + } + + @Test + def testAddLogicalRewriteRules(): Unit = { + + val cc: CalciteConfig = new CalciteConfigBuilder() + .addLogicalRewriteRuleSet(RuleSets.ofList(FilterMergeRule.INSTANCE)) + .addLogicalRewriteRuleSet(RuleSets.ofList(CalcMergeRule.INSTANCE, CalcSplitRule.INSTANCE)) + .build() + + assertEquals(false, cc.replacesLogicalRewriteRuleSet) + assertTrue(cc.logicalRewriteRuleSet.isDefined) + val cSet = cc.logicalRewriteRuleSet.get.iterator().asScala.toSet + assertEquals(3, cSet.size) + assertTrue(cSet.contains(FilterMergeRule.INSTANCE)) + assertTrue(cSet.contains(CalcMergeRule.INSTANCE)) + assertTrue(cSet.contains(CalcSplitRule.INSTANCE)) + } + @Test def testReplacePhysicalOptimizationRules(): Unit = { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/ExpressionReductionRulesTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/ExpressionReductionRulesTest.scala index f3cc8caf4f554fa4d05477c12bdb3aaac7a2e8c0..ce29bed319a3a650ac70eda0035d45401959dcf5 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/ExpressionReductionRulesTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/ExpressionReductionRulesTest.scala @@ -21,7 +21,7 @@ package org.apache.flink.table.plan import org.apache.flink.api.scala._ import org.apache.flink.table.api.Types import org.apache.flink.table.api.scala._ -import org.apache.flink.table.functions.ScalarFunction +import org.apache.flink.table.functions.{FunctionLanguage, ScalarFunction} import org.apache.flink.table.utils.TableTestBase import org.apache.flink.table.utils.TableTestUtil._ import org.junit.{Ignore, Test} @@ -499,6 +499,24 @@ class ExpressionReductionRulesTest extends TableTestBase { util.verifyTable(result, expected) } + + @Test + def testReduceDeterministicPythonUDF(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Long, String)]("MyTable", 'a, 'b, 'c) + + val result = table + .select('a, 'b, 'c, DeterministicPythonFunc() as 'd, DeterministicNullFunc() as 'e) + + val expected: String = unaryNode( + "DataStreamPythonCalc", + streamTableNode(table), + term("select", "a", "b", "c", "DeterministicPythonFunc$() AS d", + "null:VARCHAR(65536) AS e") + ) + + util.verifyTable(result, expected) + } } object NonDeterministicNullFunc extends ScalarFunction { @@ -510,3 +528,12 @@ object DeterministicNullFunc extends ScalarFunction { def eval(): String = null override def isDeterministic = true } + +object DeterministicPythonFunc extends ScalarFunction { + + override def getLanguage: FunctionLanguage = FunctionLanguage.PYTHON + + def eval(): Long = 1L + + override def isDeterministic = true +} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonScalarFunctionSplitRuleTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonScalarFunctionSplitRuleTest.scala new file mode 100644 index 0000000000000000000000000000000000000000..3db9398733376aec7331ba65d1d30ecef67d9ecd --- /dev/null +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/plan/PythonScalarFunctionSplitRuleTest.scala @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.table.plan + +import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation} +import org.apache.flink.api.scala._ +import org.apache.flink.table.api.scala._ +import org.apache.flink.table.functions.{FunctionLanguage, ScalarFunction} +import org.apache.flink.table.utils.TableTestUtil._ +import org.apache.flink.table.utils.TableTestBase +import org.junit.Test + +class PythonScalarFunctionSplitRuleTest extends TableTestBase { + + @Test + def testPythonFunctionAsInputOfJavaFunction(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Int, Int)]("MyTable", 'a, 'b, 'c) + util.tableEnv.registerFunction("pyFunc1", new PythonScalarFunction("pyFunc1")) + + val resultTable = table + .select("pyFunc1(a, b) + 1") + + val expected = unaryNode( + "DataStreamCalc", + unaryNode( + "DataStreamPythonCalc", + streamTableNode(table), + term("select", "pyFunc1(a, b) AS f0") + ), + term("select", "+(f0, 1) AS _c0") + ) + + util.verifyTable(resultTable, expected) + } + + @Test + def testPythonFunctionMixedWithJavaFunction(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Int, Int)]("MyTable", 'a, 'b, 'c) + util.tableEnv.registerFunction("pyFunc1", new PythonScalarFunction("pyFunc1")) + + val resultTable = table + .select("pyFunc1(a, b), c + 1") + + val expected = unaryNode( + "DataStreamCalc", + unaryNode( + "DataStreamPythonCalc", + streamTableNode(table), + term("select", "c", "pyFunc1(a, b) AS f0") + ), + term("select", "f0 AS _c0", "+(c, 1) AS _c1") + ) + + util.verifyTable(resultTable, expected) + } + + @Test + def testPythonFunctionMixedWithJavaFunctionInWhereClause(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Int, Int)]("MyTable", 'a, 'b, 'c) + util.tableEnv.registerFunction("pyFunc1", new PythonScalarFunction("pyFunc1")) + util.tableEnv.registerFunction("pyFunc2", new PythonScalarFunction("pyFunc2")) + + val resultTable = table + .where("pyFunc2(a, c) > 0") + .select("pyFunc1(a, b), c + 1") + + val expected = unaryNode( + "DataStreamCalc", + unaryNode( + "DataStreamPythonCalc", + streamTableNode(table), + term("select", "c", "pyFunc1(a, b) AS f0", "pyFunc2(a, c) AS f1") + ), + term("select", "f0 AS _c0", "+(c, 1) AS _c1"), + term("where", ">(f1, 0)") + ) + + util.verifyTable(resultTable, expected) + } + + @Test + def testPythonFunctionInWhereClause(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Int, Int)]("MyTable", 'a, 'b, 'c) + util.tableEnv.registerFunction("pyFunc1", new PythonScalarFunction("pyFunc1")) + util.tableEnv.registerFunction("pyFunc2", new BooleanPythonScalarFunction("pyFunc2")) + + val resultTable = table + .where("pyFunc2(a, c)") + .select("pyFunc1(a, b)") + + val expected = unaryNode( + "DataStreamCalc", + unaryNode( + "DataStreamPythonCalc", + streamTableNode(table), + term("select", "pyFunc1(a, b) AS f0", "pyFunc2(a, c) AS f1") + ), + term("select", "f0 AS _c0"), + term("where", "f1") + ) + + util.verifyTable(resultTable, expected) + } + + @Test + def testChainingPythonFunction(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Int, Int)]("MyTable", 'a, 'b, 'c) + util.tableEnv.registerFunction("pyFunc1", new PythonScalarFunction("pyFunc1")) + util.tableEnv.registerFunction("pyFunc2", new PythonScalarFunction("pyFunc2")) + util.tableEnv.registerFunction("pyFunc3", new PythonScalarFunction("pyFunc3")) + + val resultTable = table + .select("pyFunc3(pyFunc2(a + pyFunc1(a, c), b), c)") + + val expected = unaryNode( + "DataStreamPythonCalc", + unaryNode( + "DataStreamCalc", + unaryNode( + "DataStreamPythonCalc", + streamTableNode(table), + term("select", "b", "c", "a", "pyFunc1(a, c) AS f0") + ), + term("select", "b", "c", "+(a, f0) AS f0") + ), + term("select", "pyFunc3(pyFunc2(f0, b), c) AS _c0") + ) + + util.verifyTable(resultTable, expected) + } + + @Test + def testOnlyOnePythonFunction(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Int, Int)]("MyTable", 'a, 'b, 'c) + util.tableEnv.registerFunction("pyFunc1", new PythonScalarFunction("pyFunc1")) + + val resultTable = table + .select("pyFunc1(a, b)") + + val expected = unaryNode( + "DataStreamPythonCalc", + streamTableNode(table), + term("select", "pyFunc1(a, b) AS _c0") + ) + + util.verifyTable(resultTable, expected) + } + + @Test + def testOnlyOnePythonFunctionInWhereClause(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Int, Int)]("MyTable", 'a, 'b, 'c) + util.tableEnv.registerFunction("pyFunc1", new BooleanPythonScalarFunction("pyFunc1")) + + val resultTable = table + .where("pyFunc1(a, c)") + .select("a, b") + + val expected = unaryNode( + "DataStreamCalc", + unaryNode( + "DataStreamPythonCalc", + streamTableNode(table), + term("select", "a", "b", "pyFunc1(a, c) AS f0") + ), + term("select", "a", "b"), + term("where", "f0") + ) + + util.verifyTable(resultTable, expected) + } + + @Test + def testFieldNameUniquify(): Unit = { + val util = streamTestUtil() + val table = util.addTable[(Int, Int, Int)]("MyTable", 'f0, 'f1, 'f2) + util.tableEnv.registerFunction("pyFunc1", new PythonScalarFunction("pyFunc1")) + + val resultTable = table + .select("pyFunc1(f1, f2), f0 + 1") + + val expected = unaryNode( + "DataStreamCalc", + unaryNode( + "DataStreamPythonCalc", + streamTableNode(table), + term("select", "f0", "pyFunc1(f1, f2) AS f00") + ), + term("select", "f00 AS _c0", "+(f0, 1) AS _c1") + ) + + util.verifyTable(resultTable, expected) + } +} + +class PythonScalarFunction(name: String) extends ScalarFunction { + def eval(i: Int, j: Int): Int = i + j + + override def getResultType(signature: Array[Class[_]]): TypeInformation[_] = + BasicTypeInfo.INT_TYPE_INFO + + override def getLanguage: FunctionLanguage = FunctionLanguage.PYTHON + + override def toString: String = name +} + +class BooleanPythonScalarFunction(name: String) extends ScalarFunction { + def eval(i: Int, j: Int): Boolean = i + j > 1 + + override def getResultType(signature: Array[Class[_]]): TypeInformation[_] = + BasicTypeInfo.BOOLEAN_TYPE_INFO + + override def getLanguage: FunctionLanguage = FunctionLanguage.PYTHON + + override def toString: String = name +} diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala index 94c22e675f0b385ed3db2392683957cc186f51f9..488145c70b54883b397ec7d2e4477da2ddf71a53 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/runtime/harness/TemporalJoinHarnessTest.scala @@ -19,7 +19,6 @@ package org.apache.flink.table.runtime.harness import java.util import java.util.concurrent.ConcurrentLinkedQueue - import org.apache.calcite.rel.core.{JoinInfo, JoinRelType} import org.apache.calcite.rex.{RexBuilder, RexNode} import org.apache.calcite.sql.fun.SqlStdOperatorTable @@ -42,6 +41,8 @@ import org.apache.flink.table.runtime.CRowKeySelector import org.apache.flink.table.runtime.harness.HarnessTestBase.{RowResultSortComparator, TestStreamQueryConfig} import org.apache.flink.table.runtime.types.CRow import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo + +import com.google.common.collect.ImmutableList import org.hamcrest.CoreMatchers import org.hamcrest.Matchers.{endsWith, startsWith} import org.junit.Assert.assertTrue @@ -768,7 +769,7 @@ class TemporalJoinHarnessTest extends HarnessTestBase { val rightRowType: RowTypeInfo, leftKeys: ImmutableIntList, rightKeys: ImmutableIntList) - extends JoinInfo(leftKeys, rightKeys) { + extends JoinInfo(leftKeys, rightKeys, ImmutableList.of[RexNode]()) { def this( leftRowType: RowTypeInfo, diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/TestFilterableTableSource.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/TestFilterableTableSource.scala index 4f767f69949d56d8bd6f70887779cb8f2a1fbc9a..79f5f3297860851ca62b5baeccc25431dc749fec 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/TestFilterableTableSource.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/TestFilterableTableSource.scala @@ -80,6 +80,53 @@ object TestFilterableTableSource { } } +object TestFilterableTableSourceWithoutExplainSourceOverride{ + + /** + * @return The default filterable table source. + */ + def apply(): TestFilterableTableSourceWithoutExplainSourceOverride = { + apply(defaultTypeInfo, defaultRows, defaultFilterableFields) + } + + /** + * A filterable data source with custom data. + * @param rowTypeInfo The type of the data. Its expected that both types and field + * names are provided. + * @param rows The data as a sequence of rows. + * @param filterableFields The fields that are allowed to be filtered on. + * @return The table source. + */ + def apply( + rowTypeInfo: RowTypeInfo, + rows: Seq[Row], + filterableFields: Set[String]) + : TestFilterableTableSourceWithoutExplainSourceOverride = { + new TestFilterableTableSourceWithoutExplainSourceOverride(rowTypeInfo, rows, filterableFields) + } + + private lazy val defaultFilterableFields = Set("amount") + + private lazy val defaultTypeInfo: RowTypeInfo = { + val fieldNames: Array[String] = Array("name", "id", "amount", "price") + val fieldTypes: Array[TypeInformation[_]] = Array(STRING, LONG, INT, DOUBLE) + new RowTypeInfo(fieldTypes, fieldNames) + } + + private lazy val defaultRows: Seq[Row] = { + for { + cnt <- 0 until 33 + } yield { + Row.of( + s"Record_$cnt", + cnt.toLong.asInstanceOf[AnyRef], + cnt.toInt.asInstanceOf[AnyRef], + cnt.toDouble.asInstanceOf[AnyRef]) + } + } +} + + /** * A data source that implements some very basic filtering in-memory in order to test * expression push-down logic. @@ -91,6 +138,53 @@ object TestFilterableTableSource { * @param filterPushedDown Whether predicates have been pushed down yet. */ class TestFilterableTableSource( + rowTypeInfo: RowTypeInfo, + data: Seq[Row], + filterableFields: Set[String] = Set(), + filterPredicates: Seq[Expression] = Seq(), + filterPushedDown: Boolean = false) + extends TestFilterableTableSourceWithoutExplainSourceOverride( + rowTypeInfo, + data, + filterableFields, + filterPredicates, + filterPushedDown + ) { + + override def applyPredicate(predicates: JList[Expression]): TableSource[Row] = { + val predicatesToUse = new mutable.ListBuffer[Expression]() + val iterator = predicates.iterator() + while (iterator.hasNext) { + val expr = iterator.next() + if (shouldPushDown(expr)) { + predicatesToUse += expr + iterator.remove() + } + } + + new TestFilterableTableSource( + rowTypeInfo, + data, + filterableFields, + predicatesToUse, + filterPushedDown = true) + } + + override def explainSource(): String = { + if (filterPredicates.nonEmpty) { + // TODO we cast to planner expression as a temporary solution to keep the old interfaces + s"filterPushedDown=[$filterPushedDown], filter=[${filterPredicates.reduce((l, r) => + And(l.asInstanceOf[PlannerExpression], r.asInstanceOf[PlannerExpression])).toString}]" + } else { + s"filterPushedDown=[$filterPushedDown], filter=[]" + } + } +} + +/** + * A [[TestFilterableTableSource]] without explain source override. + */ +class TestFilterableTableSourceWithoutExplainSourceOverride( rowTypeInfo: RowTypeInfo, data: Seq[Row], filterableFields: Set[String] = Set(), @@ -115,16 +209,6 @@ class TestFilterableTableSource( execEnv.fromCollection[Row](applyPredicatesToRows(data).asJava, getReturnType) } - override def explainSource(): String = { - if (filterPredicates.nonEmpty) { - // TODO we cast to planner expression as a temporary solution to keep the old interfaces - s"filter=[${filterPredicates.reduce((l, r) => - And(l.asInstanceOf[PlannerExpression], r.asInstanceOf[PlannerExpression])).toString}]" - } else { - "" - } - } - override def getReturnType: TypeInformation[Row] = rowTypeInfo override def applyPredicate(predicates: JList[Expression]): TableSource[Row] = { @@ -138,7 +222,7 @@ class TestFilterableTableSource( } } - new TestFilterableTableSource( + new TestFilterableTableSourceWithoutExplainSourceOverride( rowTypeInfo, data, filterableFields, @@ -148,18 +232,18 @@ class TestFilterableTableSource( override def isFilterPushedDown: Boolean = filterPushedDown - private def applyPredicatesToRows(rows: Seq[Row]): Seq[Row] = { + private[flink] def applyPredicatesToRows(rows: Seq[Row]): Seq[Row] = { rows.filter(shouldKeep) } - private def shouldPushDown(expr: Expression): Boolean = { + private[flink] def shouldPushDown(expr: Expression): Boolean = { expr match { case binExpr: BinaryComparison => shouldPushDown(binExpr) case _ => false } } - private def shouldPushDown(expr: BinaryComparison): Boolean = { + private[flink] def shouldPushDown(expr: BinaryComparison): Boolean = { (expr.left, expr.right) match { case (f: PlannerResolvedFieldReference, v: Literal) => filterableFields.contains(f.name) @@ -171,14 +255,14 @@ class TestFilterableTableSource( } } - private def shouldKeep(row: Row): Boolean = { + private[flink] def shouldKeep(row: Row): Boolean = { filterPredicates.isEmpty || filterPredicates.forall { case expr: BinaryComparison => binaryFilterApplies(expr, row) case expr => throw new RuntimeException(expr + " not supported!") } } - private def binaryFilterApplies(expr: BinaryComparison, row: Row): Boolean = { + private[flink] def binaryFilterApplies(expr: BinaryComparison, row: Row): Boolean = { val (lhsValue, rhsValue) = extractValues(expr, row) expr match { @@ -197,7 +281,7 @@ class TestFilterableTableSource( } } - private def extractValues(expr: BinaryComparison, row: Row) + private[flink] def extractValues(expr: BinaryComparison, row: Row) : (Comparable[Any], Comparable[Any]) = { (expr.left, expr.right) match { diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/testTableSources.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/testTableSources.scala index 6f6615875918a70806eb72aff73354699f7f0ec8..6a6d691bd849f518937fde6332fa18fd115f34c3 100644 --- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/testTableSources.scala +++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/utils/testTableSources.scala @@ -94,15 +94,13 @@ class TestProjectableTableSource( rowtime: String = null, proctime: String = null, fieldMapping: Map[String, String] = null) - extends TestTableSourceWithTime[Row]( + extends TestProjectableTableSourceWithoutExplainSourceOverride( tableSchema, returnType, values, rowtime, proctime, - fieldMapping) - with ProjectableTableSource[Row] { - + fieldMapping) { override def projectFields(fields: Array[Int]): TableSource[Row] = { val rowType = returnType.asInstanceOf[RowTypeInfo] @@ -148,6 +146,62 @@ class TestProjectableTableSource( } } +class TestProjectableTableSourceWithoutExplainSourceOverride( + tableSchema: TableSchema, + returnType: TypeInformation[Row], + values: Seq[Row], + rowtime: String = null, + proctime: String = null, + fieldMapping: Map[String, String] = null) + extends TestTableSourceWithTime[Row]( + tableSchema, + returnType, + values, + rowtime, + proctime, + fieldMapping) + with ProjectableTableSource[Row] { + + override def projectFields(fields: Array[Int]): TableSource[Row] = { + + val rowType = returnType.asInstanceOf[RowTypeInfo] + + val (projectedNames: Array[String], projectedMapping) = if (fieldMapping == null) { + val projectedNames = fields.map(rowType.getFieldNames.apply(_)) + (projectedNames, null) + } else { + val invertedMapping = fieldMapping.map(_.swap) + val projectedNames = fields.map(rowType.getFieldNames.apply(_)) + + val projectedMapping: Map[String, String] = projectedNames.map{ f => + val logField = invertedMapping(f) + logField -> s"remapped-$f" + }.toMap + val renamedNames = projectedNames.map(f => s"remapped-$f") + (renamedNames, projectedMapping) + } + + val projectedTypes = fields.map(rowType.getFieldTypes.apply(_)) + val projectedReturnType = new RowTypeInfo( + projectedTypes.asInstanceOf[Array[TypeInformation[_]]], + projectedNames) + + val projectedValues = values.map { fromRow => + val pRow = new Row(fields.length) + fields.zipWithIndex.foreach{ case (from, to) => pRow.setField(to, fromRow.getField(from)) } + pRow + } + + new TestProjectableTableSourceWithoutExplainSourceOverride( + tableSchema, + projectedReturnType, + projectedValues, + rowtime, + proctime, + projectedMapping) + } +} + class TestNestedProjectableTableSource( tableSchema: TableSchema, returnType: TypeInformation[Row], diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/functions/SqlDateTimeUtils.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/functions/SqlDateTimeUtils.java index a6784bbfd6fb2b93764f8709edbdb124f7d95a33..e7f1422a94ebbd46362abcf4a469de5de5ae32cf 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/functions/SqlDateTimeUtils.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/functions/SqlDateTimeUtils.java @@ -77,7 +77,7 @@ public class SqlDateTimeUtils { *

    This is the modulo 'mask' used when converting * TIMESTAMP values to DATE and TIME values. */ - private static final long MILLIS_PER_DAY = 86400000; // = 24 * 60 * 60 * 1000 + private static final long MILLIS_PER_DAY = 86400000L; // = 24 * 60 * 60 * 1000 /** The SimpleDateFormat string for ISO dates, "yyyy-MM-dd". */ private static final String DATE_FORMAT_STRING = "yyyy-MM-dd"; diff --git a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/operators/rank/RetractableTopNFunction.java b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/operators/rank/RetractableTopNFunction.java index 11820e0a2623a22eee5480ea50f023b38c5548d4..5ed2c7b26446a073cb472f65e8fe317a92ec000e 100644 --- a/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/operators/rank/RetractableTopNFunction.java +++ b/flink-table/flink-table-runtime-blink/src/main/java/org/apache/flink/table/runtime/operators/rank/RetractableTopNFunction.java @@ -117,6 +117,9 @@ public class RetractableTopNFunction extends AbstractTopNFunction { @Override public void processElement(BaseRow input, Context ctx, Collector out) throws Exception { + long currentTime = ctx.timerService().currentProcessingTime(); + // register state-cleanup timer + registerProcessingCleanupTimer(ctx, currentTime); initRankEnd(input); SortedMap sortedMap = treeMap.value(); if (sortedMap == null) { @@ -171,6 +174,13 @@ public class RetractableTopNFunction extends AbstractTopNFunction { treeMap.update(sortedMap); } + @Override + public void onTimer(long timestamp, OnTimerContext ctx, Collector out) throws Exception { + if (stateCleaningEnabled) { + cleanupState(dataState, treeMap); + } + } + // ------------- ROW_NUMBER------------------------------- private void retractRecordWithRowNumber( diff --git a/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/runtime/operators/rank/RetractableTopNFunctionTest.java b/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/runtime/operators/rank/RetractableTopNFunctionTest.java index e404b4186b68acc60d689d45360d12d2563330d2..f852ca13843ec73d17bed4064dc84f48df0c7f90 100644 --- a/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/runtime/operators/rank/RetractableTopNFunctionTest.java +++ b/flink-table/flink-table-runtime-blink/src/test/java/org/apache/flink/table/runtime/operators/rank/RetractableTopNFunctionTest.java @@ -219,4 +219,38 @@ public class RetractableTopNFunctionTest extends TopNFunctionTestBase { .assertOutputEqualsSorted("output wrong.", expectedOutput, testHarness.getOutput()); } + @Test + public void testCleanIdleState() throws Exception { + AbstractTopNFunction func = createFunction(RankType.ROW_NUMBER, new ConstantRankRange(1, 2), true, + true); + OneInputStreamOperatorTestHarness testHarness = createTestHarness(func); + testHarness.open(); + // register cleanup timer with 20L + testHarness.setProcessingTime(0L); + testHarness.processElement(record("book", 1L, 12)); + testHarness.processElement(record("fruit", 5L, 22)); + + // register cleanup timer with 29L + testHarness.setProcessingTime(9L); + testHarness.processElement(retractRecord("book", 1L, 12)); + testHarness.processElement(record("fruit", 4L, 11)); + + // trigger the first cleanup timer and register cleanup timer with 4000 + testHarness.setProcessingTime(20L); + testHarness.processElement(record("fruit", 8L, 100)); + testHarness.processElement(record("book", 1L, 12)); + testHarness.close(); + + List expectedOutput = new ArrayList<>(); + expectedOutput.add(record("book", 1L, 12, 1L)); + expectedOutput.add(record("fruit", 5L, 22, 1L)); + expectedOutput.add(deleteRecord("book", 1L, 12, 1L)); + expectedOutput.add(deleteRecord("fruit", 5L, 22, 1L)); + expectedOutput.add(record("fruit", 5L, 22, 2L)); + expectedOutput.add(record("fruit", 4L, 11, 1L)); + // after idle state expired + expectedOutput.add(record("fruit", 8L, 100, 1L)); + expectedOutput.add(record("book", 1L, 12, 1L)); + assertorWithRowNumber.assertOutputEqualsSorted("output wrong.", expectedOutput, testHarness.getOutput()); + } } diff --git a/flink-table/pom.xml b/flink-table/pom.xml index 1e479b35239b99ebadcd3e622f8ba11ca5c8dc8b..400cd17a3e50108c5dd3c132d2a5f131b795920c 100644 --- a/flink-table/pom.xml +++ b/flink-table/pom.xml @@ -50,5 +50,6 @@ under the License. 3.0.9 + 1.21.0 diff --git a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java index 8050dba4917df7e6a7f4fe92d709c24722fd1cbe..d656184e785b424c3c21e00ecc7353e529d478da 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/accumulators/AccumulatorErrorITCase.java @@ -58,7 +58,7 @@ public class AccumulatorErrorITCase extends TestLogger { public static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "12m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "12m"); return config; } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java index 490926fa637514177709181cc84be0ef9c8a7d27..c09943ff67db2a409e23c0fbae522b7c02970e7c 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeAllWindowCheckpointingITCase.java @@ -67,7 +67,7 @@ public class EventTimeAllWindowCheckpointingITCase extends TestLogger { private static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "48m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "48m"); config.setString(AkkaOptions.LOOKUP_TIMEOUT, "60 s"); config.setString(AkkaOptions.ASK_TIMEOUT, "60 s"); return config; diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java index 9eacee8afaa6952011aed1fc3e4355af166f787c..de681255c4835fb7d870305bc4aa548562a292be 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/EventTimeWindowCheckpointingITCase.java @@ -209,7 +209,7 @@ public class EventTimeWindowCheckpointingITCase extends TestLogger { final File haDir = temporaryFolder.newFolder(); Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "48m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "48m"); // the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(80L << 20)); // 80 MB config.setString(AkkaOptions.FRAMESIZE, String.valueOf(MAX_MEM_STATE_SIZE) + "b"); diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java index 0bc50239618dc2152a446a97d7fe3eb2ec19e7cc..9ec23d3433e01c10dfc07521a7fdfdb82cae6914 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/KeyedStateCheckpointingITCase.java @@ -88,7 +88,7 @@ public class KeyedStateCheckpointingITCase extends TestLogger { private static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "12m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "12m"); return config; } diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java index 37f0c337651a0360a27dc77f35a968cffa02dce9..2f4a8572dd059505596e3850a46a6a17fe5897ed 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/SavepointITCase.java @@ -664,7 +664,7 @@ public class SavepointITCase extends TestLogger { Configuration config = getFileBasedCheckpointsConfig(); config.addAll(jobGraph.getJobConfiguration()); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "0"); MiniClusterWithClientResource cluster = new MiniClusterWithClientResource( new MiniClusterResourceConfiguration.Builder() diff --git a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java index 8cce8009c929da78ae71653cdd7351534a4199a4..c722311cff2da7c2ce4813e35c9f8bdaeef38833 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/checkpointing/WindowCheckpointingITCase.java @@ -80,7 +80,7 @@ public class WindowCheckpointingITCase extends TestLogger { private static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "48m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "48m"); return config; } diff --git a/flink-tests/src/test/java/org/apache/flink/test/classloading/ClassLoaderITCase.java b/flink-tests/src/test/java/org/apache/flink/test/classloading/ClassLoaderITCase.java index cd421f39ec1dffe895103b146192ed8655d2abe7..a09142e7ee141c2e100c6f36d8f78a41546bab1b 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/classloading/ClassLoaderITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/classloading/ClassLoaderITCase.java @@ -36,7 +36,6 @@ import org.apache.flink.streaming.util.TestStreamEnvironment; import org.apache.flink.test.testdata.KMeansData; import org.apache.flink.test.util.SuccessException; import org.apache.flink.test.util.TestEnvironment; -import org.apache.flink.testutils.junit.FailsOnJava11; import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.TestLogger; @@ -48,7 +47,6 @@ import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; @@ -118,7 +116,7 @@ public class ClassLoaderITCase extends TestLogger { FOLDER.newFolder().getAbsoluteFile().toURI().toString()); // required as we otherwise run out of memory - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "80m"); miniClusterResource = new MiniClusterResource( new MiniClusterResourceConfiguration.Builder() @@ -144,7 +142,7 @@ public class ClassLoaderITCase extends TestLogger { } @Test - public void testCustomSplitJobWithCustomClassLoaderJar() throws IOException, ProgramInvocationException { + public void testCustomSplitJobWithCustomClassLoaderJar() throws ProgramInvocationException { PackagedProgram inputSplitTestProg = new PackagedProgram(new File(INPUT_SPLITS_PROG_JAR_FILE)); @@ -152,20 +150,20 @@ public class ClassLoaderITCase extends TestLogger { miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(INPUT_SPLITS_PROG_JAR_FILE)), - Collections.emptyList()); + Collections.emptyList()); inputSplitTestProg.invokeInteractiveModeForExecution(); } @Test - public void testStreamingCustomSplitJobWithCustomClassLoader() throws IOException, ProgramInvocationException { + public void testStreamingCustomSplitJobWithCustomClassLoader() throws ProgramInvocationException { PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)); TestStreamEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)), - Collections.emptyList()); + Collections.emptyList()); streamingInputSplitTestProg.invokeInteractiveModeForExecution(); } @@ -178,14 +176,14 @@ public class ClassLoaderITCase extends TestLogger { TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, - Collections.emptyList(), + Collections.emptyList(), Collections.singleton(classpath)); inputSplitTestProg2.invokeInteractiveModeForExecution(); } @Test - public void testStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException { + public void testStreamingClassloaderJobWithCustomClassLoader() throws ProgramInvocationException { // regular streaming job PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE)); @@ -193,14 +191,13 @@ public class ClassLoaderITCase extends TestLogger { miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)), - Collections.emptyList()); + Collections.emptyList()); streamingProg.invokeInteractiveModeForExecution(); } @Test - @Category(FailsOnJava11.class) - public void testCheckpointedStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException { + public void testCheckpointedStreamingClassloaderJobWithCustomClassLoader() throws ProgramInvocationException { // checkpointed streaming job with custom classes for the checkpoint (FLINK-2543) // the test also ensures that user specific exceptions are serializable between JobManager <--> JobClient. PackagedProgram streamingCheckpointedProg = new PackagedProgram(new File(STREAMING_CHECKPOINTED_PROG_JAR_FILE)); @@ -209,7 +206,7 @@ public class ClassLoaderITCase extends TestLogger { miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(STREAMING_CHECKPOINTED_PROG_JAR_FILE)), - Collections.emptyList()); + Collections.emptyList()); try { streamingCheckpointedProg.invokeInteractiveModeForExecution(); @@ -217,25 +214,26 @@ public class ClassLoaderITCase extends TestLogger { // Program should terminate with a 'SuccessException': // the exception class is contained in the user-jar, but is not present on the maven classpath // the deserialization of the exception should thus fail here + Optional exception = ExceptionUtils.findThrowable(e, + candidate -> candidate.getClass().getName().equals("org.apache.flink.test.classloading.jar.CheckpointedStreamingProgram$SuccessException")); + + if (!exception.isPresent()) { + // if this is achieved, either we failed due to another exception or the user-specific + // exception is not serialized between JobManager and JobClient. + throw e; + } + try { - Optional exception = ExceptionUtils.findThrowable(e, - candidate -> candidate.getClass().getCanonicalName().equals("org.apache.flink.test.classloading.jar.CheckpointedStreamingProgram.SuccessException")); - - // if we reach this point we either failed due to another exception, - // or the deserialization of the user-exception did not fail - if (!exception.isPresent()) { - throw e; - } else { - Assert.fail("Deserialization of user exception should have failed."); - } - } catch (NoClassDefFoundError expected) { + Class.forName(exception.get().getClass().getName()); + Assert.fail("Deserialization of user exception should have failed."); + } catch (ClassNotFoundException expected) { // expected } } } @Test - public void testKMeansJobWithCustomClassLoader() throws IOException, ProgramInvocationException { + public void testKMeansJobWithCustomClassLoader() throws ProgramInvocationException { PackagedProgram kMeansProg = new PackagedProgram( new File(KMEANS_JAR_PATH), new String[] { @@ -248,20 +246,20 @@ public class ClassLoaderITCase extends TestLogger { miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(KMEANS_JAR_PATH)), - Collections.emptyList()); + Collections.emptyList()); kMeansProg.invokeInteractiveModeForExecution(); } @Test - public void testUserCodeTypeJobWithCustomClassLoader() throws IOException, ProgramInvocationException { + public void testUserCodeTypeJobWithCustomClassLoader() throws ProgramInvocationException { PackagedProgram userCodeTypeProg = new PackagedProgram(new File(USERCODETYPE_JAR_PATH)); TestEnvironment.setAsContext( miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(USERCODETYPE_JAR_PATH)), - Collections.emptyList()); + Collections.emptyList()); userCodeTypeProg.invokeInteractiveModeForExecution(); } @@ -282,10 +280,10 @@ public class ClassLoaderITCase extends TestLogger { miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)), - Collections.emptyList()); + Collections.emptyList()); expectedException.expectCause( - Matchers.hasProperty("cause", isA(SuccessException.class))); + Matchers.hasProperty("cause", isA(SuccessException.class))); program.invokeInteractiveModeForExecution(); } @@ -315,20 +313,17 @@ public class ClassLoaderITCase extends TestLogger { miniClusterResource.getMiniCluster(), parallelism, Collections.singleton(new Path(CUSTOM_KV_STATE_JAR_PATH)), - Collections.emptyList() + Collections.emptyList() ); // Execute detached - Thread invokeThread = new Thread(new Runnable() { - @Override - public void run() { - try { - program.invokeInteractiveModeForExecution(); - } catch (ProgramInvocationException ignored) { - if (ignored.getCause() == null || - !(ignored.getCause() instanceof JobCancellationException)) { - ignored.printStackTrace(); - } + Thread invokeThread = new Thread(() -> { + try { + program.invokeInteractiveModeForExecution(); + } catch (ProgramInvocationException ex) { + if (ex.getCause() == null || + !(ex.getCause() instanceof JobCancellationException)) { + ex.printStackTrace(); } } }); diff --git a/flink-tests/src/test/java/org/apache/flink/test/example/client/LocalExecutorITCase.java b/flink-tests/src/test/java/org/apache/flink/test/example/client/LocalExecutorITCase.java index af76ec9ceca10c7d63e68790079eed33af15c372..331f6a98cb39e6e3a42ad5f847cf72f9d72f53c4 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/example/client/LocalExecutorITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/example/client/LocalExecutorITCase.java @@ -34,6 +34,7 @@ import org.junit.Test; import java.io.File; import java.io.FileWriter; +import java.util.Collections; /** * Integration tests for {@link LocalExecutor}. @@ -62,7 +63,7 @@ public class LocalExecutorITCase extends TestLogger { Plan wcPlan = getWordCountPlan(inFile, outFile, parallelism); wcPlan.setExecutionConfig(new ExecutionConfig()); - executor.executePlan(wcPlan); + executor.executePlan(wcPlan, Collections.emptyList(), Collections.emptyList()); } catch (Exception e) { e.printStackTrace(); Assert.fail(e.getMessage()); diff --git a/flink-tests/src/test/java/org/apache/flink/test/example/failing/JobSubmissionFailsITCase.java b/flink-tests/src/test/java/org/apache/flink/test/example/failing/JobSubmissionFailsITCase.java index 3cd7674037c97ecefa582e8f3f131c4ec7418254..032a3da85ceb073074fe41517e52d2dacd87b3bf 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/example/failing/JobSubmissionFailsITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/example/failing/JobSubmissionFailsITCase.java @@ -64,7 +64,7 @@ public class JobSubmissionFailsITCase extends TestLogger { private static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "4m"); return config; } diff --git a/flink-tests/src/test/java/org/apache/flink/test/manual/StreamingScalabilityAndLatency.java b/flink-tests/src/test/java/org/apache/flink/test/manual/StreamingScalabilityAndLatency.java index 38902c7619da98fd71a16594d58e037dc3c4b0f8..2398c14f8d9a98981ddfa008b2b268c641e9989a 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/manual/StreamingScalabilityAndLatency.java +++ b/flink-tests/src/test/java/org/apache/flink/test/manual/StreamingScalabilityAndLatency.java @@ -50,7 +50,7 @@ public class StreamingScalabilityAndLatency { try { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "80m"); config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 20000); config.setInteger("taskmanager.net.server.numThreads", 1); diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/CustomSerializationITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/CustomSerializationITCase.java index a1e4add6a004915ee6f074b8c11df98bd2eb662c..0177e4a9b183bb593cda2d633363d0d76eb9ae0d 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/misc/CustomSerializationITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/misc/CustomSerializationITCase.java @@ -59,7 +59,7 @@ public class CustomSerializationITCase extends TestLogger { public static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "30m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "30m"); return config; } diff --git a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java index c61657dc9bafe4f661d94f72148169d52a10c7c2..e9523f90eca00f866315f5ade98490f44a63e958 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/misc/SuccessAfterNetworkBuffersFailureITCase.java @@ -62,7 +62,7 @@ public class SuccessAfterNetworkBuffersFailureITCase extends TestLogger { private static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "80m"); config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 800); return config; } diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java index 1677a88a3ad0f429aeccfcec7cef00e803e10fb5..c8f44e8a275f6558bf24b4dcf4986948cc2174bf 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/AbstractTaskManagerProcessFailureRecoveryTest.java @@ -102,7 +102,7 @@ public abstract class AbstractTaskManagerProcessFailureRecoveryTest extends Test config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString()); config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getAbsolutePath()); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 2); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "4m"); config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 100); try (final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(config)) { diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureRecoveryITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureRecoveryITCase.java index b7c91cbb75014499662639a4b1f5e1be04a02d02..130d38ef21ccc1d6fd85226f03bef3d4598780bb 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureRecoveryITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/JobManagerHAProcessFailureRecoveryITCase.java @@ -249,7 +249,7 @@ public class JobManagerHAProcessFailureRecoveryITCase extends TestLogger { Configuration config = ZooKeeperTestUtils.createZooKeeperHAConfig( zooKeeper.getConnectString(), zookeeperStoragePath.getPath()); // Task manager configuration - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "4m"); config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 100); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 2); diff --git a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java index 0a1712ecf9f4b6e52c4e19732a08aa0cc3e3a421..db0b1be177d54c3b03ecf9c113fc2babff41a6b9 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/recovery/ProcessFailureCancelingITCase.java @@ -42,8 +42,9 @@ import org.apache.flink.runtime.dispatcher.Dispatcher; import org.apache.flink.runtime.dispatcher.DispatcherGateway; import org.apache.flink.runtime.dispatcher.DispatcherId; import org.apache.flink.runtime.dispatcher.MemoryArchivedExecutionGraphStore; +import org.apache.flink.runtime.entrypoint.component.DefaultDispatcherResourceManagerComponentFactory; import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponent; -import org.apache.flink.runtime.entrypoint.component.SessionDispatcherResourceManagerComponentFactory; +import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponentFactory; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; @@ -74,6 +75,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Collectors; import static org.apache.flink.runtime.testutils.CommonTestUtils.getJavaCommandPath; @@ -114,7 +116,7 @@ public class ProcessFailureCancelingITCase extends TestLogger { config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString()); config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getAbsolutePath()); config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 2); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "4m"); config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 100); config.setInteger(RestOptions.PORT, 0); @@ -122,13 +124,14 @@ public class ProcessFailureCancelingITCase extends TestLogger { final int jobManagerPort = rpcService.getPort(); config.setInteger(JobManagerOptions.PORT, jobManagerPort); - final SessionDispatcherResourceManagerComponentFactory resourceManagerComponentFactory = new SessionDispatcherResourceManagerComponentFactory( + final DispatcherResourceManagerComponentFactory resourceManagerComponentFactory = DefaultDispatcherResourceManagerComponentFactory.createSessionComponentFactory( StandaloneResourceManagerFactory.INSTANCE); - DispatcherResourceManagerComponent dispatcherResourceManagerComponent = null; + DispatcherResourceManagerComponent dispatcherResourceManagerComponent = null; + final ScheduledExecutorService ioExecutor = TestingUtils.defaultExecutor(); final HighAvailabilityServices haServices = HighAvailabilityServicesUtils.createHighAvailabilityServices( config, - TestingUtils.defaultExecutor(), + ioExecutor, HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION); try { @@ -142,6 +145,7 @@ public class ProcessFailureCancelingITCase extends TestLogger { dispatcherResourceManagerComponent = resourceManagerComponentFactory.create( config, + ioExecutor, rpcService, haServices, blobServerResource.getBlobServer(), @@ -265,11 +269,13 @@ public class ProcessFailureCancelingITCase extends TestLogger { * @throws Exception if something goes wrong */ static DispatcherGateway retrieveDispatcherGateway(RpcService rpcService, HighAvailabilityServices haServices) throws Exception { - final LeaderConnectionInfo leaderConnectionInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo(haServices.getDispatcherLeaderRetriever(), Time.seconds(10L)); + final LeaderConnectionInfo leaderConnectionInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo( + haServices.getDispatcherLeaderRetriever(), + Duration.ofSeconds(10L)); return rpcService.connect( leaderConnectionInfo.getAddress(), - DispatcherId.fromUuid(leaderConnectionInfo.getLeaderSessionID()), + DispatcherId.fromUuid(leaderConnectionInfo.getLeaderSessionId()), DispatcherGateway.class).get(); } diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/FileBufferReaderITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/FileBufferReaderITCase.java index 49d9ef562b786ae35f1175bb724855fa514e38f2..91d4cf43fa838268d0e6010716d5ba7e83d248fa 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/runtime/FileBufferReaderITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/FileBufferReaderITCase.java @@ -146,7 +146,7 @@ public class FileBufferReaderITCase extends TestLogger { @Override public void invoke() throws Exception { - final RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); + final RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); final ByteArrayType bytes = new ByteArrayType(dataSource); int counter = 0; diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/IPv6HostnamesITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/IPv6HostnamesITCase.java index c312b0441009953e5f3a4d7506fa3e1ec6dddb77..8fe2aa351828b9b7f1213cf3d0b67ae598ed6b6a 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/runtime/IPv6HostnamesITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/IPv6HostnamesITCase.java @@ -77,7 +77,7 @@ public class IPv6HostnamesITCase extends TestLogger { Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, addressString); config.setString(TaskManagerOptions.HOST, addressString); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "16m"); return config; } diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java index 146cad9157b11e38898953d8ae9fc5cd42239c02..3357cb69ad04ef15acbc91b1ed5a562f754e70ab 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/NetworkStackThroughputITCase.java @@ -78,7 +78,7 @@ public class NetworkStackThroughputITCase extends TestLogger { @Override public void invoke() throws Exception { - RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); + RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); try { // Determine the amount of data to send per subtask @@ -128,7 +128,7 @@ public class NetworkStackThroughputITCase extends TestLogger { SpeedTestRecord.class, getEnvironment().getTaskManagerInfo().getTmpDirectories()); - RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); + RecordWriter writer = new RecordWriterBuilder().build(getEnvironment().getWriter(0)); try { SpeedTestRecord record; diff --git a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java index a3dfb710531b5756f184f44b61b29c78e70ed8e6..168fa1f91f2e2c2bbca598e290e8635d66912360 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/runtime/leaderelection/ZooKeeperLeaderElectionITCase.java @@ -24,6 +24,7 @@ import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.ClusterOptions; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.dispatcher.Dispatcher; +import org.apache.flink.runtime.dispatcher.runner.DispatcherRunner; import org.apache.flink.runtime.entrypoint.component.DispatcherResourceManagerComponent; import org.apache.flink.runtime.execution.Environment; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; @@ -57,6 +58,7 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assert.assertThat; /** @@ -123,7 +125,7 @@ public class ZooKeeperLeaderElectionITCase extends TestLogger { miniCluster.submitJob(jobGraph).get(); - Collection> dispatcherResourceManagerComponents = miniCluster.getDispatcherResourceManagerComponents(); + Collection dispatcherResourceManagerComponents = miniCluster.getDispatcherResourceManagerComponents(); final NewLeaderRetriever newLeaderRetriever = new NewLeaderRetriever(); final HighAvailabilityServices highAvailabilityServices = miniCluster.getHighAvailabilityServices(); @@ -131,22 +133,30 @@ public class ZooKeeperLeaderElectionITCase extends TestLogger { dispatcherLeaderRetriever.start(newLeaderRetriever); for (int i = 0; i < numDispatchers - 1; i++) { - final DispatcherResourceManagerComponent leadingDispatcherResourceManagerComponent = getLeadingDispatcherResourceManagerComponent( + final DispatcherResourceManagerComponent leadingDispatcherResourceManagerComponent = getLeadingDispatcherResourceManagerComponent( dispatcherResourceManagerComponents, newLeaderRetriever); - final Dispatcher dispatcher = leadingDispatcherResourceManagerComponent.getDispatcher(); + final DispatcherRunner dispatcherRunner = leadingDispatcherResourceManagerComponent.getDispatcherRunner(); + final Dispatcher dispatcher = dispatcherRunner.getDispatcher(); + + assertThat(dispatcher, is(notNullValue())); CommonTestUtils.waitUntilCondition(() -> dispatcher.requestJobStatus(jobGraph.getJobID(), RPC_TIMEOUT).get() == JobStatus.RUNNING, timeout, 50L); leadingDispatcherResourceManagerComponent.closeAsync(); } - final DispatcherResourceManagerComponent leadingDispatcherResourceManagerComponent = getLeadingDispatcherResourceManagerComponent( + final DispatcherResourceManagerComponent leadingDispatcherResourceManagerComponent = getLeadingDispatcherResourceManagerComponent( dispatcherResourceManagerComponents, newLeaderRetriever); - CompletableFuture jobResultFuture = leadingDispatcherResourceManagerComponent.getDispatcher().requestJobResult(jobGraph.getJobID(), RPC_TIMEOUT); + final DispatcherRunner dispatcherRunner = leadingDispatcherResourceManagerComponent.getDispatcherRunner(); + final Dispatcher dispatcher = dispatcherRunner.getDispatcher(); + + assertThat(dispatcher, is(notNullValue())); + + CompletableFuture jobResultFuture = dispatcher.requestJobResult(jobGraph.getJobID(), RPC_TIMEOUT); BlockingOperator.unblock(); assertThat(jobResultFuture.get().isSuccess(), is(true)); @@ -158,8 +168,8 @@ public class ZooKeeperLeaderElectionITCase extends TestLogger { } @Nonnull - protected DispatcherResourceManagerComponent getLeadingDispatcherResourceManagerComponent( - Collection> dispatcherResourceManagerComponents, + protected DispatcherResourceManagerComponent getLeadingDispatcherResourceManagerComponent( + Collection dispatcherResourceManagerComponents, NewLeaderRetriever newLeaderRetriever) throws Exception { final Tuple2 leaderInformation = newLeaderRetriever.waitUntilNewLeader().get(); @@ -171,9 +181,12 @@ public class ZooKeeperLeaderElectionITCase extends TestLogger { } @Nonnull - private static Optional> findLeadingDispatcherResourceManagerComponent(Collection> dispatcherResourceManagerComponents, String address) { - for (DispatcherResourceManagerComponent dispatcherResourceManagerComponent : dispatcherResourceManagerComponents) { - if (dispatcherResourceManagerComponent.getDispatcher().getAddress().equals(address)) { + private static Optional findLeadingDispatcherResourceManagerComponent(Collection dispatcherResourceManagerComponents, String address) { + for (DispatcherResourceManagerComponent dispatcherResourceManagerComponent : dispatcherResourceManagerComponents) { + final DispatcherRunner dispatcherRunner = dispatcherResourceManagerComponent.getDispatcherRunner(); + final Dispatcher dispatcher = dispatcherRunner.getDispatcher(); + + if (dispatcher != null && dispatcher.getAddress().equals(address)) { return Optional.of(dispatcherResourceManagerComponent); } } diff --git a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java index 24effd746caff206ca3fec2db008e387c615e52a..f0938a245831dd780996e91af4413bd901aa58b6 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/streaming/runtime/TimestampITCase.java @@ -89,7 +89,7 @@ public class TimestampITCase extends TestLogger { private static Configuration getConfiguration() { Configuration config = new Configuration(); - config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "12m"); + config.setString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE, "12m"); return config; } diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java index 97aa2f6bc04707083d5d85193765c6801bd71969..48c415e5d903286b0fbf6d662589af3a40ba57b5 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/CliFrontendRunWithYarnTest.java @@ -20,11 +20,9 @@ package org.apache.flink.yarn; import org.apache.flink.client.cli.CliFrontendTestBase; import org.apache.flink.client.cli.CliFrontendTestUtils; -import org.apache.flink.client.deployment.ClusterSpecification; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.JobManagerOptions; -import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.util.FlinkException; import org.apache.flink.yarn.cli.FlinkYarnSessionCli; import org.apache.flink.yarn.util.FakeClusterClient; @@ -32,7 +30,6 @@ import org.apache.flink.yarn.util.NonDeployingYarnClusterDescriptor; import org.apache.commons.cli.CommandLine; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -40,7 +37,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import static junit.framework.TestCase.assertTrue; import static org.apache.flink.client.cli.CliFrontendRunTest.verifyCliFrontend; import static org.apache.flink.yarn.util.YarnTestUtils.getTestJarPath; @@ -93,7 +89,6 @@ public class CliFrontendRunWithYarnTest extends CliFrontendTestBase { } private static class TestingFlinkYarnSessionCli extends FlinkYarnSessionCli { - @SuppressWarnings("unchecked") private final ClusterClient clusterClient; private final String configurationDirectory; @@ -109,10 +104,10 @@ public class CliFrontendRunWithYarnTest extends CliFrontendTestBase { } @Override - public AbstractYarnClusterDescriptor createClusterDescriptor(CommandLine commandLine) + public YarnClusterDescriptor createClusterDescriptor(CommandLine commandLine) throws FlinkException { - AbstractYarnClusterDescriptor parent = super.createClusterDescriptor(commandLine); - return new NonDeployingDetachedYarnClusterDescriptor( + YarnClusterDescriptor parent = super.createClusterDescriptor(commandLine); + return new NonDeployingYarnClusterDescriptor( parent.getFlinkConfiguration(), (YarnConfiguration) parent.getYarnClient().getConfig(), configurationDirectory, @@ -121,23 +116,4 @@ public class CliFrontendRunWithYarnTest extends CliFrontendTestBase { } } - private static class NonDeployingDetachedYarnClusterDescriptor extends NonDeployingYarnClusterDescriptor { - - NonDeployingDetachedYarnClusterDescriptor( - Configuration flinkConfiguration, - YarnConfiguration yarnConfiguration, String configurationDirectory, - YarnClient yarnClient, - ClusterClient clusterClient) { - super(flinkConfiguration, yarnConfiguration, configurationDirectory, yarnClient, - clusterClient); - } - - @Override - public ClusterClient deployJobCluster( - ClusterSpecification clusterSpecification, JobGraph jobGraph, boolean detached) { - assertTrue(detached); - return super.deployJobCluster(clusterSpecification, jobGraph, true); - } - } - } diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java index 82d565740e6958bb9206b5e34352355819d69057..e894f654a5ba291057ee8d0318013f76ef4776f8 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java @@ -21,7 +21,6 @@ package org.apache.flink.yarn; import org.apache.flink.api.common.time.Deadline; import org.apache.flink.client.deployment.ClusterSpecification; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.rest.RestClusterClient; import org.apache.flink.configuration.AkkaOptions; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; @@ -44,7 +43,6 @@ import java.time.Duration; import java.util.Arrays; import java.util.concurrent.CompletableFuture; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assert.assertThat; @@ -102,20 +100,14 @@ public class YARNITCase extends YarnTestBase { jobGraph.addJar(new org.apache.flink.core.fs.Path(testingJar.toURI())); - ApplicationId applicationId = null; - ClusterClient clusterClient = null; + try (ClusterClient clusterClient = yarnClusterDescriptor.deployJobCluster( + clusterSpecification, + jobGraph, + false)) { - try { - clusterClient = yarnClusterDescriptor.deployJobCluster( - clusterSpecification, - jobGraph, - false); - applicationId = clusterClient.getClusterId(); + ApplicationId applicationId = clusterClient.getClusterId(); - assertThat(clusterClient, is(instanceOf(RestClusterClient.class))); - final RestClusterClient restClusterClient = (RestClusterClient) clusterClient; - - final CompletableFuture jobResultCompletableFuture = restClusterClient.requestJobResult(jobGraph.getJobID()); + final CompletableFuture jobResultCompletableFuture = clusterClient.requestJobResult(jobGraph.getJobID()); final JobResult jobResult = jobResultCompletableFuture.get(); @@ -123,10 +115,6 @@ public class YARNITCase extends YarnTestBase { assertThat(jobResult.getSerializedThrowable().isPresent(), is(false)); waitApplicationFinishedElseKillIt(applicationId, yarnAppTerminateTimeout, yarnClusterDescriptor); - } finally { - if (clusterClient != null) { - clusterClient.close(); - } } } }); diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java index 1cf46c6eb684d1bc4df81468b873d3036d62a8d6..6b809bea9ca18d46c493cc728bf5bced5d1ed880 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionCapacitySchedulerITCase.java @@ -395,7 +395,7 @@ public class YARNSessionCapacitySchedulerITCase extends YarnTestBase { public void testNonexistingQueueWARNmessage() throws Exception { runTest(() -> { LOG.info("Starting testNonexistingQueueWARNmessage()"); - addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN); + addTestAppender(YarnClusterDescriptor.class, Level.WARN); try { runWithArgs(new String[]{"-j", flinkUberjar.getAbsolutePath(), "-t", flinkLibFolder.getAbsolutePath(), diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java index 6acaf5ab660ed104ae66bbacd949b0d6077702bb..cfb41b2133ecb0d605eb65f6f01b38ed853e7a5c 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java @@ -239,7 +239,7 @@ public class YARNSessionFIFOITCase extends YarnTestBase { @Test public void testResourceComputation() throws Exception { runTest(() -> { - addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN); + addTestAppender(YarnClusterDescriptor.class, Level.WARN); LOG.info("Starting testResourceComputation()"); runWithArgs(new String[]{ "-j", flinkUberjar.getAbsolutePath(), @@ -271,7 +271,7 @@ public class YARNSessionFIFOITCase extends YarnTestBase { @Test public void testfullAlloc() throws Exception { runTest(() -> { - addTestAppender(AbstractYarnClusterDescriptor.class, Level.WARN); + addTestAppender(YarnClusterDescriptor.class, Level.WARN); LOG.info("Starting testfullAlloc()"); runWithArgs(new String[]{ "-j", flinkUberjar.getAbsolutePath(), diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java index de15e49173f0d3e2e8da2518c7850f6f1f4845db..3faf7776369a356ccf1ed2a8458291a865283d70 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNSessionFIFOSecuredITCase.java @@ -25,7 +25,6 @@ import org.apache.flink.runtime.security.SecurityUtils; import org.apache.flink.runtime.security.modules.HadoopModule; import org.apache.flink.test.util.SecureTestEnvironment; import org.apache.flink.test.util.TestingSecurityContext; -import org.apache.flink.testutils.junit.FailsOnJava11; import org.apache.flink.shaded.guava18.com.google.common.collect.Lists; @@ -38,7 +37,6 @@ import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,7 +106,6 @@ public class YARNSessionFIFOSecuredITCase extends YARNSessionFIFOITCase { @Test(timeout = 60000) // timeout after a minute. @Override - @Category(FailsOnJava11.class) public void testDetachedMode() throws Exception { runTest(() -> { runDetachedModeTest(); diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java index 563d646b7ee2d2ce90b7fa5896f2bd017ba45a9a..7459bac030474ebae7fce1e3244108dcf7d379f4 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnConfigurationITCase.java @@ -218,6 +218,6 @@ public class YarnConfigurationITCase extends YarnTestBase { private static int calculateManagedMemorySizeMB(Configuration configuration) { Configuration resourceManagerConfig = ActiveResourceManagerFactory.createActiveResourceManagerConfiguration(configuration); - return MemorySize.parse(resourceManagerConfig.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE)).getMebiBytes(); + return MemorySize.parse(resourceManagerConfig.getString(TaskManagerOptions.LEGACY_MANAGED_MEMORY_SIZE)).getMebiBytes(); } } diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java index 3381f10544c1c09808fcb60fd6e19aefaafd1808..6481420d201709f99d04f37dbf6ae26bb99fa292 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/YarnTestBase.java @@ -131,6 +131,9 @@ public abstract class YarnTestBase extends TestLogger { "Remote connection to [null] failed with java.nio.channels.NotYetConnectedException", "java.io.IOException: Connection reset by peer", + // filter out expected ResourceManagerException caused by intended shutdown request + YarnResourceManager.ERROR_MASSAGE_ON_SHUTDOWN_REQUEST, + // this can happen in Akka 2.4 on shutdown. "java.util.concurrent.RejectedExecutionException: Worker has already been shutdown", diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/FakeClusterClient.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/FakeClusterClient.java index 63219520fb192a9c3efef658d81721e1ff2644e5..82a9ea7d4a3e2e10b1408f27f349db214c552b7f 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/FakeClusterClient.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/FakeClusterClient.java @@ -25,11 +25,13 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.client.JobStatusMessage; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.jobgraph.JobStatus; +import org.apache.flink.runtime.jobmaster.JobResult; import org.apache.flink.runtime.messages.Acknowledge; import org.apache.flink.util.OptionalFailure; import org.apache.hadoop.yarn.api.records.ApplicationId; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.Collection; @@ -61,14 +63,17 @@ public class FakeClusterClient extends ClusterClient { throw new UnsupportedOperationException("Not needed in test."); } + @Override public CompletableFuture getJobStatus(JobID jobId) { throw new UnsupportedOperationException("Not needed in test."); } + @Override public void cancel(JobID jobId) { // no op } + @Override public String cancelWithSavepoint(JobID jobId, @Nullable String savepointDirectory) { throw new UnsupportedOperationException("Not needed in test."); } @@ -78,19 +83,33 @@ public class FakeClusterClient extends ClusterClient { throw new UnsupportedOperationException("Not needed in test."); } + @Override public CompletableFuture triggerSavepoint(JobID jobId, @Nullable String savepointDirectory) { throw new UnsupportedOperationException("Not needed in test."); } + @Override public CompletableFuture disposeSavepoint(String savepointPath) { return CompletableFuture.completedFuture(Acknowledge.get()); } + @Override public CompletableFuture> listJobs() { return CompletableFuture.completedFuture(Collections.emptyList()); } + @Override public Map> getAccumulators(JobID jobID, ClassLoader loader) { return Collections.emptyMap(); } + + @Override + public CompletableFuture submitJob(@Nonnull JobGraph jobGraph) { + throw new UnsupportedOperationException("Not needed in test."); + } + + @Override + public CompletableFuture requestJobResult(@Nonnull JobID jobId) { + throw new UnsupportedOperationException("Not needed in test."); + } } diff --git a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/NonDeployingYarnClusterDescriptor.java b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/NonDeployingYarnClusterDescriptor.java index de0e1d7a95f6718f1ed1c3bad2c60b8ed3b559b5..26719362c803855145ffad19ede49b8cd28af87c 100644 --- a/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/NonDeployingYarnClusterDescriptor.java +++ b/flink-yarn-tests/src/test/java/org/apache/flink/yarn/util/NonDeployingYarnClusterDescriptor.java @@ -23,7 +23,7 @@ import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.util.Preconditions; -import org.apache.flink.yarn.AbstractYarnClusterDescriptor; +import org.apache.flink.yarn.YarnClusterDescriptor; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -31,9 +31,9 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; /** - * Dummy {@link AbstractYarnClusterDescriptor} without an actual deployment for tests. + * Dummy {@link YarnClusterDescriptor} without an actual deployment for tests. */ -public class NonDeployingYarnClusterDescriptor extends AbstractYarnClusterDescriptor { +public class NonDeployingYarnClusterDescriptor extends YarnClusterDescriptor { private final ClusterClient clusterClient; @@ -45,24 +45,18 @@ public class NonDeployingYarnClusterDescriptor extends AbstractYarnClusterDescri ClusterClient clusterClient) { super(flinkConfiguration, yarnConfiguration, configurationDirectory, yarnClient, true); - //noinspection unchecked this.clusterClient = Preconditions.checkNotNull(clusterClient); } @Override public String getClusterDescription() { - // return parent.getClusterDescription(); return "NonDeployingYarnClusterDescriptor"; } @Override protected ClusterClient createYarnClusterClient( - AbstractYarnClusterDescriptor descriptor, - int numberTaskManagers, - int slotsPerTaskManager, - ApplicationReport report, - Configuration flinkConfiguration, - boolean perJobCluster) { + ApplicationReport report, + Configuration flinkConfiguration) { return clusterClient; } @@ -78,21 +72,9 @@ public class NonDeployingYarnClusterDescriptor extends AbstractYarnClusterDescri @Override public ClusterClient deployJobCluster( - ClusterSpecification clusterSpecification, JobGraph jobGraph, boolean detached) { + ClusterSpecification clusterSpecification, + JobGraph jobGraph, + boolean detached) { return clusterClient; } - - @Override - public void killCluster(ApplicationId clusterId) { - } - - @Override - protected String getYarnSessionClusterEntrypoint() { - throw new UnsupportedOperationException("Not needed in test."); - } - - @Override - protected String getYarnJobClusterEntrypoint() { - throw new UnsupportedOperationException("Not needed in test."); - } } diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java deleted file mode 100644 index a20a902d95f94e825b47336a274fc8539c1640b3..0000000000000000000000000000000000000000 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/AbstractYarnClusterDescriptor.java +++ /dev/null @@ -1,1652 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.yarn; - -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.client.deployment.ClusterDeploymentException; -import org.apache.flink.client.deployment.ClusterDescriptor; -import org.apache.flink.client.deployment.ClusterRetrieveException; -import org.apache.flink.client.deployment.ClusterSpecification; -import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.configuration.ConfigConstants; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.CoreOptions; -import org.apache.flink.configuration.HighAvailabilityOptions; -import org.apache.flink.configuration.IllegalConfigurationException; -import org.apache.flink.configuration.JobManagerOptions; -import org.apache.flink.configuration.ResourceManagerOptions; -import org.apache.flink.configuration.RestOptions; -import org.apache.flink.configuration.SecurityOptions; -import org.apache.flink.configuration.TaskManagerOptions; -import org.apache.flink.core.plugin.PluginUtils; -import org.apache.flink.runtime.clusterframework.BootstrapTools; -import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters; -import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.runtime.jobmanager.HighAvailabilityMode; -import org.apache.flink.runtime.taskexecutor.TaskManagerServices; -import org.apache.flink.util.FlinkException; -import org.apache.flink.util.Preconditions; -import org.apache.flink.util.ShutdownHookUtil; -import org.apache.flink.yarn.configuration.YarnConfigOptions; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; -import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; -import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; -import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; -import org.apache.hadoop.yarn.api.records.LocalResource; -import org.apache.hadoop.yarn.api.records.NodeReport; -import org.apache.hadoop.yarn.api.records.NodeState; -import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.QueueInfo; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.YarnApplicationState; -import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; -import org.apache.hadoop.yarn.client.api.YarnClient; -import org.apache.hadoop.yarn.client.api.YarnClientApplication; -import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; -import org.apache.hadoop.yarn.util.Records; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.io.PrintStream; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.apache.flink.configuration.ConfigConstants.ENV_FLINK_LIB_DIR; -import static org.apache.flink.configuration.ConfigConstants.ENV_FLINK_PLUGINS_DIR; -import static org.apache.flink.runtime.entrypoint.component.FileJobGraphRetriever.JOB_GRAPH_FILE_PATH; -import static org.apache.flink.yarn.cli.FlinkYarnSessionCli.CONFIG_FILE_LOG4J_NAME; -import static org.apache.flink.yarn.cli.FlinkYarnSessionCli.CONFIG_FILE_LOGBACK_NAME; -import static org.apache.flink.yarn.cli.FlinkYarnSessionCli.getDynamicProperties; - -/** - * The descriptor with deployment information for deploying a Flink cluster on Yarn. - */ -public abstract class AbstractYarnClusterDescriptor implements ClusterDescriptor { - private static final Logger LOG = LoggerFactory.getLogger(AbstractYarnClusterDescriptor.class); - - private final YarnConfiguration yarnConfiguration; - - private final YarnClient yarnClient; - - /** True if the descriptor must not shut down the YarnClient. */ - private final boolean sharedYarnClient; - - private String yarnQueue; - - private String configurationDirectory; - - private Path flinkJarPath; - - private String dynamicPropertiesEncoded; - - /** Lazily initialized list of files to ship. */ - protected List shipFiles = new LinkedList<>(); - - private final Configuration flinkConfiguration; - - private boolean detached; - - private String customName; - - private String zookeeperNamespace; - - private String nodeLabel; - - private String applicationType; - - private YarnConfigOptions.UserJarInclusion userJarInclusion; - - public AbstractYarnClusterDescriptor( - Configuration flinkConfiguration, - YarnConfiguration yarnConfiguration, - String configurationDirectory, - YarnClient yarnClient, - boolean sharedYarnClient) { - - this.yarnConfiguration = Preconditions.checkNotNull(yarnConfiguration); - this.yarnClient = Preconditions.checkNotNull(yarnClient); - this.sharedYarnClient = sharedYarnClient; - - this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration); - userJarInclusion = getUserJarInclusionMode(flinkConfiguration); - - this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory); - } - - public YarnClient getYarnClient() { - return yarnClient; - } - - /** - * The class to start the application master with. This class runs the main - * method in case of session cluster. - */ - protected abstract String getYarnSessionClusterEntrypoint(); - - /** - * The class to start the application master with. This class runs the main - * method in case of the job cluster. - */ - protected abstract String getYarnJobClusterEntrypoint(); - - public Configuration getFlinkConfiguration() { - return flinkConfiguration; - } - - public void setQueue(String queue) { - this.yarnQueue = queue; - } - - public void setLocalJarPath(Path localJarPath) { - if (!localJarPath.toString().endsWith("jar")) { - throw new IllegalArgumentException("The passed jar path ('" + localJarPath + "') does not end with the 'jar' extension"); - } - this.flinkJarPath = localJarPath; - } - - /** - * Adds the given files to the list of files to ship. - * - *

    Note that any file matching "flink-dist*.jar" will be excluded from the upload by - * {@link #uploadAndRegisterFiles(Collection, FileSystem, Path, ApplicationId, List, Map, StringBuilder)} - * since we upload the Flink uber jar ourselves and do not need to deploy it multiple times. - * - * @param shipFiles files to ship - */ - public void addShipFiles(List shipFiles) { - this.shipFiles.addAll(shipFiles); - } - - public void setDynamicPropertiesEncoded(String dynamicPropertiesEncoded) { - this.dynamicPropertiesEncoded = dynamicPropertiesEncoded; - } - - public String getDynamicPropertiesEncoded() { - return this.dynamicPropertiesEncoded; - } - - private void isReadyForDeployment(ClusterSpecification clusterSpecification) throws YarnDeploymentException { - - if (clusterSpecification.getNumberTaskManagers() <= 0) { - throw new YarnDeploymentException("Taskmanager count must be positive"); - } - if (this.flinkJarPath == null) { - throw new YarnDeploymentException("The Flink jar path is null"); - } - if (this.configurationDirectory == null) { - throw new YarnDeploymentException("Configuration directory not set"); - } - if (this.flinkConfiguration == null) { - throw new YarnDeploymentException("Flink configuration object has not been set"); - } - - // Check if we don't exceed YARN's maximum virtual cores. - // Fetch numYarnMaxVcores from all the RUNNING nodes via yarnClient - final int numYarnMaxVcores; - try { - numYarnMaxVcores = yarnClient.getNodeReports(NodeState.RUNNING) - .stream() - .mapToInt(report -> report.getCapability().getVirtualCores()) - .max() - .orElse(0); - } catch (Exception e) { - throw new YarnDeploymentException("Couldn't get cluster description, please check on the YarnConfiguration", e); - } - - int configuredAmVcores = flinkConfiguration.getInteger(YarnConfigOptions.APP_MASTER_VCORES); - if (configuredAmVcores > numYarnMaxVcores) { - throw new IllegalConfigurationException( - String.format("The number of requested virtual cores for application master %d" + - " exceeds the maximum number of virtual cores %d available in the Yarn Cluster.", - configuredAmVcores, numYarnMaxVcores)); - } - - int configuredVcores = flinkConfiguration.getInteger(YarnConfigOptions.VCORES, clusterSpecification.getSlotsPerTaskManager()); - // don't configure more than the maximum configured number of vcores - if (configuredVcores > numYarnMaxVcores) { - throw new IllegalConfigurationException( - String.format("The number of requested virtual cores per node %d" + - " exceeds the maximum number of virtual cores %d available in the Yarn Cluster." + - " Please note that the number of virtual cores is set to the number of task slots by default" + - " unless configured in the Flink config with '%s.'", - configuredVcores, numYarnMaxVcores, YarnConfigOptions.VCORES.key())); - } - - // check if required Hadoop environment variables are set. If not, warn user - if (System.getenv("HADOOP_CONF_DIR") == null && - System.getenv("YARN_CONF_DIR") == null) { - LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set. " + - "The Flink YARN Client needs one of these to be set to properly load the Hadoop " + - "configuration for accessing YARN."); - } - } - - private static boolean allocateResource(int[] nodeManagers, int toAllocate) { - for (int i = 0; i < nodeManagers.length; i++) { - if (nodeManagers[i] >= toAllocate) { - nodeManagers[i] -= toAllocate; - return true; - } - } - return false; - } - - /** - * @deprecated The cluster descriptor should not know about this option. - */ - @Deprecated - public void setDetachedMode(boolean detachedMode) { - this.detached = detachedMode; - } - - /** - * @deprecated The cluster descriptor should not know about this option. - */ - @Deprecated - public boolean isDetachedMode() { - return detached; - } - - public String getZookeeperNamespace() { - return zookeeperNamespace; - } - - public void setZookeeperNamespace(String zookeeperNamespace) { - this.zookeeperNamespace = zookeeperNamespace; - } - - public String getNodeLabel() { - return nodeLabel; - } - - public void setNodeLabel(String nodeLabel) { - this.nodeLabel = nodeLabel; - } - - // ------------------------------------------------------------- - // Lifecycle management - // ------------------------------------------------------------- - - @Override - public void close() { - if (!sharedYarnClient) { - yarnClient.stop(); - } - } - - // ------------------------------------------------------------- - // ClusterClient overrides - // ------------------------------------------------------------- - - @Override - public ClusterClient retrieve(ApplicationId applicationId) throws ClusterRetrieveException { - - try { - // check if required Hadoop environment variables are set. If not, warn user - if (System.getenv("HADOOP_CONF_DIR") == null && - System.getenv("YARN_CONF_DIR") == null) { - LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set." + - "The Flink YARN Client needs one of these to be set to properly load the Hadoop " + - "configuration for accessing YARN."); - } - - final ApplicationReport report = yarnClient.getApplicationReport(applicationId); - - if (report.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) { - // Flink cluster is not running anymore - LOG.error("The application {} doesn't run anymore. It has previously completed with final status: {}", - applicationId, report.getFinalApplicationStatus()); - throw new RuntimeException("The Yarn application " + applicationId + " doesn't run anymore."); - } - - final String host = report.getHost(); - final int port = report.getRpcPort(); - - LOG.info("Found Web Interface {}:{} of application '{}'.", host, port, applicationId); - - flinkConfiguration.setString(JobManagerOptions.ADDRESS, host); - flinkConfiguration.setInteger(JobManagerOptions.PORT, port); - - flinkConfiguration.setString(RestOptions.ADDRESS, host); - flinkConfiguration.setInteger(RestOptions.PORT, port); - - return createYarnClusterClient( - this, - -1, // we don't know the number of task managers of a started Flink cluster - -1, // we don't know how many slots each task manager has for a started Flink cluster - report, - flinkConfiguration, - false); - } catch (Exception e) { - throw new ClusterRetrieveException("Couldn't retrieve Yarn cluster", e); - } - } - - @Override - public ClusterClient deploySessionCluster(ClusterSpecification clusterSpecification) throws ClusterDeploymentException { - try { - return deployInternal( - clusterSpecification, - "Flink session cluster", - getYarnSessionClusterEntrypoint(), - null, - false); - } catch (Exception e) { - throw new ClusterDeploymentException("Couldn't deploy Yarn session cluster", e); - } - } - - @Override - public void killCluster(ApplicationId applicationId) throws FlinkException { - try { - yarnClient.killApplication(applicationId); - Utils.deleteApplicationFiles(Collections.singletonMap( - YarnConfigKeys.FLINK_YARN_FILES, - getYarnFilesDir(applicationId).toUri().toString())); - } catch (YarnException | IOException e) { - throw new FlinkException("Could not kill the Yarn Flink cluster with id " + applicationId + '.', e); - } - } - - /** - * Method to validate cluster specification before deploy it, it will throw - * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. - * - * @param clusterSpecification cluster specification to check against the configuration of the - * AbstractYarnClusterDescriptor - * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} - */ - private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { - try { - final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); - // We do the validation by calling the calculation methods here - // Internally these methods will check whether the cluster can be started with the provided - // ClusterSpecification and the configured memory requirements - final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); - TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); - } catch (IllegalArgumentException iae) { - throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + - "cluster specification. Please increase the memory of the cluster.", iae); - } - } - - /** - * This method will block until the ApplicationMaster/JobManager have been deployed on YARN. - * - * @param clusterSpecification Initial cluster specification for the Flink cluster to be deployed - * @param applicationName name of the Yarn application to start - * @param yarnClusterEntrypoint Class name of the Yarn cluster entry point. - * @param jobGraph A job graph which is deployed with the Flink cluster, {@code null} if none - * @param detached True if the cluster should be started in detached mode - */ - protected ClusterClient deployInternal( - ClusterSpecification clusterSpecification, - String applicationName, - String yarnClusterEntrypoint, - @Nullable JobGraph jobGraph, - boolean detached) throws Exception { - - // ------------------ Check if configuration is valid -------------------- - validateClusterSpecification(clusterSpecification); - - if (UserGroupInformation.isSecurityEnabled()) { - // note: UGI::hasKerberosCredentials inaccurately reports false - // for logins based on a keytab (fixed in Hadoop 2.6.1, see HADOOP-10786), - // so we check only in ticket cache scenario. - boolean useTicketCache = flinkConfiguration.getBoolean(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE); - - UserGroupInformation loginUser = UserGroupInformation.getCurrentUser(); - if (loginUser.getAuthenticationMethod() == UserGroupInformation.AuthenticationMethod.KERBEROS - && useTicketCache && !loginUser.hasKerberosCredentials()) { - LOG.error("Hadoop security with Kerberos is enabled but the login user does not have Kerberos credentials"); - throw new RuntimeException("Hadoop security with Kerberos is enabled but the login user " + - "does not have Kerberos credentials"); - } - } - - isReadyForDeployment(clusterSpecification); - - // ------------------ Check if the specified queue exists -------------------- - - checkYarnQueues(yarnClient); - - // ------------------ Add dynamic properties to local flinkConfiguraton ------ - Map dynProperties = getDynamicProperties(dynamicPropertiesEncoded); - for (Map.Entry dynProperty : dynProperties.entrySet()) { - flinkConfiguration.setString(dynProperty.getKey(), dynProperty.getValue()); - } - - // ------------------ Check if the YARN ClusterClient has the requested resources -------------- - - // Create application via yarnClient - final YarnClientApplication yarnApplication = yarnClient.createApplication(); - final GetNewApplicationResponse appResponse = yarnApplication.getNewApplicationResponse(); - - Resource maxRes = appResponse.getMaximumResourceCapability(); - - final ClusterResourceDescription freeClusterMem; - try { - freeClusterMem = getCurrentFreeClusterResources(yarnClient); - } catch (YarnException | IOException e) { - failSessionDuringDeployment(yarnClient, yarnApplication); - throw new YarnDeploymentException("Could not retrieve information about free cluster resources.", e); - } - - final int yarnMinAllocationMB = yarnConfiguration.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); - - final ClusterSpecification validClusterSpecification; - try { - validClusterSpecification = validateClusterResources( - clusterSpecification, - yarnMinAllocationMB, - maxRes, - freeClusterMem); - } catch (YarnDeploymentException yde) { - failSessionDuringDeployment(yarnClient, yarnApplication); - throw yde; - } - - LOG.info("Cluster specification: {}", validClusterSpecification); - - final ClusterEntrypoint.ExecutionMode executionMode = detached ? - ClusterEntrypoint.ExecutionMode.DETACHED - : ClusterEntrypoint.ExecutionMode.NORMAL; - - flinkConfiguration.setString(ClusterEntrypoint.EXECUTION_MODE, executionMode.toString()); - - ApplicationReport report = startAppMaster( - flinkConfiguration, - applicationName, - yarnClusterEntrypoint, - jobGraph, - yarnClient, - yarnApplication, - validClusterSpecification); - - final String host = report.getHost(); - final int port = report.getRpcPort(); - - flinkConfiguration.setString(JobManagerOptions.ADDRESS, host); - flinkConfiguration.setInteger(JobManagerOptions.PORT, port); - - flinkConfiguration.setString(RestOptions.ADDRESS, host); - flinkConfiguration.setInteger(RestOptions.PORT, port); - - // the Flink cluster is deployed in YARN. Represent cluster - return createYarnClusterClient( - this, - validClusterSpecification.getNumberTaskManagers(), - validClusterSpecification.getSlotsPerTaskManager(), - report, - flinkConfiguration, - true); - } - - protected ClusterSpecification validateClusterResources( - ClusterSpecification clusterSpecification, - int yarnMinAllocationMB, - Resource maximumResourceCapability, - ClusterResourceDescription freeClusterResources) throws YarnDeploymentException { - - int taskManagerCount = clusterSpecification.getNumberTaskManagers(); - int jobManagerMemoryMb = clusterSpecification.getMasterMemoryMB(); - int taskManagerMemoryMb = clusterSpecification.getTaskManagerMemoryMB(); - - if (jobManagerMemoryMb < yarnMinAllocationMB || taskManagerMemoryMb < yarnMinAllocationMB) { - LOG.warn("The JobManager or TaskManager memory is below the smallest possible YARN Container size. " - + "The value of 'yarn.scheduler.minimum-allocation-mb' is '" + yarnMinAllocationMB + "'. Please increase the memory size." + - "YARN will allocate the smaller containers but the scheduler will account for the minimum-allocation-mb, maybe not all instances " + - "you requested will start."); - } - - // set the memory to minAllocationMB to do the next checks correctly - if (jobManagerMemoryMb < yarnMinAllocationMB) { - jobManagerMemoryMb = yarnMinAllocationMB; - } - if (taskManagerMemoryMb < yarnMinAllocationMB) { - taskManagerMemoryMb = yarnMinAllocationMB; - } - - final String note = "Please check the 'yarn.scheduler.maximum-allocation-mb' and the 'yarn.nodemanager.resource.memory-mb' configuration values\n"; - if (jobManagerMemoryMb > maximumResourceCapability.getMemory()) { - throw new YarnDeploymentException("The cluster does not have the requested resources for the JobManager available!\n" - + "Maximum Memory: " + maximumResourceCapability.getMemory() + "MB Requested: " + jobManagerMemoryMb + "MB. " + note); - } - - if (taskManagerMemoryMb > maximumResourceCapability.getMemory()) { - throw new YarnDeploymentException("The cluster does not have the requested resources for the TaskManagers available!\n" - + "Maximum Memory: " + maximumResourceCapability.getMemory() + " Requested: " + taskManagerMemoryMb + "MB. " + note); - } - - final String noteRsc = "\nThe Flink YARN client will try to allocate the YARN session, but maybe not all TaskManagers are " + - "connecting from the beginning because the resources are currently not available in the cluster. " + - "The allocation might take more time than usual because the Flink YARN client needs to wait until " + - "the resources become available."; - int totalMemoryRequired = jobManagerMemoryMb + taskManagerMemoryMb * taskManagerCount; - - if (freeClusterResources.totalFreeMemory < totalMemoryRequired) { - LOG.warn("This YARN session requires " + totalMemoryRequired + "MB of memory in the cluster. " - + "There are currently only " + freeClusterResources.totalFreeMemory + "MB available." + noteRsc); - - } - if (taskManagerMemoryMb > freeClusterResources.containerLimit) { - LOG.warn("The requested amount of memory for the TaskManagers (" + taskManagerMemoryMb + "MB) is more than " - + "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc); - } - if (jobManagerMemoryMb > freeClusterResources.containerLimit) { - LOG.warn("The requested amount of memory for the JobManager (" + jobManagerMemoryMb + "MB) is more than " - + "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc); - } - - // ----------------- check if the requested containers fit into the cluster. - - int[] nmFree = Arrays.copyOf(freeClusterResources.nodeManagersFree, freeClusterResources.nodeManagersFree.length); - // first, allocate the jobManager somewhere. - if (!allocateResource(nmFree, jobManagerMemoryMb)) { - LOG.warn("Unable to find a NodeManager that can fit the JobManager/Application master. " + - "The JobManager requires " + jobManagerMemoryMb + "MB. NodeManagers available: " + - Arrays.toString(freeClusterResources.nodeManagersFree) + noteRsc); - } - // allocate TaskManagers - for (int i = 0; i < taskManagerCount; i++) { - if (!allocateResource(nmFree, taskManagerMemoryMb)) { - LOG.warn("There is not enough memory available in the YARN cluster. " + - "The TaskManager(s) require " + taskManagerMemoryMb + "MB each. " + - "NodeManagers available: " + Arrays.toString(freeClusterResources.nodeManagersFree) + "\n" + - "After allocating the JobManager (" + jobManagerMemoryMb + "MB) and (" + i + "/" + taskManagerCount + ") TaskManagers, " + - "the following NodeManagers are available: " + Arrays.toString(nmFree) + noteRsc); - } - } - - return new ClusterSpecification.ClusterSpecificationBuilder() - .setMasterMemoryMB(jobManagerMemoryMb) - .setTaskManagerMemoryMB(taskManagerMemoryMb) - .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) - .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) - .createClusterSpecification(); - - } - - private void checkYarnQueues(YarnClient yarnClient) { - try { - List queues = yarnClient.getAllQueues(); - if (queues.size() > 0 && this.yarnQueue != null) { // check only if there are queues configured in yarn and for this session. - boolean queueFound = false; - for (QueueInfo queue : queues) { - if (queue.getQueueName().equals(this.yarnQueue)) { - queueFound = true; - break; - } - } - if (!queueFound) { - String queueNames = ""; - for (QueueInfo queue : queues) { - queueNames += queue.getQueueName() + ", "; - } - LOG.warn("The specified queue '" + this.yarnQueue + "' does not exist. " + - "Available queues: " + queueNames); - } - } else { - LOG.debug("The YARN cluster does not have any queues configured"); - } - } catch (Throwable e) { - LOG.warn("Error while getting queue information from YARN: " + e.getMessage()); - if (LOG.isDebugEnabled()) { - LOG.debug("Error details", e); - } - } - } - - public ApplicationReport startAppMaster( - Configuration configuration, - String applicationName, - String yarnClusterEntrypoint, - JobGraph jobGraph, - YarnClient yarnClient, - YarnClientApplication yarnApplication, - ClusterSpecification clusterSpecification) throws Exception { - - // ------------------ Initialize the file systems ------------------------- - - org.apache.flink.core.fs.FileSystem.initialize( - configuration, - PluginUtils.createPluginManagerFromRootFolder(configuration)); - - // initialize file system - // Copy the application master jar to the filesystem - // Create a local resource to point to the destination jar path - final FileSystem fs = FileSystem.get(yarnConfiguration); - final Path homeDir = fs.getHomeDirectory(); - - // hard coded check for the GoogleHDFS client because its not overriding the getScheme() method. - if (!fs.getClass().getSimpleName().equals("GoogleHadoopFileSystem") && - fs.getScheme().startsWith("file")) { - LOG.warn("The file system scheme is '" + fs.getScheme() + "'. This indicates that the " - + "specified Hadoop configuration path is wrong and the system is using the default Hadoop configuration values." - + "The Flink YARN client needs to store its files in a distributed file system"); - } - - ApplicationSubmissionContext appContext = yarnApplication.getApplicationSubmissionContext(); - Set systemShipFiles = new HashSet<>(shipFiles.size()); - for (File file : shipFiles) { - systemShipFiles.add(file.getAbsoluteFile()); - } - - //check if there is a logback or log4j file - File logbackFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOGBACK_NAME); - final boolean hasLogback = logbackFile.exists(); - if (hasLogback) { - systemShipFiles.add(logbackFile); - } - - File log4jFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOG4J_NAME); - final boolean hasLog4j = log4jFile.exists(); - if (hasLog4j) { - systemShipFiles.add(log4jFile); - if (hasLogback) { - // this means there is already a logback configuration file --> fail - LOG.warn("The configuration directory ('" + configurationDirectory + "') contains both LOG4J and " + - "Logback configuration files. Please delete or rename one of them."); - } - } - - addEnvironmentFoldersToShipFiles(systemShipFiles); - - // Set-up ApplicationSubmissionContext for the application - - final ApplicationId appId = appContext.getApplicationId(); - - // ------------------ Add Zookeeper namespace to local flinkConfiguraton ------ - String zkNamespace = getZookeeperNamespace(); - // no user specified cli argument for namespace? - if (zkNamespace == null || zkNamespace.isEmpty()) { - // namespace defined in config? else use applicationId as default. - zkNamespace = configuration.getString(HighAvailabilityOptions.HA_CLUSTER_ID, String.valueOf(appId)); - setZookeeperNamespace(zkNamespace); - } - - configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace); - - if (HighAvailabilityMode.isHighAvailabilityModeActivated(configuration)) { - // activate re-execution of failed applications - appContext.setMaxAppAttempts( - configuration.getInteger( - YarnConfigOptions.APPLICATION_ATTEMPTS.key(), - YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)); - - activateHighAvailabilitySupport(appContext); - } else { - // set number of application retries to 1 in the default case - appContext.setMaxAppAttempts( - configuration.getInteger( - YarnConfigOptions.APPLICATION_ATTEMPTS.key(), - 1)); - } - - final Set userJarFiles = (jobGraph == null) - // not per-job submission - ? Collections.emptySet() - // add user code jars from the provided JobGraph - : jobGraph.getUserJars().stream().map(f -> f.toUri()).map(File::new).collect(Collectors.toSet()); - - // local resource map for Yarn - final Map localResources = new HashMap<>(2 + systemShipFiles.size() + userJarFiles.size()); - // list of remote paths (after upload) - final List paths = new ArrayList<>(2 + systemShipFiles.size() + userJarFiles.size()); - // ship list that enables reuse of resources for task manager containers - StringBuilder envShipFileList = new StringBuilder(); - - // upload and register ship files - List systemClassPaths = uploadAndRegisterFiles( - systemShipFiles, - fs, - homeDir, - appId, - paths, - localResources, - envShipFileList); - - final List userClassPaths = uploadAndRegisterFiles( - userJarFiles, - fs, - homeDir, - appId, - paths, - localResources, - envShipFileList); - - if (userJarInclusion == YarnConfigOptions.UserJarInclusion.ORDER) { - systemClassPaths.addAll(userClassPaths); - } - - // normalize classpath by sorting - Collections.sort(systemClassPaths); - Collections.sort(userClassPaths); - - // classpath assembler - StringBuilder classPathBuilder = new StringBuilder(); - if (userJarInclusion == YarnConfigOptions.UserJarInclusion.FIRST) { - for (String userClassPath : userClassPaths) { - classPathBuilder.append(userClassPath).append(File.pathSeparator); - } - } - for (String classPath : systemClassPaths) { - classPathBuilder.append(classPath).append(File.pathSeparator); - } - - // Setup jar for ApplicationMaster - Path remotePathJar = setupSingleLocalResource( - "flink.jar", - fs, - appId, - flinkJarPath, - localResources, - homeDir, - ""); - - // set the right configuration values for the TaskManager - configuration.setInteger( - TaskManagerOptions.NUM_TASK_SLOTS, - clusterSpecification.getSlotsPerTaskManager()); - - configuration.setString( - TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, - clusterSpecification.getTaskManagerMemoryMB() + "m"); - - // Upload the flink configuration - // write out configuration file - File tmpConfigurationFile = File.createTempFile(appId + "-flink-conf.yaml", null); - tmpConfigurationFile.deleteOnExit(); - BootstrapTools.writeConfiguration(configuration, tmpConfigurationFile); - - Path remotePathConf = setupSingleLocalResource( - "flink-conf.yaml", - fs, - appId, - new Path(tmpConfigurationFile.getAbsolutePath()), - localResources, - homeDir, - ""); - - paths.add(remotePathJar); - classPathBuilder.append("flink.jar").append(File.pathSeparator); - paths.add(remotePathConf); - classPathBuilder.append("flink-conf.yaml").append(File.pathSeparator); - - if (userJarInclusion == YarnConfigOptions.UserJarInclusion.LAST) { - for (String userClassPath : userClassPaths) { - classPathBuilder.append(userClassPath).append(File.pathSeparator); - } - } - - // write job graph to tmp file and add it to local resource - // TODO: server use user main method to generate job graph - if (jobGraph != null) { - try { - File fp = File.createTempFile(appId.toString(), null); - fp.deleteOnExit(); - try (FileOutputStream output = new FileOutputStream(fp); - ObjectOutputStream obOutput = new ObjectOutputStream(output);){ - obOutput.writeObject(jobGraph); - } - - final String jobGraphFilename = "job.graph"; - flinkConfiguration.setString(JOB_GRAPH_FILE_PATH, jobGraphFilename); - - Path pathFromYarnURL = setupSingleLocalResource( - jobGraphFilename, - fs, - appId, - new Path(fp.toURI()), - localResources, - homeDir, - ""); - paths.add(pathFromYarnURL); - classPathBuilder.append(jobGraphFilename).append(File.pathSeparator); - } catch (Exception e) { - LOG.warn("Add job graph to local resource fail"); - throw e; - } - } - - final Path yarnFilesDir = getYarnFilesDir(appId); - FsPermission permission = new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE); - fs.setPermission(yarnFilesDir, permission); // set permission for path. - - //To support Yarn Secure Integration Test Scenario - //In Integration test setup, the Yarn containers created by YarnMiniCluster does not have the Yarn site XML - //and KRB5 configuration files. We are adding these files as container local resources for the container - //applications (JM/TMs) to have proper secure cluster setup - Path remoteKrb5Path = null; - Path remoteYarnSiteXmlPath = null; - boolean hasKrb5 = false; - if (System.getenv("IN_TESTS") != null) { - File f = new File(System.getenv("YARN_CONF_DIR"), Utils.YARN_SITE_FILE_NAME); - LOG.info("Adding Yarn configuration {} to the AM container local resource bucket", f.getAbsolutePath()); - Path yarnSitePath = new Path(f.getAbsolutePath()); - remoteYarnSiteXmlPath = setupSingleLocalResource( - Utils.YARN_SITE_FILE_NAME, - fs, - appId, - yarnSitePath, - localResources, - homeDir, - ""); - - String krb5Config = System.getProperty("java.security.krb5.conf"); - if (krb5Config != null && krb5Config.length() != 0) { - File krb5 = new File(krb5Config); - LOG.info("Adding KRB5 configuration {} to the AM container local resource bucket", krb5.getAbsolutePath()); - Path krb5ConfPath = new Path(krb5.getAbsolutePath()); - remoteKrb5Path = setupSingleLocalResource( - Utils.KRB5_FILE_NAME, - fs, - appId, - krb5ConfPath, - localResources, - homeDir, - ""); - hasKrb5 = true; - } - } - - // setup security tokens - Path remotePathKeytab = null; - String keytab = configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB); - if (keytab != null) { - LOG.info("Adding keytab {} to the AM container local resource bucket", keytab); - remotePathKeytab = setupSingleLocalResource( - Utils.KEYTAB_FILE_NAME, - fs, - appId, - new Path(keytab), - localResources, - homeDir, - ""); - } - - final ContainerLaunchContext amContainer = setupApplicationMasterContainer( - yarnClusterEntrypoint, - hasLogback, - hasLog4j, - hasKrb5, - clusterSpecification.getMasterMemoryMB()); - - if (UserGroupInformation.isSecurityEnabled()) { - // set HDFS delegation tokens when security is enabled - LOG.info("Adding delegation token to the AM container.."); - Utils.setTokensFor(amContainer, paths, yarnConfiguration); - } - - amContainer.setLocalResources(localResources); - fs.close(); - - // Setup CLASSPATH and environment variables for ApplicationMaster - final Map appMasterEnv = new HashMap<>(); - // set user specified app master environment variables - appMasterEnv.putAll(Utils.getEnvironmentVariables(ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX, configuration)); - // set Flink app class path - appMasterEnv.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, classPathBuilder.toString()); - - // set Flink on YARN internal configuration values - appMasterEnv.put(YarnConfigKeys.ENV_TM_COUNT, String.valueOf(clusterSpecification.getNumberTaskManagers())); - appMasterEnv.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(clusterSpecification.getTaskManagerMemoryMB())); - appMasterEnv.put(YarnConfigKeys.FLINK_JAR_PATH, remotePathJar.toString()); - appMasterEnv.put(YarnConfigKeys.ENV_APP_ID, appId.toString()); - appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, homeDir.toString()); - appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, envShipFileList.toString()); - appMasterEnv.put(YarnConfigKeys.ENV_SLOTS, String.valueOf(clusterSpecification.getSlotsPerTaskManager())); - appMasterEnv.put(YarnConfigKeys.ENV_DETACHED, String.valueOf(detached)); - appMasterEnv.put(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE, getZookeeperNamespace()); - appMasterEnv.put(YarnConfigKeys.FLINK_YARN_FILES, yarnFilesDir.toUri().toString()); - - // https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md#identity-on-an-insecure-cluster-hadoop_user_name - appMasterEnv.put(YarnConfigKeys.ENV_HADOOP_USER_NAME, UserGroupInformation.getCurrentUser().getUserName()); - - if (remotePathKeytab != null) { - appMasterEnv.put(YarnConfigKeys.KEYTAB_PATH, remotePathKeytab.toString()); - String principal = configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL); - appMasterEnv.put(YarnConfigKeys.KEYTAB_PRINCIPAL, principal); - } - - //To support Yarn Secure Integration Test Scenario - if (remoteYarnSiteXmlPath != null) { - appMasterEnv.put(YarnConfigKeys.ENV_YARN_SITE_XML_PATH, remoteYarnSiteXmlPath.toString()); - } - if (remoteKrb5Path != null) { - appMasterEnv.put(YarnConfigKeys.ENV_KRB5_PATH, remoteKrb5Path.toString()); - } - - if (dynamicPropertiesEncoded != null) { - appMasterEnv.put(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES, dynamicPropertiesEncoded); - } - - // set classpath from YARN configuration - Utils.setupYarnClassPath(yarnConfiguration, appMasterEnv); - - amContainer.setEnvironment(appMasterEnv); - - // Set up resource type requirements for ApplicationMaster - Resource capability = Records.newRecord(Resource.class); - capability.setMemory(clusterSpecification.getMasterMemoryMB()); - capability.setVirtualCores(flinkConfiguration.getInteger(YarnConfigOptions.APP_MASTER_VCORES)); - - final String customApplicationName = customName != null ? customName : applicationName; - - appContext.setApplicationName(customApplicationName); - appContext.setApplicationType(applicationType != null ? applicationType : "Apache Flink"); - appContext.setAMContainerSpec(amContainer); - appContext.setResource(capability); - - // Set priority for application - int priorityNum = flinkConfiguration.getInteger(YarnConfigOptions.APPLICATION_PRIORITY); - if (priorityNum >= 0) { - Priority priority = Priority.newInstance(priorityNum); - appContext.setPriority(priority); - } - - if (yarnQueue != null) { - appContext.setQueue(yarnQueue); - } - - setApplicationNodeLabel(appContext); - - setApplicationTags(appContext); - - // add a hook to clean up in case deployment fails - Thread deploymentFailureHook = new DeploymentFailureHook(yarnClient, yarnApplication, yarnFilesDir); - Runtime.getRuntime().addShutdownHook(deploymentFailureHook); - LOG.info("Submitting application master " + appId); - yarnClient.submitApplication(appContext); - - LOG.info("Waiting for the cluster to be allocated"); - final long startTime = System.currentTimeMillis(); - ApplicationReport report; - YarnApplicationState lastAppState = YarnApplicationState.NEW; - loop: while (true) { - try { - report = yarnClient.getApplicationReport(appId); - } catch (IOException e) { - throw new YarnDeploymentException("Failed to deploy the cluster.", e); - } - YarnApplicationState appState = report.getYarnApplicationState(); - LOG.debug("Application State: {}", appState); - switch(appState) { - case FAILED: - case FINISHED: - case KILLED: - throw new YarnDeploymentException("The YARN application unexpectedly switched to state " - + appState + " during deployment. \n" + - "Diagnostics from YARN: " + report.getDiagnostics() + "\n" + - "If log aggregation is enabled on your cluster, use this command to further investigate the issue:\n" + - "yarn logs -applicationId " + appId); - //break .. - case RUNNING: - LOG.info("YARN application has been deployed successfully."); - break loop; - default: - if (appState != lastAppState) { - LOG.info("Deploying cluster, current state " + appState); - } - if (System.currentTimeMillis() - startTime > 60000) { - LOG.info("Deployment took more than 60 seconds. Please check if the requested resources are available in the YARN cluster"); - } - - } - lastAppState = appState; - Thread.sleep(250); - } - // print the application id for user to cancel themselves. - if (isDetachedMode()) { - LOG.info("The Flink YARN client has been started in detached mode. In order to stop " + - "Flink on YARN, use the following command or a YARN web interface to stop " + - "it:\nyarn application -kill " + appId + "\nPlease also note that the " + - "temporary files of the YARN session in the home directory will not be removed."); - } - // since deployment was successful, remove the hook - ShutdownHookUtil.removeShutdownHook(deploymentFailureHook, getClass().getSimpleName(), LOG); - return report; - } - - /** - * Returns the Path where the YARN application files should be uploaded to. - * - * @param appId YARN application id - */ - private Path getYarnFilesDir(final ApplicationId appId) throws IOException { - final FileSystem fileSystem = FileSystem.get(yarnConfiguration); - final Path homeDir = fileSystem.getHomeDirectory(); - return new Path(homeDir, ".flink/" + appId + '/'); - } - - /** - * Uploads and registers a single resource and adds it to localResources. - * - * @param key - * the key to add the resource under - * @param fs - * the remote file system to upload to - * @param appId - * application ID - * @param localSrcPath - * local path to the file - * @param localResources - * map of resources - * - * @return the remote path to the uploaded resource - */ - private static Path setupSingleLocalResource( - String key, - FileSystem fs, - ApplicationId appId, - Path localSrcPath, - Map localResources, - Path targetHomeDir, - String relativeTargetPath) throws IOException { - Tuple2 resource = Utils.setupLocalResource( - fs, - appId.toString(), - localSrcPath, - targetHomeDir, - relativeTargetPath); - - localResources.put(key, resource.f1); - - return resource.f0; - } - - /** - * Match file name for "flink-dist*.jar" pattern. - * - * @param fileName file name to check - * @return true if file is a dist jar - */ - private static boolean isDistJar(String fileName) { - return fileName.startsWith("flink-dist") && fileName.endsWith("jar"); - } - - /** - * Recursively uploads (and registers) any (user and system) files in shipFiles except - * for files matching "flink-dist*.jar" which should be uploaded separately. - * - * @param shipFiles - * files to upload - * @param fs - * file system to upload to - * @param targetHomeDir - * remote home directory to upload to - * @param appId - * application ID - * @param remotePaths - * paths of the remote resources (uploaded resources will be added) - * @param localResources - * map of resources (uploaded resources will be added) - * @param envShipFileList - * list of shipped files in a format understood by {@link Utils#createTaskExecutorContext} - * - * @return list of class paths with the the proper resource keys from the registration - */ - static List uploadAndRegisterFiles( - Collection shipFiles, - FileSystem fs, - Path targetHomeDir, - ApplicationId appId, - List remotePaths, - Map localResources, - StringBuilder envShipFileList) throws IOException { - final List localPaths = new ArrayList<>(); - final List relativePaths = new ArrayList<>(); - for (File shipFile : shipFiles) { - if (shipFile.isDirectory()) { - // add directories to the classpath - final java.nio.file.Path shipPath = shipFile.toPath(); - final java.nio.file.Path parentPath = shipPath.getParent(); - Files.walkFileTree(shipPath, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs) { - localPaths.add(new Path(file.toUri())); - relativePaths.add(new Path(parentPath.relativize(file).toString())); - return FileVisitResult.CONTINUE; - } - }); - } else { - localPaths.add(new Path(shipFile.toURI())); - relativePaths.add(new Path(shipFile.getName())); - } - } - - final Set archives = new HashSet<>(); - final Set resources = new HashSet<>(); - for (int i = 0; i < localPaths.size(); i++) { - final Path localPath = localPaths.get(i); - final Path relativePath = relativePaths.get(i); - if (!isDistJar(relativePath.getName())) { - final String key = relativePath.toString(); - final Path remotePath = setupSingleLocalResource( - key, - fs, - appId, - localPath, - localResources, - targetHomeDir, - relativePath.getParent().toString()); - remotePaths.add(remotePath); - envShipFileList.append(key).append("=").append(remotePath).append(","); - // add files to the classpath - if (key.endsWith("jar")) { - archives.add(relativePath.toString()); - } else { - resources.add(relativePath.getParent().toString()); - } - } - } - - // construct classpath, we always want resource directories to go first, we also sort - // both resources and archives in order to make classpath deterministic - final ArrayList classPaths = new ArrayList<>(); - resources.stream().sorted().forEach(classPaths::add); - archives.stream().sorted().forEach(classPaths::add); - return classPaths; - } - - /** - * Kills YARN application and stops YARN client. - * - *

    Use this method to kill the App before it has been properly deployed - */ - private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) { - LOG.info("Killing YARN application"); - - try { - yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId()); - } catch (Exception e) { - // we only log a debug message here because the "killApplication" call is a best-effort - // call (we don't know if the application has been deployed when the error occured). - LOG.debug("Error while killing YARN application", e); - } - yarnClient.stop(); - } - - private static class ClusterResourceDescription { - public final int totalFreeMemory; - public final int containerLimit; - public final int[] nodeManagersFree; - - public ClusterResourceDescription(int totalFreeMemory, int containerLimit, int[] nodeManagersFree) { - this.totalFreeMemory = totalFreeMemory; - this.containerLimit = containerLimit; - this.nodeManagersFree = nodeManagersFree; - } - } - - private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { - List nodes = yarnClient.getNodeReports(NodeState.RUNNING); - - int totalFreeMemory = 0; - int containerLimit = 0; - int[] nodeManagersFree = new int[nodes.size()]; - - for (int i = 0; i < nodes.size(); i++) { - NodeReport rep = nodes.get(i); - int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); - nodeManagersFree[i] = free; - totalFreeMemory += free; - if (free > containerLimit) { - containerLimit = free; - } - } - return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); - } - - @Override - public String getClusterDescription() { - - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - PrintStream ps = new PrintStream(baos); - - YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics(); - - ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers()); - List nodes = yarnClient.getNodeReports(NodeState.RUNNING); - final String format = "|%-16s |%-16s %n"; - ps.printf("|Property |Value %n"); - ps.println("+---------------------------------------+"); - int totalMemory = 0; - int totalCores = 0; - for (NodeReport rep : nodes) { - final Resource res = rep.getCapability(); - totalMemory += res.getMemory(); - totalCores += res.getVirtualCores(); - ps.format(format, "NodeID", rep.getNodeId()); - ps.format(format, "Memory", res.getMemory() + " MB"); - ps.format(format, "vCores", res.getVirtualCores()); - ps.format(format, "HealthReport", rep.getHealthReport()); - ps.format(format, "Containers", rep.getNumContainers()); - ps.println("+---------------------------------------+"); - } - ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores); - List qInfo = yarnClient.getAllQueues(); - for (QueueInfo q : qInfo) { - ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " + - q.getMaximumCapacity() + " Applications: " + q.getApplications().size()); - } - return baos.toString(); - } catch (Exception e) { - throw new RuntimeException("Couldn't get cluster description", e); - } - } - - public void setName(String name) { - this.customName = Preconditions.checkNotNull(name, "The customized name must not be null"); - } - - public void setApplicationType(String type) { - this.applicationType = Preconditions.checkNotNull(type, "The customized application type must not be null"); - } - - private void activateHighAvailabilitySupport(ApplicationSubmissionContext appContext) throws - InvocationTargetException, IllegalAccessException { - - ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance(); - - reflector.setKeepContainersAcrossApplicationAttempts(appContext, true); - - reflector.setAttemptFailuresValidityInterval( - appContext, - flinkConfiguration.getLong(YarnConfigOptions.APPLICATION_ATTEMPT_FAILURE_VALIDITY_INTERVAL)); - } - - private void setApplicationTags(final ApplicationSubmissionContext appContext) throws InvocationTargetException, - IllegalAccessException { - - final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance(); - final String tagsString = flinkConfiguration.getString(YarnConfigOptions.APPLICATION_TAGS); - - final Set applicationTags = new HashSet<>(); - - // Trim whitespace and cull empty tags - for (final String tag : tagsString.split(",")) { - final String trimmedTag = tag.trim(); - if (!trimmedTag.isEmpty()) { - applicationTags.add(trimmedTag); - } - } - - reflector.setApplicationTags(appContext, applicationTags); - } - - private void setApplicationNodeLabel(final ApplicationSubmissionContext appContext) throws InvocationTargetException, - IllegalAccessException { - - if (nodeLabel != null) { - final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance(); - reflector.setApplicationNodeLabel(appContext, nodeLabel); - } - } - - /** - * Singleton object which uses reflection to determine whether the {@link ApplicationSubmissionContext} - * supports various methods which, depending on the Hadoop version, may or may not be supported. - * - *

    If an unsupported method is invoked, nothing happens. - * - *

    Currently three methods are proxied: - * - setApplicationTags (>= 2.4.0) - * - setAttemptFailuresValidityInterval (>= 2.6.0) - * - setKeepContainersAcrossApplicationAttempts (>= 2.4.0) - * - setNodeLabelExpression (>= 2.6.0) - */ - private static class ApplicationSubmissionContextReflector { - private static final Logger LOG = LoggerFactory.getLogger(ApplicationSubmissionContextReflector.class); - - private static final ApplicationSubmissionContextReflector instance = - new ApplicationSubmissionContextReflector(ApplicationSubmissionContext.class); - - public static ApplicationSubmissionContextReflector getInstance() { - return instance; - } - - private static final String APPLICATION_TAGS_METHOD_NAME = "setApplicationTags"; - private static final String ATTEMPT_FAILURES_METHOD_NAME = "setAttemptFailuresValidityInterval"; - private static final String KEEP_CONTAINERS_METHOD_NAME = "setKeepContainersAcrossApplicationAttempts"; - private static final String NODE_LABEL_EXPRESSION_NAME = "setNodeLabelExpression"; - - private final Method applicationTagsMethod; - private final Method attemptFailuresValidityIntervalMethod; - private final Method keepContainersMethod; - @Nullable - private final Method nodeLabelExpressionMethod; - - private ApplicationSubmissionContextReflector(Class clazz) { - Method applicationTagsMethod; - Method attemptFailuresValidityIntervalMethod; - Method keepContainersMethod; - Method nodeLabelExpressionMethod; - - try { - // this method is only supported by Hadoop 2.4.0 onwards - applicationTagsMethod = clazz.getMethod(APPLICATION_TAGS_METHOD_NAME, Set.class); - LOG.debug("{} supports method {}.", clazz.getCanonicalName(), APPLICATION_TAGS_METHOD_NAME); - } catch (NoSuchMethodException e) { - LOG.debug("{} does not support method {}.", clazz.getCanonicalName(), APPLICATION_TAGS_METHOD_NAME); - // assign null because the Hadoop version apparently does not support this call. - applicationTagsMethod = null; - } - - this.applicationTagsMethod = applicationTagsMethod; - - try { - // this method is only supported by Hadoop 2.6.0 onwards - attemptFailuresValidityIntervalMethod = clazz.getMethod(ATTEMPT_FAILURES_METHOD_NAME, long.class); - LOG.debug("{} supports method {}.", clazz.getCanonicalName(), ATTEMPT_FAILURES_METHOD_NAME); - } catch (NoSuchMethodException e) { - LOG.debug("{} does not support method {}.", clazz.getCanonicalName(), ATTEMPT_FAILURES_METHOD_NAME); - // assign null because the Hadoop version apparently does not support this call. - attemptFailuresValidityIntervalMethod = null; - } - - this.attemptFailuresValidityIntervalMethod = attemptFailuresValidityIntervalMethod; - - try { - // this method is only supported by Hadoop 2.4.0 onwards - keepContainersMethod = clazz.getMethod(KEEP_CONTAINERS_METHOD_NAME, boolean.class); - LOG.debug("{} supports method {}.", clazz.getCanonicalName(), KEEP_CONTAINERS_METHOD_NAME); - } catch (NoSuchMethodException e) { - LOG.debug("{} does not support method {}.", clazz.getCanonicalName(), KEEP_CONTAINERS_METHOD_NAME); - // assign null because the Hadoop version apparently does not support this call. - keepContainersMethod = null; - } - - this.keepContainersMethod = keepContainersMethod; - - try { - nodeLabelExpressionMethod = clazz.getMethod(NODE_LABEL_EXPRESSION_NAME, String.class); - LOG.debug("{} supports method {}.", clazz.getCanonicalName(), NODE_LABEL_EXPRESSION_NAME); - } catch (NoSuchMethodException e) { - LOG.debug("{} does not support method {}.", clazz.getCanonicalName(), NODE_LABEL_EXPRESSION_NAME); - nodeLabelExpressionMethod = null; - } - - this.nodeLabelExpressionMethod = nodeLabelExpressionMethod; - } - - public void setApplicationTags( - ApplicationSubmissionContext appContext, - Set applicationTags) throws InvocationTargetException, IllegalAccessException { - if (applicationTagsMethod != null) { - LOG.debug("Calling method {} of {}.", - applicationTagsMethod.getName(), - appContext.getClass().getCanonicalName()); - applicationTagsMethod.invoke(appContext, applicationTags); - } else { - LOG.debug("{} does not support method {}. Doing nothing.", - appContext.getClass().getCanonicalName(), - APPLICATION_TAGS_METHOD_NAME); - } - } - - public void setApplicationNodeLabel( - ApplicationSubmissionContext appContext, - String nodeLabel) throws InvocationTargetException, IllegalAccessException { - if (nodeLabelExpressionMethod != null) { - LOG.debug("Calling method {} of {}.", - nodeLabelExpressionMethod.getName(), - appContext.getClass().getCanonicalName()); - nodeLabelExpressionMethod.invoke(appContext, nodeLabel); - } else { - LOG.debug("{} does not support method {}. Doing nothing.", - appContext.getClass().getCanonicalName(), - NODE_LABEL_EXPRESSION_NAME); - } - } - - public void setAttemptFailuresValidityInterval( - ApplicationSubmissionContext appContext, - long validityInterval) throws InvocationTargetException, IllegalAccessException { - if (attemptFailuresValidityIntervalMethod != null) { - LOG.debug("Calling method {} of {}.", - attemptFailuresValidityIntervalMethod.getName(), - appContext.getClass().getCanonicalName()); - attemptFailuresValidityIntervalMethod.invoke(appContext, validityInterval); - } else { - LOG.debug("{} does not support method {}. Doing nothing.", - appContext.getClass().getCanonicalName(), - ATTEMPT_FAILURES_METHOD_NAME); - } - } - - public void setKeepContainersAcrossApplicationAttempts( - ApplicationSubmissionContext appContext, - boolean keepContainers) throws InvocationTargetException, IllegalAccessException { - - if (keepContainersMethod != null) { - LOG.debug("Calling method {} of {}.", keepContainersMethod.getName(), - appContext.getClass().getCanonicalName()); - keepContainersMethod.invoke(appContext, keepContainers); - } else { - LOG.debug("{} does not support method {}. Doing nothing.", - appContext.getClass().getCanonicalName(), KEEP_CONTAINERS_METHOD_NAME); - } - } - } - - private static class YarnDeploymentException extends RuntimeException { - private static final long serialVersionUID = -812040641215388943L; - - public YarnDeploymentException(String message) { - super(message); - } - - public YarnDeploymentException(String message, Throwable cause) { - super(message, cause); - } - } - - private class DeploymentFailureHook extends Thread { - - private final YarnClient yarnClient; - private final YarnClientApplication yarnApplication; - private final Path yarnFilesDir; - - DeploymentFailureHook(YarnClient yarnClient, YarnClientApplication yarnApplication, Path yarnFilesDir) { - this.yarnClient = Preconditions.checkNotNull(yarnClient); - this.yarnApplication = Preconditions.checkNotNull(yarnApplication); - this.yarnFilesDir = Preconditions.checkNotNull(yarnFilesDir); - } - - @Override - public void run() { - LOG.info("Cancelling deployment from Deployment Failure Hook"); - failSessionDuringDeployment(yarnClient, yarnApplication); - LOG.info("Deleting files in {}.", yarnFilesDir); - try { - FileSystem fs = FileSystem.get(yarnConfiguration); - - if (!fs.delete(yarnFilesDir, true)) { - throw new IOException("Deleting files in " + yarnFilesDir + " was unsuccessful"); - } - - fs.close(); - } catch (IOException e) { - LOG.error("Failed to delete Flink Jar and configuration files in HDFS", e); - } - } - } - - protected void addEnvironmentFoldersToShipFiles(Collection effectiveShipFiles) { - addLibFoldersToShipFiles(effectiveShipFiles); - addPluginsFoldersToShipFiles(effectiveShipFiles); - } - - private void addLibFoldersToShipFiles(Collection effectiveShipFiles) { - // Add lib folder to the ship files if the environment variable is set. - // This is for convenience when running from the command-line. - // (for other files users explicitly set the ship files) - String libDir = System.getenv().get(ENV_FLINK_LIB_DIR); - if (libDir != null) { - File directoryFile = new File(libDir); - if (directoryFile.isDirectory()) { - effectiveShipFiles.add(directoryFile); - } else { - throw new YarnDeploymentException("The environment variable '" + ENV_FLINK_LIB_DIR + - "' is set to '" + libDir + "' but the directory doesn't exist."); - } - } else if (this.shipFiles.isEmpty()) { - LOG.warn("Environment variable '{}' not set and ship files have not been provided manually. " + - "Not shipping any library files.", ENV_FLINK_LIB_DIR); - } - } - - private void addPluginsFoldersToShipFiles(Collection effectiveShipFiles) { - String pluginsDir = System.getenv().get(ENV_FLINK_PLUGINS_DIR); - if (pluginsDir != null) { - File directoryFile = new File(pluginsDir); - if (directoryFile.isDirectory()) { - effectiveShipFiles.add(directoryFile); - } else { - LOG.warn("The environment variable '" + ENV_FLINK_PLUGINS_DIR + - "' is set to '" + pluginsDir + "' but the directory doesn't exist."); - } - } - } - - protected ContainerLaunchContext setupApplicationMasterContainer( - String yarnClusterEntrypoint, - boolean hasLogback, - boolean hasLog4j, - boolean hasKrb5, - int jobManagerMemoryMb) { - // ------------------ Prepare Application Master Container ------------------------------ - - // respect custom JVM options in the YAML file - String javaOpts = flinkConfiguration.getString(CoreOptions.FLINK_JVM_OPTIONS); - if (flinkConfiguration.getString(CoreOptions.FLINK_JM_JVM_OPTIONS).length() > 0) { - javaOpts += " " + flinkConfiguration.getString(CoreOptions.FLINK_JM_JVM_OPTIONS); - } - //applicable only for YarnMiniCluster secure test run - //krb5.conf file will be available as local resource in JM/TM container - if (hasKrb5) { - javaOpts += " -Djava.security.krb5.conf=krb5.conf"; - } - - // Set up the container launch context for the application master - ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); - - final Map startCommandValues = new HashMap<>(); - startCommandValues.put("java", "$JAVA_HOME/bin/java"); - - int heapSize = Utils.calculateHeapSize(jobManagerMemoryMb, flinkConfiguration); - String jvmHeapMem = String.format("-Xms%sm -Xmx%sm", heapSize, heapSize); - startCommandValues.put("jvmmem", jvmHeapMem); - - startCommandValues.put("jvmopts", javaOpts); - String logging = ""; - - if (hasLogback || hasLog4j) { - logging = "-Dlog.file=\"" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.log\""; - - if (hasLogback) { - logging += " -Dlogback.configurationFile=file:" + CONFIG_FILE_LOGBACK_NAME; - } - - if (hasLog4j) { - logging += " -Dlog4j.configuration=file:" + CONFIG_FILE_LOG4J_NAME; - } - } - - startCommandValues.put("logging", logging); - startCommandValues.put("class", yarnClusterEntrypoint); - startCommandValues.put("redirects", - "1> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.out " + - "2> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.err"); - startCommandValues.put("args", ""); - - final String commandTemplate = flinkConfiguration - .getString(ConfigConstants.YARN_CONTAINER_START_COMMAND_TEMPLATE, - ConfigConstants.DEFAULT_YARN_CONTAINER_START_COMMAND_TEMPLATE); - final String amCommand = - BootstrapTools.getStartCommand(commandTemplate, startCommandValues); - - amContainer.setCommands(Collections.singletonList(amCommand)); - - LOG.debug("Application Master start command: " + amCommand); - - return amContainer; - } - - private static YarnConfigOptions.UserJarInclusion getUserJarInclusionMode(org.apache.flink.configuration.Configuration config) { - throwIfUserTriesToDisableUserJarInclusionInSystemClassPath(config); - - return config.getEnum(YarnConfigOptions.UserJarInclusion.class, YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR); - } - - private static void throwIfUserTriesToDisableUserJarInclusionInSystemClassPath(final Configuration config) { - final String userJarInclusion = config.getString(YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR); - if ("DISABLED".equalsIgnoreCase(userJarInclusion)) { - throw new IllegalArgumentException(String.format("Config option %s cannot be set to DISABLED anymore (see FLINK-11781)", - YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR.key())); - } - } - - /** - * Creates a YarnClusterClient; may be overridden in tests. - */ - protected abstract ClusterClient createYarnClusterClient( - AbstractYarnClusterDescriptor descriptor, - int numberTaskManagers, - int slotsPerTaskManager, - ApplicationReport report, - org.apache.flink.configuration.Configuration flinkConfiguration, - boolean perJobCluster) throws Exception; -} - diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java index 3dff72fb8cff8365e52f5b2936409db20692b2fb..6b7e3d910626c5ecddb6038d7a716cefdb078351 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java @@ -18,25 +18,136 @@ package org.apache.flink.yarn; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.client.deployment.ClusterDeploymentException; +import org.apache.flink.client.deployment.ClusterDescriptor; +import org.apache.flink.client.deployment.ClusterRetrieveException; import org.apache.flink.client.deployment.ClusterSpecification; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.client.program.rest.RestClusterClient; +import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.CoreOptions; +import org.apache.flink.configuration.HighAvailabilityOptions; +import org.apache.flink.configuration.IllegalConfigurationException; +import org.apache.flink.configuration.JobManagerOptions; +import org.apache.flink.configuration.ResourceManagerOptions; +import org.apache.flink.configuration.RestOptions; +import org.apache.flink.configuration.SecurityOptions; +import org.apache.flink.configuration.TaskManagerOptions; +import org.apache.flink.core.plugin.PluginUtils; +import org.apache.flink.runtime.clusterframework.BootstrapTools; +import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters; +import org.apache.flink.runtime.entrypoint.ClusterEntrypoint; import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.runtime.jobmanager.HighAvailabilityMode; +import org.apache.flink.runtime.taskexecutor.TaskManagerServices; +import org.apache.flink.util.FlinkException; +import org.apache.flink.util.Preconditions; +import org.apache.flink.util.ShutdownHookUtil; +import org.apache.flink.yarn.configuration.YarnConfigOptions; import org.apache.flink.yarn.entrypoint.YarnJobClusterEntrypoint; import org.apache.flink.yarn.entrypoint.YarnSessionClusterEntrypoint; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.client.api.YarnClientApplication; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.Records; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.io.PrintStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.flink.configuration.ConfigConstants.ENV_FLINK_LIB_DIR; +import static org.apache.flink.configuration.ConfigConstants.ENV_FLINK_PLUGINS_DIR; +import static org.apache.flink.runtime.entrypoint.component.FileJobGraphRetriever.JOB_GRAPH_FILE_PATH; +import static org.apache.flink.yarn.cli.FlinkYarnSessionCli.CONFIG_FILE_LOG4J_NAME; +import static org.apache.flink.yarn.cli.FlinkYarnSessionCli.CONFIG_FILE_LOGBACK_NAME; +import static org.apache.flink.yarn.cli.FlinkYarnSessionCli.getDynamicProperties; /** - * Implementation of {@link AbstractYarnClusterDescriptor} which is used to start the - * application master. + * The descriptor with deployment information for deploying a Flink cluster on Yarn. */ -public class YarnClusterDescriptor extends AbstractYarnClusterDescriptor { +public class YarnClusterDescriptor implements ClusterDescriptor { + private static final Logger LOG = LoggerFactory.getLogger(YarnClusterDescriptor.class); + + private final YarnConfiguration yarnConfiguration; + + private final YarnClient yarnClient; + + /** True if the descriptor must not shut down the YarnClient. */ + private final boolean sharedYarnClient; + + /** Lazily initialized list of files to ship. */ + private final List shipFiles = new LinkedList<>(); + + private String yarnQueue; + + private String configurationDirectory; + + private Path flinkJarPath; + + private String dynamicPropertiesEncoded; + + private final Configuration flinkConfiguration; + + private boolean detached; + + private String customName; + + private String zookeeperNamespace; + + private String nodeLabel; + + private String applicationType; + + private YarnConfigOptions.UserJarInclusion userJarInclusion; public YarnClusterDescriptor( Configuration flinkConfiguration, @@ -44,24 +155,246 @@ public class YarnClusterDescriptor extends AbstractYarnClusterDescriptor { String configurationDirectory, YarnClient yarnClient, boolean sharedYarnClient) { - super( - flinkConfiguration, - yarnConfiguration, - configurationDirectory, - yarnClient, - sharedYarnClient); + + this.yarnConfiguration = Preconditions.checkNotNull(yarnConfiguration); + this.yarnClient = Preconditions.checkNotNull(yarnClient); + this.sharedYarnClient = sharedYarnClient; + + this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration); + this.userJarInclusion = getUserJarInclusionMode(flinkConfiguration); + + this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory); } - @Override + @VisibleForTesting + List getShipFiles() { + return shipFiles; + } + + public YarnClient getYarnClient() { + return yarnClient; + } + + /** + * The class to start the application master with. This class runs the main + * method in case of session cluster. + */ protected String getYarnSessionClusterEntrypoint() { return YarnSessionClusterEntrypoint.class.getName(); } - @Override + /** + * The class to start the application master with. This class runs the main + * method in case of the job cluster. + */ protected String getYarnJobClusterEntrypoint() { return YarnJobClusterEntrypoint.class.getName(); } + public Configuration getFlinkConfiguration() { + return flinkConfiguration; + } + + public void setQueue(String queue) { + this.yarnQueue = queue; + } + + public void setLocalJarPath(Path localJarPath) { + if (!localJarPath.toString().endsWith("jar")) { + throw new IllegalArgumentException("The passed jar path ('" + localJarPath + "') does not end with the 'jar' extension"); + } + this.flinkJarPath = localJarPath; + } + + /** + * Adds the given files to the list of files to ship. + * + *

    Note that any file matching "flink-dist*.jar" will be excluded from the upload by + * {@link #uploadAndRegisterFiles(Collection, FileSystem, Path, ApplicationId, List, Map, StringBuilder)} + * since we upload the Flink uber jar ourselves and do not need to deploy it multiple times. + * + * @param shipFiles files to ship + */ + public void addShipFiles(List shipFiles) { + this.shipFiles.addAll(shipFiles); + } + + public void setDynamicPropertiesEncoded(String dynamicPropertiesEncoded) { + this.dynamicPropertiesEncoded = dynamicPropertiesEncoded; + } + + public String getDynamicPropertiesEncoded() { + return this.dynamicPropertiesEncoded; + } + + private void isReadyForDeployment(ClusterSpecification clusterSpecification) throws YarnDeploymentException { + + if (clusterSpecification.getNumberTaskManagers() <= 0) { + throw new YarnDeploymentException("Taskmanager count must be positive"); + } + if (this.flinkJarPath == null) { + throw new YarnDeploymentException("The Flink jar path is null"); + } + if (this.configurationDirectory == null) { + throw new YarnDeploymentException("Configuration directory not set"); + } + if (this.flinkConfiguration == null) { + throw new YarnDeploymentException("Flink configuration object has not been set"); + } + + // Check if we don't exceed YARN's maximum virtual cores. + // Fetch numYarnMaxVcores from all the RUNNING nodes via yarnClient + final int numYarnMaxVcores; + try { + numYarnMaxVcores = yarnClient.getNodeReports(NodeState.RUNNING) + .stream() + .mapToInt(report -> report.getCapability().getVirtualCores()) + .max() + .orElse(0); + } catch (Exception e) { + throw new YarnDeploymentException("Couldn't get cluster description, please check on the YarnConfiguration", e); + } + + int configuredAmVcores = flinkConfiguration.getInteger(YarnConfigOptions.APP_MASTER_VCORES); + if (configuredAmVcores > numYarnMaxVcores) { + throw new IllegalConfigurationException( + String.format("The number of requested virtual cores for application master %d" + + " exceeds the maximum number of virtual cores %d available in the Yarn Cluster.", + configuredAmVcores, numYarnMaxVcores)); + } + + int configuredVcores = flinkConfiguration.getInteger(YarnConfigOptions.VCORES, clusterSpecification.getSlotsPerTaskManager()); + // don't configure more than the maximum configured number of vcores + if (configuredVcores > numYarnMaxVcores) { + throw new IllegalConfigurationException( + String.format("The number of requested virtual cores per node %d" + + " exceeds the maximum number of virtual cores %d available in the Yarn Cluster." + + " Please note that the number of virtual cores is set to the number of task slots by default" + + " unless configured in the Flink config with '%s.'", + configuredVcores, numYarnMaxVcores, YarnConfigOptions.VCORES.key())); + } + + // check if required Hadoop environment variables are set. If not, warn user + if (System.getenv("HADOOP_CONF_DIR") == null && + System.getenv("YARN_CONF_DIR") == null) { + LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set. " + + "The Flink YARN Client needs one of these to be set to properly load the Hadoop " + + "configuration for accessing YARN."); + } + } + + private static boolean allocateResource(int[] nodeManagers, int toAllocate) { + for (int i = 0; i < nodeManagers.length; i++) { + if (nodeManagers[i] >= toAllocate) { + nodeManagers[i] -= toAllocate; + return true; + } + } + return false; + } + + /** + * @deprecated The cluster descriptor should not know about this option. + */ + @Deprecated + public void setDetachedMode(boolean detachedMode) { + this.detached = detachedMode; + } + + /** + * @deprecated The cluster descriptor should not know about this option. + */ + @Deprecated + public boolean isDetachedMode() { + return detached; + } + + public String getZookeeperNamespace() { + return zookeeperNamespace; + } + + public void setZookeeperNamespace(String zookeeperNamespace) { + this.zookeeperNamespace = zookeeperNamespace; + } + + public String getNodeLabel() { + return nodeLabel; + } + + public void setNodeLabel(String nodeLabel) { + this.nodeLabel = nodeLabel; + } + + // ------------------------------------------------------------- + // Lifecycle management + // ------------------------------------------------------------- + + @Override + public void close() { + if (!sharedYarnClient) { + yarnClient.stop(); + } + } + + // ------------------------------------------------------------- + // ClusterClient overrides + // ------------------------------------------------------------- + + @Override + public ClusterClient retrieve(ApplicationId applicationId) throws ClusterRetrieveException { + + try { + // check if required Hadoop environment variables are set. If not, warn user + if (System.getenv("HADOOP_CONF_DIR") == null && + System.getenv("YARN_CONF_DIR") == null) { + LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set." + + "The Flink YARN Client needs one of these to be set to properly load the Hadoop " + + "configuration for accessing YARN."); + } + + final ApplicationReport report = yarnClient.getApplicationReport(applicationId); + + if (report.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) { + // Flink cluster is not running anymore + LOG.error("The application {} doesn't run anymore. It has previously completed with final status: {}", + applicationId, report.getFinalApplicationStatus()); + throw new RuntimeException("The Yarn application " + applicationId + " doesn't run anymore."); + } + + final String host = report.getHost(); + final int port = report.getRpcPort(); + + LOG.info("Found Web Interface {}:{} of application '{}'.", host, port, applicationId); + + flinkConfiguration.setString(JobManagerOptions.ADDRESS, host); + flinkConfiguration.setInteger(JobManagerOptions.PORT, port); + + flinkConfiguration.setString(RestOptions.ADDRESS, host); + flinkConfiguration.setInteger(RestOptions.PORT, port); + + return createYarnClusterClient( + report, + flinkConfiguration + ); + } catch (Exception e) { + throw new ClusterRetrieveException("Couldn't retrieve Yarn cluster", e); + } + } + + @Override + public ClusterClient deploySessionCluster(ClusterSpecification clusterSpecification) throws ClusterDeploymentException { + try { + return deployInternal( + clusterSpecification, + "Flink session cluster", + getYarnSessionClusterEntrypoint(), + null, + false); + } catch (Exception e) { + throw new ClusterDeploymentException("Couldn't deploy Yarn session cluster", e); + } + } + @Override public ClusterClient deployJobCluster( ClusterSpecification clusterSpecification, @@ -84,15 +417,1264 @@ public class YarnClusterDescriptor extends AbstractYarnClusterDescriptor { } @Override + public void killCluster(ApplicationId applicationId) throws FlinkException { + try { + yarnClient.killApplication(applicationId); + Utils.deleteApplicationFiles(Collections.singletonMap( + YarnConfigKeys.FLINK_YARN_FILES, + getYarnFilesDir(applicationId).toUri().toString())); + } catch (YarnException | IOException e) { + throw new FlinkException("Could not kill the Yarn Flink cluster with id " + applicationId + '.', e); + } + } + + /** + * Method to validate cluster specification before deploy it, it will throw + * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. + * + * @param clusterSpecification cluster specification to check against the configuration of the + * YarnClusterDescriptor + * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} + */ + private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { + try { + final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); + // We do the validation by calling the calculation methods here + // Internally these methods will check whether the cluster can be started with the provided + // ClusterSpecification and the configured memory requirements + final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); + TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); + } catch (IllegalArgumentException iae) { + throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + + "cluster specification. Please increase the memory of the cluster.", iae); + } + } + + /** + * This method will block until the ApplicationMaster/JobManager have been deployed on YARN. + * + * @param clusterSpecification Initial cluster specification for the Flink cluster to be deployed + * @param applicationName name of the Yarn application to start + * @param yarnClusterEntrypoint Class name of the Yarn cluster entry point. + * @param jobGraph A job graph which is deployed with the Flink cluster, {@code null} if none + * @param detached True if the cluster should be started in detached mode + */ + private ClusterClient deployInternal( + ClusterSpecification clusterSpecification, + String applicationName, + String yarnClusterEntrypoint, + @Nullable JobGraph jobGraph, + boolean detached) throws Exception { + + // ------------------ Check if configuration is valid -------------------- + validateClusterSpecification(clusterSpecification); + + if (UserGroupInformation.isSecurityEnabled()) { + // note: UGI::hasKerberosCredentials inaccurately reports false + // for logins based on a keytab (fixed in Hadoop 2.6.1, see HADOOP-10786), + // so we check only in ticket cache scenario. + boolean useTicketCache = flinkConfiguration.getBoolean(SecurityOptions.KERBEROS_LOGIN_USETICKETCACHE); + + UserGroupInformation loginUser = UserGroupInformation.getCurrentUser(); + if (loginUser.getAuthenticationMethod() == UserGroupInformation.AuthenticationMethod.KERBEROS + && useTicketCache && !loginUser.hasKerberosCredentials()) { + LOG.error("Hadoop security with Kerberos is enabled but the login user does not have Kerberos credentials"); + throw new RuntimeException("Hadoop security with Kerberos is enabled but the login user " + + "does not have Kerberos credentials"); + } + } + + isReadyForDeployment(clusterSpecification); + + // ------------------ Check if the specified queue exists -------------------- + + checkYarnQueues(yarnClient); + + // ------------------ Add dynamic properties to local flinkConfiguraton ------ + Map dynProperties = getDynamicProperties(dynamicPropertiesEncoded); + for (Map.Entry dynProperty : dynProperties.entrySet()) { + flinkConfiguration.setString(dynProperty.getKey(), dynProperty.getValue()); + } + + // ------------------ Check if the YARN ClusterClient has the requested resources -------------- + + // Create application via yarnClient + final YarnClientApplication yarnApplication = yarnClient.createApplication(); + final GetNewApplicationResponse appResponse = yarnApplication.getNewApplicationResponse(); + + Resource maxRes = appResponse.getMaximumResourceCapability(); + + final ClusterResourceDescription freeClusterMem; + try { + freeClusterMem = getCurrentFreeClusterResources(yarnClient); + } catch (YarnException | IOException e) { + failSessionDuringDeployment(yarnClient, yarnApplication); + throw new YarnDeploymentException("Could not retrieve information about free cluster resources.", e); + } + + final int yarnMinAllocationMB = yarnConfiguration.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0); + + final ClusterSpecification validClusterSpecification; + try { + validClusterSpecification = validateClusterResources( + clusterSpecification, + yarnMinAllocationMB, + maxRes, + freeClusterMem); + } catch (YarnDeploymentException yde) { + failSessionDuringDeployment(yarnClient, yarnApplication); + throw yde; + } + + LOG.info("Cluster specification: {}", validClusterSpecification); + + final ClusterEntrypoint.ExecutionMode executionMode = detached ? + ClusterEntrypoint.ExecutionMode.DETACHED + : ClusterEntrypoint.ExecutionMode.NORMAL; + + flinkConfiguration.setString(ClusterEntrypoint.EXECUTION_MODE, executionMode.toString()); + + ApplicationReport report = startAppMaster( + flinkConfiguration, + applicationName, + yarnClusterEntrypoint, + jobGraph, + yarnClient, + yarnApplication, + validClusterSpecification); + + final String host = report.getHost(); + final int port = report.getRpcPort(); + + flinkConfiguration.setString(JobManagerOptions.ADDRESS, host); + flinkConfiguration.setInteger(JobManagerOptions.PORT, port); + + flinkConfiguration.setString(RestOptions.ADDRESS, host); + flinkConfiguration.setInteger(RestOptions.PORT, port); + + // the Flink cluster is deployed in YARN. Represent cluster + return createYarnClusterClient( + report, + flinkConfiguration + ); + } + + private ClusterSpecification validateClusterResources( + ClusterSpecification clusterSpecification, + int yarnMinAllocationMB, + Resource maximumResourceCapability, + ClusterResourceDescription freeClusterResources) throws YarnDeploymentException { + + int taskManagerCount = clusterSpecification.getNumberTaskManagers(); + int jobManagerMemoryMb = clusterSpecification.getMasterMemoryMB(); + int taskManagerMemoryMb = clusterSpecification.getTaskManagerMemoryMB(); + + if (jobManagerMemoryMb < yarnMinAllocationMB || taskManagerMemoryMb < yarnMinAllocationMB) { + LOG.warn("The JobManager or TaskManager memory is below the smallest possible YARN Container size. " + + "The value of 'yarn.scheduler.minimum-allocation-mb' is '" + yarnMinAllocationMB + "'. Please increase the memory size." + + "YARN will allocate the smaller containers but the scheduler will account for the minimum-allocation-mb, maybe not all instances " + + "you requested will start."); + } + + // set the memory to minAllocationMB to do the next checks correctly + if (jobManagerMemoryMb < yarnMinAllocationMB) { + jobManagerMemoryMb = yarnMinAllocationMB; + } + if (taskManagerMemoryMb < yarnMinAllocationMB) { + taskManagerMemoryMb = yarnMinAllocationMB; + } + + final String note = "Please check the 'yarn.scheduler.maximum-allocation-mb' and the 'yarn.nodemanager.resource.memory-mb' configuration values\n"; + if (jobManagerMemoryMb > maximumResourceCapability.getMemory()) { + throw new YarnDeploymentException("The cluster does not have the requested resources for the JobManager available!\n" + + "Maximum Memory: " + maximumResourceCapability.getMemory() + "MB Requested: " + jobManagerMemoryMb + "MB. " + note); + } + + if (taskManagerMemoryMb > maximumResourceCapability.getMemory()) { + throw new YarnDeploymentException("The cluster does not have the requested resources for the TaskManagers available!\n" + + "Maximum Memory: " + maximumResourceCapability.getMemory() + " Requested: " + taskManagerMemoryMb + "MB. " + note); + } + + final String noteRsc = "\nThe Flink YARN client will try to allocate the YARN session, but maybe not all TaskManagers are " + + "connecting from the beginning because the resources are currently not available in the cluster. " + + "The allocation might take more time than usual because the Flink YARN client needs to wait until " + + "the resources become available."; + int totalMemoryRequired = jobManagerMemoryMb + taskManagerMemoryMb * taskManagerCount; + + if (freeClusterResources.totalFreeMemory < totalMemoryRequired) { + LOG.warn("This YARN session requires " + totalMemoryRequired + "MB of memory in the cluster. " + + "There are currently only " + freeClusterResources.totalFreeMemory + "MB available." + noteRsc); + + } + if (taskManagerMemoryMb > freeClusterResources.containerLimit) { + LOG.warn("The requested amount of memory for the TaskManagers (" + taskManagerMemoryMb + "MB) is more than " + + "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc); + } + if (jobManagerMemoryMb > freeClusterResources.containerLimit) { + LOG.warn("The requested amount of memory for the JobManager (" + jobManagerMemoryMb + "MB) is more than " + + "the largest possible YARN container: " + freeClusterResources.containerLimit + noteRsc); + } + + // ----------------- check if the requested containers fit into the cluster. + + int[] nmFree = Arrays.copyOf(freeClusterResources.nodeManagersFree, freeClusterResources.nodeManagersFree.length); + // first, allocate the jobManager somewhere. + if (!allocateResource(nmFree, jobManagerMemoryMb)) { + LOG.warn("Unable to find a NodeManager that can fit the JobManager/Application master. " + + "The JobManager requires " + jobManagerMemoryMb + "MB. NodeManagers available: " + + Arrays.toString(freeClusterResources.nodeManagersFree) + noteRsc); + } + // allocate TaskManagers + for (int i = 0; i < taskManagerCount; i++) { + if (!allocateResource(nmFree, taskManagerMemoryMb)) { + LOG.warn("There is not enough memory available in the YARN cluster. " + + "The TaskManager(s) require " + taskManagerMemoryMb + "MB each. " + + "NodeManagers available: " + Arrays.toString(freeClusterResources.nodeManagersFree) + "\n" + + "After allocating the JobManager (" + jobManagerMemoryMb + "MB) and (" + i + "/" + taskManagerCount + ") TaskManagers, " + + "the following NodeManagers are available: " + Arrays.toString(nmFree) + noteRsc); + } + } + + return new ClusterSpecification.ClusterSpecificationBuilder() + .setMasterMemoryMB(jobManagerMemoryMb) + .setTaskManagerMemoryMB(taskManagerMemoryMb) + .setNumberTaskManagers(clusterSpecification.getNumberTaskManagers()) + .setSlotsPerTaskManager(clusterSpecification.getSlotsPerTaskManager()) + .createClusterSpecification(); + + } + + private void checkYarnQueues(YarnClient yarnClient) { + try { + List queues = yarnClient.getAllQueues(); + if (queues.size() > 0 && this.yarnQueue != null) { // check only if there are queues configured in yarn and for this session. + boolean queueFound = false; + for (QueueInfo queue : queues) { + if (queue.getQueueName().equals(this.yarnQueue)) { + queueFound = true; + break; + } + } + if (!queueFound) { + String queueNames = ""; + for (QueueInfo queue : queues) { + queueNames += queue.getQueueName() + ", "; + } + LOG.warn("The specified queue '" + this.yarnQueue + "' does not exist. " + + "Available queues: " + queueNames); + } + } else { + LOG.debug("The YARN cluster does not have any queues configured"); + } + } catch (Throwable e) { + LOG.warn("Error while getting queue information from YARN: " + e.getMessage()); + if (LOG.isDebugEnabled()) { + LOG.debug("Error details", e); + } + } + } + + private ApplicationReport startAppMaster( + Configuration configuration, + String applicationName, + String yarnClusterEntrypoint, + JobGraph jobGraph, + YarnClient yarnClient, + YarnClientApplication yarnApplication, + ClusterSpecification clusterSpecification) throws Exception { + + // ------------------ Initialize the file systems ------------------------- + + org.apache.flink.core.fs.FileSystem.initialize( + configuration, + PluginUtils.createPluginManagerFromRootFolder(configuration)); + + // initialize file system + // Copy the application master jar to the filesystem + // Create a local resource to point to the destination jar path + final FileSystem fs = FileSystem.get(yarnConfiguration); + final Path homeDir = fs.getHomeDirectory(); + + // hard coded check for the GoogleHDFS client because its not overriding the getScheme() method. + if (!fs.getClass().getSimpleName().equals("GoogleHadoopFileSystem") && + fs.getScheme().startsWith("file")) { + LOG.warn("The file system scheme is '" + fs.getScheme() + "'. This indicates that the " + + "specified Hadoop configuration path is wrong and the system is using the default Hadoop configuration values." + + "The Flink YARN client needs to store its files in a distributed file system"); + } + + ApplicationSubmissionContext appContext = yarnApplication.getApplicationSubmissionContext(); + Set systemShipFiles = new HashSet<>(shipFiles.size()); + for (File file : shipFiles) { + systemShipFiles.add(file.getAbsoluteFile()); + } + + //check if there is a logback or log4j file + File logbackFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOGBACK_NAME); + final boolean hasLogback = logbackFile.exists(); + if (hasLogback) { + systemShipFiles.add(logbackFile); + } + + File log4jFile = new File(configurationDirectory + File.separator + CONFIG_FILE_LOG4J_NAME); + final boolean hasLog4j = log4jFile.exists(); + if (hasLog4j) { + systemShipFiles.add(log4jFile); + if (hasLogback) { + // this means there is already a logback configuration file --> fail + LOG.warn("The configuration directory ('" + configurationDirectory + "') contains both LOG4J and " + + "Logback configuration files. Please delete or rename one of them."); + } + } + + addEnvironmentFoldersToShipFiles(systemShipFiles); + + // Set-up ApplicationSubmissionContext for the application + + final ApplicationId appId = appContext.getApplicationId(); + + // ------------------ Add Zookeeper namespace to local flinkConfiguraton ------ + String zkNamespace = getZookeeperNamespace(); + // no user specified cli argument for namespace? + if (zkNamespace == null || zkNamespace.isEmpty()) { + // namespace defined in config? else use applicationId as default. + zkNamespace = configuration.getString(HighAvailabilityOptions.HA_CLUSTER_ID, String.valueOf(appId)); + setZookeeperNamespace(zkNamespace); + } + + configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace); + + if (HighAvailabilityMode.isHighAvailabilityModeActivated(configuration)) { + // activate re-execution of failed applications + appContext.setMaxAppAttempts( + configuration.getInteger( + YarnConfigOptions.APPLICATION_ATTEMPTS.key(), + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS)); + + activateHighAvailabilitySupport(appContext); + } else { + // set number of application retries to 1 in the default case + appContext.setMaxAppAttempts( + configuration.getInteger( + YarnConfigOptions.APPLICATION_ATTEMPTS.key(), + 1)); + } + + final Set userJarFiles = (jobGraph == null) + // not per-job submission + ? Collections.emptySet() + // add user code jars from the provided JobGraph + : jobGraph.getUserJars().stream().map(f -> f.toUri()).map(File::new).collect(Collectors.toSet()); + + // local resource map for Yarn + final Map localResources = new HashMap<>(2 + systemShipFiles.size() + userJarFiles.size()); + // list of remote paths (after upload) + final List paths = new ArrayList<>(2 + systemShipFiles.size() + userJarFiles.size()); + // ship list that enables reuse of resources for task manager containers + StringBuilder envShipFileList = new StringBuilder(); + + // upload and register ship files + List systemClassPaths = uploadAndRegisterFiles( + systemShipFiles, + fs, + homeDir, + appId, + paths, + localResources, + envShipFileList); + + final List userClassPaths = uploadAndRegisterFiles( + userJarFiles, + fs, + homeDir, + appId, + paths, + localResources, + envShipFileList); + + if (userJarInclusion == YarnConfigOptions.UserJarInclusion.ORDER) { + systemClassPaths.addAll(userClassPaths); + } + + // normalize classpath by sorting + Collections.sort(systemClassPaths); + Collections.sort(userClassPaths); + + // classpath assembler + StringBuilder classPathBuilder = new StringBuilder(); + if (userJarInclusion == YarnConfigOptions.UserJarInclusion.FIRST) { + for (String userClassPath : userClassPaths) { + classPathBuilder.append(userClassPath).append(File.pathSeparator); + } + } + for (String classPath : systemClassPaths) { + classPathBuilder.append(classPath).append(File.pathSeparator); + } + + // Setup jar for ApplicationMaster + Path remotePathJar = setupSingleLocalResource( + "flink.jar", + fs, + appId, + flinkJarPath, + localResources, + homeDir, + ""); + + // set the right configuration values for the TaskManager + configuration.setInteger( + TaskManagerOptions.NUM_TASK_SLOTS, + clusterSpecification.getSlotsPerTaskManager()); + + configuration.setString( + TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, + clusterSpecification.getTaskManagerMemoryMB() + "m"); + + // Upload the flink configuration + // write out configuration file + File tmpConfigurationFile = File.createTempFile(appId + "-flink-conf.yaml", null); + tmpConfigurationFile.deleteOnExit(); + BootstrapTools.writeConfiguration(configuration, tmpConfigurationFile); + + Path remotePathConf = setupSingleLocalResource( + "flink-conf.yaml", + fs, + appId, + new Path(tmpConfigurationFile.getAbsolutePath()), + localResources, + homeDir, + ""); + + paths.add(remotePathJar); + classPathBuilder.append("flink.jar").append(File.pathSeparator); + paths.add(remotePathConf); + classPathBuilder.append("flink-conf.yaml").append(File.pathSeparator); + + if (userJarInclusion == YarnConfigOptions.UserJarInclusion.LAST) { + for (String userClassPath : userClassPaths) { + classPathBuilder.append(userClassPath).append(File.pathSeparator); + } + } + + // write job graph to tmp file and add it to local resource + // TODO: server use user main method to generate job graph + if (jobGraph != null) { + try { + File fp = File.createTempFile(appId.toString(), null); + fp.deleteOnExit(); + try (FileOutputStream output = new FileOutputStream(fp); + ObjectOutputStream obOutput = new ObjectOutputStream(output);){ + obOutput.writeObject(jobGraph); + } + + final String jobGraphFilename = "job.graph"; + flinkConfiguration.setString(JOB_GRAPH_FILE_PATH, jobGraphFilename); + + Path pathFromYarnURL = setupSingleLocalResource( + jobGraphFilename, + fs, + appId, + new Path(fp.toURI()), + localResources, + homeDir, + ""); + paths.add(pathFromYarnURL); + classPathBuilder.append(jobGraphFilename).append(File.pathSeparator); + } catch (Exception e) { + LOG.warn("Add job graph to local resource fail"); + throw e; + } + } + + final Path yarnFilesDir = getYarnFilesDir(appId); + FsPermission permission = new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE); + fs.setPermission(yarnFilesDir, permission); // set permission for path. + + //To support Yarn Secure Integration Test Scenario + //In Integration test setup, the Yarn containers created by YarnMiniCluster does not have the Yarn site XML + //and KRB5 configuration files. We are adding these files as container local resources for the container + //applications (JM/TMs) to have proper secure cluster setup + Path remoteKrb5Path = null; + Path remoteYarnSiteXmlPath = null; + boolean hasKrb5 = false; + if (System.getenv("IN_TESTS") != null) { + File f = new File(System.getenv("YARN_CONF_DIR"), Utils.YARN_SITE_FILE_NAME); + LOG.info("Adding Yarn configuration {} to the AM container local resource bucket", f.getAbsolutePath()); + Path yarnSitePath = new Path(f.getAbsolutePath()); + remoteYarnSiteXmlPath = setupSingleLocalResource( + Utils.YARN_SITE_FILE_NAME, + fs, + appId, + yarnSitePath, + localResources, + homeDir, + ""); + + String krb5Config = System.getProperty("java.security.krb5.conf"); + if (krb5Config != null && krb5Config.length() != 0) { + File krb5 = new File(krb5Config); + LOG.info("Adding KRB5 configuration {} to the AM container local resource bucket", krb5.getAbsolutePath()); + Path krb5ConfPath = new Path(krb5.getAbsolutePath()); + remoteKrb5Path = setupSingleLocalResource( + Utils.KRB5_FILE_NAME, + fs, + appId, + krb5ConfPath, + localResources, + homeDir, + ""); + hasKrb5 = true; + } + } + + // setup security tokens + Path remotePathKeytab = null; + String keytab = configuration.getString(SecurityOptions.KERBEROS_LOGIN_KEYTAB); + if (keytab != null) { + LOG.info("Adding keytab {} to the AM container local resource bucket", keytab); + remotePathKeytab = setupSingleLocalResource( + Utils.KEYTAB_FILE_NAME, + fs, + appId, + new Path(keytab), + localResources, + homeDir, + ""); + } + + final ContainerLaunchContext amContainer = setupApplicationMasterContainer( + yarnClusterEntrypoint, + hasLogback, + hasLog4j, + hasKrb5, + clusterSpecification.getMasterMemoryMB()); + + if (UserGroupInformation.isSecurityEnabled()) { + // set HDFS delegation tokens when security is enabled + LOG.info("Adding delegation token to the AM container.."); + Utils.setTokensFor(amContainer, paths, yarnConfiguration); + } + + amContainer.setLocalResources(localResources); + fs.close(); + + // Setup CLASSPATH and environment variables for ApplicationMaster + final Map appMasterEnv = new HashMap<>(); + // set user specified app master environment variables + appMasterEnv.putAll(Utils.getEnvironmentVariables(ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX, configuration)); + // set Flink app class path + appMasterEnv.put(YarnConfigKeys.ENV_FLINK_CLASSPATH, classPathBuilder.toString()); + + // set Flink on YARN internal configuration values + appMasterEnv.put(YarnConfigKeys.ENV_TM_COUNT, String.valueOf(clusterSpecification.getNumberTaskManagers())); + appMasterEnv.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(clusterSpecification.getTaskManagerMemoryMB())); + appMasterEnv.put(YarnConfigKeys.FLINK_JAR_PATH, remotePathJar.toString()); + appMasterEnv.put(YarnConfigKeys.ENV_APP_ID, appId.toString()); + appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, homeDir.toString()); + appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, envShipFileList.toString()); + appMasterEnv.put(YarnConfigKeys.ENV_SLOTS, String.valueOf(clusterSpecification.getSlotsPerTaskManager())); + appMasterEnv.put(YarnConfigKeys.ENV_DETACHED, String.valueOf(detached)); + appMasterEnv.put(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE, getZookeeperNamespace()); + appMasterEnv.put(YarnConfigKeys.FLINK_YARN_FILES, yarnFilesDir.toUri().toString()); + + // https://github.com/apache/hadoop/blob/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnApplicationSecurity.md#identity-on-an-insecure-cluster-hadoop_user_name + appMasterEnv.put(YarnConfigKeys.ENV_HADOOP_USER_NAME, UserGroupInformation.getCurrentUser().getUserName()); + + if (remotePathKeytab != null) { + appMasterEnv.put(YarnConfigKeys.KEYTAB_PATH, remotePathKeytab.toString()); + String principal = configuration.getString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL); + appMasterEnv.put(YarnConfigKeys.KEYTAB_PRINCIPAL, principal); + } + + //To support Yarn Secure Integration Test Scenario + if (remoteYarnSiteXmlPath != null) { + appMasterEnv.put(YarnConfigKeys.ENV_YARN_SITE_XML_PATH, remoteYarnSiteXmlPath.toString()); + } + if (remoteKrb5Path != null) { + appMasterEnv.put(YarnConfigKeys.ENV_KRB5_PATH, remoteKrb5Path.toString()); + } + + if (dynamicPropertiesEncoded != null) { + appMasterEnv.put(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES, dynamicPropertiesEncoded); + } + + // set classpath from YARN configuration + Utils.setupYarnClassPath(yarnConfiguration, appMasterEnv); + + amContainer.setEnvironment(appMasterEnv); + + // Set up resource type requirements for ApplicationMaster + Resource capability = Records.newRecord(Resource.class); + capability.setMemory(clusterSpecification.getMasterMemoryMB()); + capability.setVirtualCores(flinkConfiguration.getInteger(YarnConfigOptions.APP_MASTER_VCORES)); + + final String customApplicationName = customName != null ? customName : applicationName; + + appContext.setApplicationName(customApplicationName); + appContext.setApplicationType(applicationType != null ? applicationType : "Apache Flink"); + appContext.setAMContainerSpec(amContainer); + appContext.setResource(capability); + + // Set priority for application + int priorityNum = flinkConfiguration.getInteger(YarnConfigOptions.APPLICATION_PRIORITY); + if (priorityNum >= 0) { + Priority priority = Priority.newInstance(priorityNum); + appContext.setPriority(priority); + } + + if (yarnQueue != null) { + appContext.setQueue(yarnQueue); + } + + setApplicationNodeLabel(appContext); + + setApplicationTags(appContext); + + // add a hook to clean up in case deployment fails + Thread deploymentFailureHook = new DeploymentFailureHook(yarnClient, yarnApplication, yarnFilesDir); + Runtime.getRuntime().addShutdownHook(deploymentFailureHook); + LOG.info("Submitting application master " + appId); + yarnClient.submitApplication(appContext); + + LOG.info("Waiting for the cluster to be allocated"); + final long startTime = System.currentTimeMillis(); + ApplicationReport report; + YarnApplicationState lastAppState = YarnApplicationState.NEW; + loop: while (true) { + try { + report = yarnClient.getApplicationReport(appId); + } catch (IOException e) { + throw new YarnDeploymentException("Failed to deploy the cluster.", e); + } + YarnApplicationState appState = report.getYarnApplicationState(); + LOG.debug("Application State: {}", appState); + switch(appState) { + case FAILED: + case FINISHED: + case KILLED: + throw new YarnDeploymentException("The YARN application unexpectedly switched to state " + + appState + " during deployment. \n" + + "Diagnostics from YARN: " + report.getDiagnostics() + "\n" + + "If log aggregation is enabled on your cluster, use this command to further investigate the issue:\n" + + "yarn logs -applicationId " + appId); + //break .. + case RUNNING: + LOG.info("YARN application has been deployed successfully."); + break loop; + default: + if (appState != lastAppState) { + LOG.info("Deploying cluster, current state " + appState); + } + if (System.currentTimeMillis() - startTime > 60000) { + LOG.info("Deployment took more than 60 seconds. Please check if the requested resources are available in the YARN cluster"); + } + + } + lastAppState = appState; + Thread.sleep(250); + } + // print the application id for user to cancel themselves. + if (isDetachedMode()) { + LOG.info("The Flink YARN client has been started in detached mode. In order to stop " + + "Flink on YARN, use the following command or a YARN web interface to stop " + + "it:\nyarn application -kill " + appId + "\nPlease also note that the " + + "temporary files of the YARN session in the home directory will not be removed."); + } + // since deployment was successful, remove the hook + ShutdownHookUtil.removeShutdownHook(deploymentFailureHook, getClass().getSimpleName(), LOG); + return report; + } + + /** + * Returns the Path where the YARN application files should be uploaded to. + * + * @param appId YARN application id + */ + private Path getYarnFilesDir(final ApplicationId appId) throws IOException { + final FileSystem fileSystem = FileSystem.get(yarnConfiguration); + final Path homeDir = fileSystem.getHomeDirectory(); + return new Path(homeDir, ".flink/" + appId + '/'); + } + + /** + * Uploads and registers a single resource and adds it to localResources. + * + * @param key + * the key to add the resource under + * @param fs + * the remote file system to upload to + * @param appId + * application ID + * @param localSrcPath + * local path to the file + * @param localResources + * map of resources + * + * @return the remote path to the uploaded resource + */ + private static Path setupSingleLocalResource( + String key, + FileSystem fs, + ApplicationId appId, + Path localSrcPath, + Map localResources, + Path targetHomeDir, + String relativeTargetPath) throws IOException { + Tuple2 resource = Utils.setupLocalResource( + fs, + appId.toString(), + localSrcPath, + targetHomeDir, + relativeTargetPath); + + localResources.put(key, resource.f1); + + return resource.f0; + } + + /** + * Match file name for "flink-dist*.jar" pattern. + * + * @param fileName file name to check + * @return true if file is a dist jar + */ + private static boolean isDistJar(String fileName) { + return fileName.startsWith("flink-dist") && fileName.endsWith("jar"); + } + + /** + * Recursively uploads (and registers) any (user and system) files in shipFiles except + * for files matching "flink-dist*.jar" which should be uploaded separately. + * + * @param shipFiles + * files to upload + * @param fs + * file system to upload to + * @param targetHomeDir + * remote home directory to upload to + * @param appId + * application ID + * @param remotePaths + * paths of the remote resources (uploaded resources will be added) + * @param localResources + * map of resources (uploaded resources will be added) + * @param envShipFileList + * list of shipped files in a format understood by {@link Utils#createTaskExecutorContext} + * + * @return list of class paths with the the proper resource keys from the registration + */ + static List uploadAndRegisterFiles( + Collection shipFiles, + FileSystem fs, + Path targetHomeDir, + ApplicationId appId, + List remotePaths, + Map localResources, + StringBuilder envShipFileList) throws IOException { + final List localPaths = new ArrayList<>(); + final List relativePaths = new ArrayList<>(); + for (File shipFile : shipFiles) { + if (shipFile.isDirectory()) { + // add directories to the classpath + final java.nio.file.Path shipPath = shipFile.toPath(); + final java.nio.file.Path parentPath = shipPath.getParent(); + Files.walkFileTree(shipPath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(java.nio.file.Path file, BasicFileAttributes attrs) { + localPaths.add(new Path(file.toUri())); + relativePaths.add(new Path(parentPath.relativize(file).toString())); + return FileVisitResult.CONTINUE; + } + }); + } else { + localPaths.add(new Path(shipFile.toURI())); + relativePaths.add(new Path(shipFile.getName())); + } + } + + final Set archives = new HashSet<>(); + final Set resources = new HashSet<>(); + for (int i = 0; i < localPaths.size(); i++) { + final Path localPath = localPaths.get(i); + final Path relativePath = relativePaths.get(i); + if (!isDistJar(relativePath.getName())) { + final String key = relativePath.toString(); + final Path remotePath = setupSingleLocalResource( + key, + fs, + appId, + localPath, + localResources, + targetHomeDir, + relativePath.getParent().toString()); + remotePaths.add(remotePath); + envShipFileList.append(key).append("=").append(remotePath).append(","); + // add files to the classpath + if (key.endsWith("jar")) { + archives.add(relativePath.toString()); + } else { + resources.add(relativePath.getParent().toString()); + } + } + } + + // construct classpath, we always want resource directories to go first, we also sort + // both resources and archives in order to make classpath deterministic + final ArrayList classPaths = new ArrayList<>(); + resources.stream().sorted().forEach(classPaths::add); + archives.stream().sorted().forEach(classPaths::add); + return classPaths; + } + + /** + * Kills YARN application and stops YARN client. + * + *

    Use this method to kill the App before it has been properly deployed + */ + private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) { + LOG.info("Killing YARN application"); + + try { + yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId()); + } catch (Exception e) { + // we only log a debug message here because the "killApplication" call is a best-effort + // call (we don't know if the application has been deployed when the error occured). + LOG.debug("Error while killing YARN application", e); + } + yarnClient.stop(); + } + + private static class ClusterResourceDescription { + public final int totalFreeMemory; + public final int containerLimit; + public final int[] nodeManagersFree; + + public ClusterResourceDescription(int totalFreeMemory, int containerLimit, int[] nodeManagersFree) { + this.totalFreeMemory = totalFreeMemory; + this.containerLimit = containerLimit; + this.nodeManagersFree = nodeManagersFree; + } + } + + private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { + List nodes = yarnClient.getNodeReports(NodeState.RUNNING); + + int totalFreeMemory = 0; + int containerLimit = 0; + int[] nodeManagersFree = new int[nodes.size()]; + + for (int i = 0; i < nodes.size(); i++) { + NodeReport rep = nodes.get(i); + int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); + nodeManagersFree[i] = free; + totalFreeMemory += free; + if (free > containerLimit) { + containerLimit = free; + } + } + return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); + } + + @Override + public String getClusterDescription() { + + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos); + + YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics(); + + ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers()); + List nodes = yarnClient.getNodeReports(NodeState.RUNNING); + final String format = "|%-16s |%-16s %n"; + ps.printf("|Property |Value %n"); + ps.println("+---------------------------------------+"); + int totalMemory = 0; + int totalCores = 0; + for (NodeReport rep : nodes) { + final Resource res = rep.getCapability(); + totalMemory += res.getMemory(); + totalCores += res.getVirtualCores(); + ps.format(format, "NodeID", rep.getNodeId()); + ps.format(format, "Memory", res.getMemory() + " MB"); + ps.format(format, "vCores", res.getVirtualCores()); + ps.format(format, "HealthReport", rep.getHealthReport()); + ps.format(format, "Containers", rep.getNumContainers()); + ps.println("+---------------------------------------+"); + } + ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores); + List qInfo = yarnClient.getAllQueues(); + for (QueueInfo q : qInfo) { + ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " + + q.getMaximumCapacity() + " Applications: " + q.getApplications().size()); + } + return baos.toString(); + } catch (Exception e) { + throw new RuntimeException("Couldn't get cluster description", e); + } + } + + public void setName(String name) { + this.customName = Preconditions.checkNotNull(name, "The customized name must not be null"); + } + + public void setApplicationType(String type) { + this.applicationType = Preconditions.checkNotNull(type, "The customized application type must not be null"); + } + + private void activateHighAvailabilitySupport(ApplicationSubmissionContext appContext) throws + InvocationTargetException, IllegalAccessException { + + ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance(); + + reflector.setKeepContainersAcrossApplicationAttempts(appContext, true); + + reflector.setAttemptFailuresValidityInterval( + appContext, + flinkConfiguration.getLong(YarnConfigOptions.APPLICATION_ATTEMPT_FAILURE_VALIDITY_INTERVAL)); + } + + private void setApplicationTags(final ApplicationSubmissionContext appContext) throws InvocationTargetException, + IllegalAccessException { + + final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance(); + final String tagsString = flinkConfiguration.getString(YarnConfigOptions.APPLICATION_TAGS); + + final Set applicationTags = new HashSet<>(); + + // Trim whitespace and cull empty tags + for (final String tag : tagsString.split(",")) { + final String trimmedTag = tag.trim(); + if (!trimmedTag.isEmpty()) { + applicationTags.add(trimmedTag); + } + } + + reflector.setApplicationTags(appContext, applicationTags); + } + + private void setApplicationNodeLabel(final ApplicationSubmissionContext appContext) throws InvocationTargetException, + IllegalAccessException { + + if (nodeLabel != null) { + final ApplicationSubmissionContextReflector reflector = ApplicationSubmissionContextReflector.getInstance(); + reflector.setApplicationNodeLabel(appContext, nodeLabel); + } + } + + /** + * Singleton object which uses reflection to determine whether the {@link ApplicationSubmissionContext} + * supports various methods which, depending on the Hadoop version, may or may not be supported. + * + *

    If an unsupported method is invoked, nothing happens. + * + *

    Currently three methods are proxied: + * - setApplicationTags (>= 2.4.0) + * - setAttemptFailuresValidityInterval (>= 2.6.0) + * - setKeepContainersAcrossApplicationAttempts (>= 2.4.0) + * - setNodeLabelExpression (>= 2.6.0) + */ + private static class ApplicationSubmissionContextReflector { + private static final Logger LOG = LoggerFactory.getLogger(ApplicationSubmissionContextReflector.class); + + private static final ApplicationSubmissionContextReflector instance = + new ApplicationSubmissionContextReflector(ApplicationSubmissionContext.class); + + public static ApplicationSubmissionContextReflector getInstance() { + return instance; + } + + private static final String APPLICATION_TAGS_METHOD_NAME = "setApplicationTags"; + private static final String ATTEMPT_FAILURES_METHOD_NAME = "setAttemptFailuresValidityInterval"; + private static final String KEEP_CONTAINERS_METHOD_NAME = "setKeepContainersAcrossApplicationAttempts"; + private static final String NODE_LABEL_EXPRESSION_NAME = "setNodeLabelExpression"; + + private final Method applicationTagsMethod; + private final Method attemptFailuresValidityIntervalMethod; + private final Method keepContainersMethod; + @Nullable + private final Method nodeLabelExpressionMethod; + + private ApplicationSubmissionContextReflector(Class clazz) { + Method applicationTagsMethod; + Method attemptFailuresValidityIntervalMethod; + Method keepContainersMethod; + Method nodeLabelExpressionMethod; + + try { + // this method is only supported by Hadoop 2.4.0 onwards + applicationTagsMethod = clazz.getMethod(APPLICATION_TAGS_METHOD_NAME, Set.class); + LOG.debug("{} supports method {}.", clazz.getCanonicalName(), APPLICATION_TAGS_METHOD_NAME); + } catch (NoSuchMethodException e) { + LOG.debug("{} does not support method {}.", clazz.getCanonicalName(), APPLICATION_TAGS_METHOD_NAME); + // assign null because the Hadoop version apparently does not support this call. + applicationTagsMethod = null; + } + + this.applicationTagsMethod = applicationTagsMethod; + + try { + // this method is only supported by Hadoop 2.6.0 onwards + attemptFailuresValidityIntervalMethod = clazz.getMethod(ATTEMPT_FAILURES_METHOD_NAME, long.class); + LOG.debug("{} supports method {}.", clazz.getCanonicalName(), ATTEMPT_FAILURES_METHOD_NAME); + } catch (NoSuchMethodException e) { + LOG.debug("{} does not support method {}.", clazz.getCanonicalName(), ATTEMPT_FAILURES_METHOD_NAME); + // assign null because the Hadoop version apparently does not support this call. + attemptFailuresValidityIntervalMethod = null; + } + + this.attemptFailuresValidityIntervalMethod = attemptFailuresValidityIntervalMethod; + + try { + // this method is only supported by Hadoop 2.4.0 onwards + keepContainersMethod = clazz.getMethod(KEEP_CONTAINERS_METHOD_NAME, boolean.class); + LOG.debug("{} supports method {}.", clazz.getCanonicalName(), KEEP_CONTAINERS_METHOD_NAME); + } catch (NoSuchMethodException e) { + LOG.debug("{} does not support method {}.", clazz.getCanonicalName(), KEEP_CONTAINERS_METHOD_NAME); + // assign null because the Hadoop version apparently does not support this call. + keepContainersMethod = null; + } + + this.keepContainersMethod = keepContainersMethod; + + try { + nodeLabelExpressionMethod = clazz.getMethod(NODE_LABEL_EXPRESSION_NAME, String.class); + LOG.debug("{} supports method {}.", clazz.getCanonicalName(), NODE_LABEL_EXPRESSION_NAME); + } catch (NoSuchMethodException e) { + LOG.debug("{} does not support method {}.", clazz.getCanonicalName(), NODE_LABEL_EXPRESSION_NAME); + nodeLabelExpressionMethod = null; + } + + this.nodeLabelExpressionMethod = nodeLabelExpressionMethod; + } + + public void setApplicationTags( + ApplicationSubmissionContext appContext, + Set applicationTags) throws InvocationTargetException, IllegalAccessException { + if (applicationTagsMethod != null) { + LOG.debug("Calling method {} of {}.", + applicationTagsMethod.getName(), + appContext.getClass().getCanonicalName()); + applicationTagsMethod.invoke(appContext, applicationTags); + } else { + LOG.debug("{} does not support method {}. Doing nothing.", + appContext.getClass().getCanonicalName(), + APPLICATION_TAGS_METHOD_NAME); + } + } + + public void setApplicationNodeLabel( + ApplicationSubmissionContext appContext, + String nodeLabel) throws InvocationTargetException, IllegalAccessException { + if (nodeLabelExpressionMethod != null) { + LOG.debug("Calling method {} of {}.", + nodeLabelExpressionMethod.getName(), + appContext.getClass().getCanonicalName()); + nodeLabelExpressionMethod.invoke(appContext, nodeLabel); + } else { + LOG.debug("{} does not support method {}. Doing nothing.", + appContext.getClass().getCanonicalName(), + NODE_LABEL_EXPRESSION_NAME); + } + } + + public void setAttemptFailuresValidityInterval( + ApplicationSubmissionContext appContext, + long validityInterval) throws InvocationTargetException, IllegalAccessException { + if (attemptFailuresValidityIntervalMethod != null) { + LOG.debug("Calling method {} of {}.", + attemptFailuresValidityIntervalMethod.getName(), + appContext.getClass().getCanonicalName()); + attemptFailuresValidityIntervalMethod.invoke(appContext, validityInterval); + } else { + LOG.debug("{} does not support method {}. Doing nothing.", + appContext.getClass().getCanonicalName(), + ATTEMPT_FAILURES_METHOD_NAME); + } + } + + public void setKeepContainersAcrossApplicationAttempts( + ApplicationSubmissionContext appContext, + boolean keepContainers) throws InvocationTargetException, IllegalAccessException { + + if (keepContainersMethod != null) { + LOG.debug("Calling method {} of {}.", keepContainersMethod.getName(), + appContext.getClass().getCanonicalName()); + keepContainersMethod.invoke(appContext, keepContainers); + } else { + LOG.debug("{} does not support method {}. Doing nothing.", + appContext.getClass().getCanonicalName(), KEEP_CONTAINERS_METHOD_NAME); + } + } + } + + private static class YarnDeploymentException extends RuntimeException { + private static final long serialVersionUID = -812040641215388943L; + + public YarnDeploymentException(String message) { + super(message); + } + + public YarnDeploymentException(String message, Throwable cause) { + super(message, cause); + } + } + + private class DeploymentFailureHook extends Thread { + + private final YarnClient yarnClient; + private final YarnClientApplication yarnApplication; + private final Path yarnFilesDir; + + DeploymentFailureHook(YarnClient yarnClient, YarnClientApplication yarnApplication, Path yarnFilesDir) { + this.yarnClient = Preconditions.checkNotNull(yarnClient); + this.yarnApplication = Preconditions.checkNotNull(yarnApplication); + this.yarnFilesDir = Preconditions.checkNotNull(yarnFilesDir); + } + + @Override + public void run() { + LOG.info("Cancelling deployment from Deployment Failure Hook"); + failSessionDuringDeployment(yarnClient, yarnApplication); + LOG.info("Deleting files in {}.", yarnFilesDir); + try { + FileSystem fs = FileSystem.get(yarnConfiguration); + + if (!fs.delete(yarnFilesDir, true)) { + throw new IOException("Deleting files in " + yarnFilesDir + " was unsuccessful"); + } + + fs.close(); + } catch (IOException e) { + LOG.error("Failed to delete Flink Jar and configuration files in HDFS", e); + } + } + } + + void addEnvironmentFoldersToShipFiles(Collection effectiveShipFiles) { + addLibFoldersToShipFiles(effectiveShipFiles); + addPluginsFoldersToShipFiles(effectiveShipFiles); + } + + private void addLibFoldersToShipFiles(Collection effectiveShipFiles) { + // Add lib folder to the ship files if the environment variable is set. + // This is for convenience when running from the command-line. + // (for other files users explicitly set the ship files) + String libDir = System.getenv().get(ENV_FLINK_LIB_DIR); + if (libDir != null) { + File directoryFile = new File(libDir); + if (directoryFile.isDirectory()) { + effectiveShipFiles.add(directoryFile); + } else { + throw new YarnDeploymentException("The environment variable '" + ENV_FLINK_LIB_DIR + + "' is set to '" + libDir + "' but the directory doesn't exist."); + } + } else if (shipFiles.isEmpty()) { + LOG.warn("Environment variable '{}' not set and ship files have not been provided manually. " + + "Not shipping any library files.", ENV_FLINK_LIB_DIR); + } + } + + private void addPluginsFoldersToShipFiles(Collection effectiveShipFiles) { + String pluginsDir = System.getenv().get(ENV_FLINK_PLUGINS_DIR); + if (pluginsDir != null) { + File directoryFile = new File(pluginsDir); + if (directoryFile.isDirectory()) { + effectiveShipFiles.add(directoryFile); + } else { + LOG.warn("The environment variable '" + ENV_FLINK_PLUGINS_DIR + + "' is set to '" + pluginsDir + "' but the directory doesn't exist."); + } + } + } + + ContainerLaunchContext setupApplicationMasterContainer( + String yarnClusterEntrypoint, + boolean hasLogback, + boolean hasLog4j, + boolean hasKrb5, + int jobManagerMemoryMb) { + // ------------------ Prepare Application Master Container ------------------------------ + + // respect custom JVM options in the YAML file + String javaOpts = flinkConfiguration.getString(CoreOptions.FLINK_JVM_OPTIONS); + if (flinkConfiguration.getString(CoreOptions.FLINK_JM_JVM_OPTIONS).length() > 0) { + javaOpts += " " + flinkConfiguration.getString(CoreOptions.FLINK_JM_JVM_OPTIONS); + } + //applicable only for YarnMiniCluster secure test run + //krb5.conf file will be available as local resource in JM/TM container + if (hasKrb5) { + javaOpts += " -Djava.security.krb5.conf=krb5.conf"; + } + + // Set up the container launch context for the application master + ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); + + final Map startCommandValues = new HashMap<>(); + startCommandValues.put("java", "$JAVA_HOME/bin/java"); + + int heapSize = Utils.calculateHeapSize(jobManagerMemoryMb, flinkConfiguration); + String jvmHeapMem = String.format("-Xms%sm -Xmx%sm", heapSize, heapSize); + startCommandValues.put("jvmmem", jvmHeapMem); + + startCommandValues.put("jvmopts", javaOpts); + String logging = ""; + + if (hasLogback || hasLog4j) { + logging = "-Dlog.file=\"" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.log\""; + + if (hasLogback) { + logging += " -Dlogback.configurationFile=file:" + CONFIG_FILE_LOGBACK_NAME; + } + + if (hasLog4j) { + logging += " -Dlog4j.configuration=file:" + CONFIG_FILE_LOG4J_NAME; + } + } + + startCommandValues.put("logging", logging); + startCommandValues.put("class", yarnClusterEntrypoint); + startCommandValues.put("redirects", + "1> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.out " + + "2> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.err"); + startCommandValues.put("args", ""); + + final String commandTemplate = flinkConfiguration + .getString(ConfigConstants.YARN_CONTAINER_START_COMMAND_TEMPLATE, + ConfigConstants.DEFAULT_YARN_CONTAINER_START_COMMAND_TEMPLATE); + final String amCommand = + BootstrapTools.getStartCommand(commandTemplate, startCommandValues); + + amContainer.setCommands(Collections.singletonList(amCommand)); + + LOG.debug("Application Master start command: " + amCommand); + + return amContainer; + } + + private static YarnConfigOptions.UserJarInclusion getUserJarInclusionMode(org.apache.flink.configuration.Configuration config) { + throwIfUserTriesToDisableUserJarInclusionInSystemClassPath(config); + + return config.getEnum(YarnConfigOptions.UserJarInclusion.class, YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR); + } + + private static void throwIfUserTriesToDisableUserJarInclusionInSystemClassPath(final Configuration config) { + final String userJarInclusion = config.getString(YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR); + if ("DISABLED".equalsIgnoreCase(userJarInclusion)) { + throw new IllegalArgumentException(String.format("Config option %s cannot be set to DISABLED anymore (see FLINK-11781)", + YarnConfigOptions.CLASSPATH_INCLUDE_USER_JAR.key())); + } + } + + /** + * Creates a YarnClusterClient; may be overridden in tests. + */ protected ClusterClient createYarnClusterClient( - AbstractYarnClusterDescriptor descriptor, - int numberTaskManagers, - int slotsPerTaskManager, - ApplicationReport report, - Configuration flinkConfiguration, - boolean perJobCluster) throws Exception { + ApplicationReport report, + Configuration flinkConfiguration) throws Exception { return new RestClusterClient<>( flinkConfiguration, report.getApplicationId()); } } + diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java index 2e1f62bef0461391c892c60d6c04f275e1ec7790..3a70086b67a59aec49a92c0d959bae889c71a80c 100755 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/YarnResourceManager.java @@ -32,8 +32,7 @@ import org.apache.flink.runtime.concurrent.FutureUtils; import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.JobLeaderIdService; import org.apache.flink.runtime.resourcemanager.ResourceManager; import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException; @@ -96,6 +95,8 @@ public class YarnResourceManager extends ResourceManager impleme * In task executor we use the hostnames given by YARN consistently throughout akka */ static final String ENV_FLINK_NODE_ID = "_FLINK_NODE_ID"; + static final String ERROR_MASSAGE_ON_SHUTDOWN_REQUEST = "Received shutdown request from YARN ResourceManager."; + /** Default heartbeat interval between this resource manager and the YARN ResourceManager. */ private final int yarnHeartbeatIntervalMillis; @@ -137,12 +138,11 @@ public class YarnResourceManager extends ResourceManager impleme HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, SlotManager slotManager, - MetricRegistry metricRegistry, JobLeaderIdService jobLeaderIdService, ClusterInformation clusterInformation, FatalErrorHandler fatalErrorHandler, @Nullable String webInterfaceUrl, - JobManagerMetricGroup jobManagerMetricGroup) { + ResourceManagerMetricGroup resourceManagerMetricGroup) { super( rpcService, resourceManagerEndpointId, @@ -150,11 +150,10 @@ public class YarnResourceManager extends ResourceManager impleme highAvailabilityServices, heartbeatServices, slotManager, - metricRegistry, jobLeaderIdService, clusterInformation, fatalErrorHandler, - jobManagerMetricGroup); + resourceManagerMetricGroup); this.flinkConfig = flinkConfig; this.yarnConfig = new YarnConfiguration(); this.env = env; @@ -452,7 +451,7 @@ public class YarnResourceManager extends ResourceManager impleme @Override public void onShutdownRequest() { - closeAsync(); + onFatalError(new ResourceManagerException(ERROR_MASSAGE_ON_SHUTDOWN_REQUEST)); } @Override diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java index 06ff77fa32ec203ba9eba08ebadbb4946c7e4dea..e87927897acd82365f5f097f43837e75255af7ba 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/cli/FlinkYarnSessionCli.java @@ -39,7 +39,6 @@ import org.apache.flink.util.ExecutorUtils; import org.apache.flink.util.FlinkException; import org.apache.flink.util.Preconditions; import org.apache.flink.util.ShutdownHookUtil; -import org.apache.flink.yarn.AbstractYarnClusterDescriptor; import org.apache.flink.yarn.YarnClusterDescriptor; import org.apache.flink.yarn.configuration.YarnConfigOptions; @@ -263,13 +262,13 @@ public class FlinkYarnSessionCli extends AbstractCustomCommandLine createDispatcherResourceManagerComponentFactory(Configuration configuration) { - return new JobDispatcherResourceManagerComponentFactory( + protected DefaultDispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration) { + return DefaultDispatcherResourceManagerComponentFactory.createJobComponentFactory( YarnResourceManagerFactory.getInstance(), FileJobGraphRetriever.createFrom(configuration)); } diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnResourceManagerFactory.java b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnResourceManagerFactory.java index 312bb419c1478cd6f8dd1278718e3429429ada46..318ca097938b7dd139ed9d638c0801ad333c1534 100644 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnResourceManagerFactory.java +++ b/flink-yarn/src/main/java/org/apache/flink/yarn/entrypoint/YarnResourceManagerFactory.java @@ -23,8 +23,7 @@ import org.apache.flink.runtime.clusterframework.types.ResourceID; import org.apache.flink.runtime.entrypoint.ClusterInformation; import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.resourcemanager.ActiveResourceManagerFactory; import org.apache.flink.runtime.resourcemanager.ResourceManager; import org.apache.flink.runtime.resourcemanager.ResourceManagerFactory; @@ -57,11 +56,10 @@ public class YarnResourceManagerFactory extends ActiveResourceManagerFactory createDispatcherResourceManagerComponentFactory(Configuration configuration) { - return new SessionDispatcherResourceManagerComponentFactory(YarnResourceManagerFactory.getInstance()); + protected DispatcherResourceManagerComponentFactory createDispatcherResourceManagerComponentFactory(Configuration configuration) { + return DefaultDispatcherResourceManagerComponentFactory.createSessionComponentFactory(YarnResourceManagerFactory.getInstance()); } public static void main(String[] args) { diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/AbstractYarnNonHaServices.java b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/AbstractYarnNonHaServices.java deleted file mode 100644 index cf0b2f945f8080580b5cdbb897d380cb76e65687..0000000000000000000000000000000000000000 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/AbstractYarnNonHaServices.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.yarn.highavailability; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.checkpoint.CheckpointRecoveryFactory; -import org.apache.flink.runtime.checkpoint.StandaloneCheckpointRecoveryFactory; -import org.apache.flink.runtime.highavailability.FsNegativeRunningJobsRegistry; -import org.apache.flink.runtime.highavailability.RunningJobsRegistry; -import org.apache.flink.runtime.jobmanager.JobGraphStore; - -import java.io.IOException; - -/** - * Abstract base class for the high availability services for Flink YARN applications that support - * no master fail over. - * - *

    Internally, these services put their recovery data into YARN's working directory, - * except for checkpoints, which are in the configured checkpoint directory. That way, - * checkpoints can be resumed with a new job/application, even if the complete YARN application - * is killed and cleaned up. - */ -public abstract class AbstractYarnNonHaServices extends YarnHighAvailabilityServices { - - // ------------------------------------------------------------------------ - - /** - * Creates new YARN high-availability services, configuring the file system and recovery - * data directory based on the working directory in the given Hadoop configuration. - * - *

    This class requires that the default Hadoop file system configured in the given - * Hadoop configuration is an HDFS. - * - * @param config The Flink configuration of this component / process. - * @param hadoopConf The Hadoop configuration for the YARN cluster. - * - * @throws IOException Thrown, if the initialization of the Hadoop file system used by YARN fails. - */ - protected AbstractYarnNonHaServices( - Configuration config, - org.apache.hadoop.conf.Configuration hadoopConf) throws IOException { - super(config, hadoopConf); - } - - // ------------------------------------------------------------------------ - // Services - // ------------------------------------------------------------------------ - - @Override - public RunningJobsRegistry getRunningJobsRegistry() throws IOException { - enter(); - try { - // IMPORTANT: The registry must NOT place its data in a directory that is - // cleaned up by these services. - return new FsNegativeRunningJobsRegistry(flinkFileSystem, workingDirectory); - } - finally { - exit(); - } - } - - @Override - public CheckpointRecoveryFactory getCheckpointRecoveryFactory() { - enter(); - try { - return new StandaloneCheckpointRecoveryFactory(); - } - finally { - exit(); - } - } - - @Override - public JobGraphStore getJobGraphStore() throws Exception { - throw new UnsupportedOperationException("These High-Availability Services do not support storing job graphs"); - } -} diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnHighAvailabilityServices.java b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnHighAvailabilityServices.java deleted file mode 100644 index 0cf9dc483c2ea2594e99a29b1f3bbfbc14f1a8ba..0000000000000000000000000000000000000000 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnHighAvailabilityServices.java +++ /dev/null @@ -1,371 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.yarn.highavailability; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.IllegalConfigurationException; -import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.Path; -import org.apache.flink.runtime.blob.BlobStore; -import org.apache.flink.runtime.blob.BlobStoreService; -import org.apache.flink.runtime.blob.FileSystemBlobStore; -import org.apache.flink.runtime.fs.hdfs.HadoopFileSystem; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; -import org.apache.flink.runtime.jobmanager.HighAvailabilityMode; -import org.apache.flink.util.ExceptionUtils; -import org.apache.flink.util.InstantiationUtil; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.util.concurrent.locks.ReentrantLock; - -import static org.apache.flink.util.ExceptionUtils.firstOrSuppressed; -import static org.apache.flink.util.Preconditions.checkNotNull; -import static org.apache.flink.util.Preconditions.checkState; - -/** - * The basis of {@link HighAvailabilityServices} for YARN setups. - * These high-availability services auto-configure YARN's HDFS and the YARN application's - * working directory to be used to store job recovery data. - * - *

    Note for implementers: This class locks access to and creation of services, - * to make sure all services are properly shut down when shutting down this class. - * To participate in the checks, overriding methods should frame method body with - * calls to {@code enter()} and {@code exit()} as shown in the following pattern: - * - * 

    {@code
- * public LeaderRetrievalService getResourceManagerLeaderRetriever() {
- *     enter();
- *     try {
- *         CuratorClient client = getCuratorClient();
- *         return ZooKeeperUtils.createLeaderRetrievalService(client, configuration, RESOURCE_MANAGER_LEADER_PATH);
- *     } finally {
- *         exit();
- *     }
- * }
- * }
    - */ -public abstract class YarnHighAvailabilityServices implements HighAvailabilityServices { - - /** The name of the sub directory in which Flink stores the recovery data. */ - public static final String FLINK_RECOVERY_DATA_DIR = "flink_recovery_data"; - - /** Logger for these services, shared with subclasses. */ - protected static final Logger LOG = LoggerFactory.getLogger(YarnHighAvailabilityServices.class); - - // ------------------------------------------------------------------------ - - /** The lock that guards all accesses to methods in this class. */ - private final ReentrantLock lock; - - /** The Flink FileSystem object that represent the HDFS used by YARN. */ - protected final FileSystem flinkFileSystem; - - /** The Hadoop FileSystem object that represent the HDFS used by YARN. */ - protected final org.apache.hadoop.fs.FileSystem hadoopFileSystem; - - /** The working directory of this YARN application. - * This MUST NOT be deleted when the HA services clean up */ - protected final Path workingDirectory; - - /** The directory for HA persistent data. This should be deleted when the - * HA services clean up. */ - protected final Path haDataDirectory; - - /** Blob store service to be used for the BlobServer and BlobCache. */ - protected final BlobStoreService blobStoreService; - - /** Flag marking this instance as shut down. */ - private volatile boolean closed; - - // ------------------------------------------------------------------------ - - /** - * Creates new YARN high-availability services, configuring the file system and recovery - * data directory based on the working directory in the given Hadoop configuration. - * - *

    This class requires that the default Hadoop file system configured in the given - * Hadoop configuration is an HDFS. - * - * @param config The Flink configuration of this component / process. - * @param hadoopConf The Hadoop configuration for the YARN cluster. - * - * @throws IOException Thrown, if the initialization of the Hadoop file system used by YARN fails. - */ - protected YarnHighAvailabilityServices( - Configuration config, - org.apache.hadoop.conf.Configuration hadoopConf) throws IOException { - - checkNotNull(config); - checkNotNull(hadoopConf); - - this.lock = new ReentrantLock(); - - // get and verify the YARN HDFS URI - final URI fsUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConf); - if (fsUri.getScheme() == null || !"hdfs".equals(fsUri.getScheme().toLowerCase())) { - throw new IOException("Invalid file system found for YarnHighAvailabilityServices: " + - "Expected 'hdfs', but found '" + fsUri.getScheme() + "'."); - } - - // initialize the Hadoop File System - // we go through this special code path here to make sure we get no shared cached - // instance of the FileSystem - try { - final Class fsClass = - org.apache.hadoop.fs.FileSystem.getFileSystemClass(fsUri.getScheme(), hadoopConf); - - this.hadoopFileSystem = InstantiationUtil.instantiate(fsClass); - this.hadoopFileSystem.initialize(fsUri, hadoopConf); - } - catch (Exception e) { - throw new IOException("Cannot instantiate YARN's Hadoop file system for " + fsUri, e); - } - - this.flinkFileSystem = new HadoopFileSystem(hadoopFileSystem); - - this.workingDirectory = new Path(hadoopFileSystem.getWorkingDirectory().toUri()); - this.haDataDirectory = new Path(workingDirectory, FLINK_RECOVERY_DATA_DIR); - - // test the file system, to make sure we fail fast if access does not work - try { - flinkFileSystem.mkdirs(haDataDirectory); - } - catch (Exception e) { - throw new IOException("Could not create the directory for recovery data in YARN's file system at '" - + haDataDirectory + "'.", e); - } - - LOG.info("Flink YARN application will store recovery data at {}", haDataDirectory); - - blobStoreService = new FileSystemBlobStore(flinkFileSystem, haDataDirectory.toString()); - } - - // ------------------------------------------------------------------------ - // high availability services - // ------------------------------------------------------------------------ - - @Override - public BlobStore createBlobStore() throws IOException { - enter(); - try { - return blobStoreService; - } finally { - exit(); - } - } - - // ------------------------------------------------------------------------ - // Shutdown - // ------------------------------------------------------------------------ - - /** - * Checks whether these services have been shut down. - * - * @return True, if this instance has been shut down, false if it still operational. - */ - public boolean isClosed() { - return closed; - } - - @Override - public void close() throws Exception { - lock.lock(); - try { - // close only once - if (closed) { - return; - } - closed = true; - - Throwable exception = null; - - try { - blobStoreService.close(); - } catch (Throwable t) { - exception = t; - } - - // we do not propagate exceptions here, but only log them - try { - hadoopFileSystem.close(); - } catch (Throwable t) { - exception = ExceptionUtils.firstOrSuppressed(t, exception); - } - - if (exception != null) { - ExceptionUtils.rethrowException(exception, "Could not properly close the YarnHighAvailabilityServices."); - } - } - finally { - lock.unlock(); - } - } - - @Override - public void closeAndCleanupAllData() throws Exception { - lock.lock(); - try { - checkState(!closed, "YarnHighAvailabilityServices are already closed"); - - // we remember exceptions only, then continue cleanup, and re-throw at the end - Throwable exception = null; - - try { - blobStoreService.closeAndCleanupAllData(); - } catch (Throwable t) { - exception = t; - } - - // first, we delete all data in Flink's data directory - try { - flinkFileSystem.delete(haDataDirectory, true); - } - catch (Throwable t) { - exception = ExceptionUtils.firstOrSuppressed(t, exception); - } - - // now we actually close the services - try { - close(); - } - catch (Throwable t) { - exception = firstOrSuppressed(t, exception); - } - - // if some exception occurred, rethrow - if (exception != null) { - ExceptionUtils.rethrowException(exception, exception.getMessage()); - } - } - finally { - lock.unlock(); - } - } - - // ------------------------------------------------------------------------ - // Utilities - // ------------------------------------------------------------------------ - - /** - * To be called at the beginning of every method that creates an HA service. Acquires the lock - * and check whether this HighAvailabilityServices instance is shut down. - */ - void enter() { - if (!enterUnlessClosed()) { - throw new IllegalStateException("closed"); - } - } - - /** - * Acquires the lock and checks whether the services are already closed. If they are - * already closed, the method releases the lock and returns {@code false}. - * - * @return True, if the lock was acquired and the services are not closed, false if the services are closed. - */ - boolean enterUnlessClosed() { - lock.lock(); - if (!closed) { - return true; - } else { - lock.unlock(); - return false; - } - } - - /** - * To be called at the end of every method that creates an HA service. Releases the lock. - */ - void exit() { - lock.unlock(); - } - - // ------------------------------------------------------------------------ - // Factory from Configuration - // ------------------------------------------------------------------------ - - /** - * Creates the high-availability services for a single-job Flink YARN application, to be - * used in the Application Master that runs both ResourceManager and JobManager. - * - * @param flinkConfig The Flink configuration. - * @param hadoopConfig The Hadoop configuration for the YARN cluster. - * - * @return The created high-availability services. - * - * @throws IOException Thrown, if the high-availability services could not be initialized. - */ - public static YarnHighAvailabilityServices forSingleJobAppMaster( - Configuration flinkConfig, - org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException { - - checkNotNull(flinkConfig, "flinkConfig"); - checkNotNull(hadoopConfig, "hadoopConfig"); - - final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig); - switch (mode) { - case NONE: - return new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig); - - case ZOOKEEPER: - throw new UnsupportedOperationException("to be implemented"); - - default: - throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode); - } - } - - /** - * Creates the high-availability services for the TaskManagers participating in - * a Flink YARN application. - * - * @param flinkConfig The Flink configuration. - * @param hadoopConfig The Hadoop configuration for the YARN cluster. - * - * @return The created high-availability services. - * - * @throws IOException Thrown, if the high-availability services could not be initialized. - */ - public static YarnHighAvailabilityServices forYarnTaskManager( - Configuration flinkConfig, - org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException { - - checkNotNull(flinkConfig, "flinkConfig"); - checkNotNull(hadoopConfig, "hadoopConfig"); - - final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig); - switch (mode) { - case NONE: - return new YarnPreConfiguredMasterNonHaServices( - flinkConfig, - hadoopConfig, - HighAvailabilityServicesUtils.AddressResolution.TRY_ADDRESS_RESOLUTION); - - case ZOOKEEPER: - throw new UnsupportedOperationException("to be implemented"); - - default: - throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode); - } - } -} diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServices.java b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServices.java deleted file mode 100644 index 466e198dda3a87b5332e5b1592c2d72ed6e19b43..0000000000000000000000000000000000000000 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServices.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.yarn.highavailability; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.IllegalConfigurationException; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.highavailability.ServicesThreadFactory; -import org.apache.flink.runtime.highavailability.nonha.leaderelection.SingleLeaderElectionService; -import org.apache.flink.runtime.leaderelection.LeaderElectionService; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; - -import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * These YarnHighAvailabilityServices are for the Application Master in setups where there is one - * ResourceManager that is statically configured in the Flink configuration. - * - *

    Handled failure types

    - *
      - *
    • User code & operator failures: Failed operators are recovered from checkpoints.
      • - *
    • Task Manager Failures: Failed Task Managers are restarted and their tasks are - * recovered from checkpoints.
      • - *
    - * - *

    Non-recoverable failure types

    - *
      - *
    • Application Master failures: These failures cannot be recovered, because TaskManagers - * have no way to discover the new Application Master's address.
      • - *
    - * - *

    Internally, these services put their recovery data into YARN's working directory, - * except for checkpoints, which are in the configured checkpoint directory. That way, - * checkpoints can be resumed with a new job/application, even if the complete YARN application - * is killed and cleaned up. - * - *

    Because ResourceManager and JobManager run both in the same process (Application Master), they - * use an embedded leader election service to find each other. - * - *

    A typical YARN setup that uses these HA services first starts the ResourceManager - * inside the ApplicationMaster and puts its RPC endpoint address into the configuration with which - * the TaskManagers are started. Because of this static addressing scheme, the setup cannot handle failures - * of the JobManager and ResourceManager, which are running as part of the Application Master. - * - * @see HighAvailabilityServices - */ -public class YarnIntraNonHaMasterServices extends AbstractYarnNonHaServices { - - /** The dispatcher thread pool for these services. */ - private final ExecutorService dispatcher; - - /** The embedded leader election service used by JobManagers to find the resource manager. */ - private final SingleLeaderElectionService resourceManagerLeaderElectionService; - - /** The embedded leader election service for the dispatcher. */ - private final SingleLeaderElectionService dispatcherLeaderElectionService; - - // ------------------------------------------------------------------------ - - /** - * Creates new YarnIntraNonHaMasterServices for the given Flink and YARN configuration. - * - *

    This constructor initializes access to the HDFS to store recovery data, and creates the - * embedded leader election services through which ResourceManager and JobManager find and - * confirm each other. - * - * @param config The Flink configuration of this component / process. - * @param hadoopConf The Hadoop configuration for the YARN cluster. - * - * @throws IOException - * Thrown, if the initialization of the Hadoop file system used by YARN fails. - * @throws IllegalConfigurationException - * Thrown, if the Flink configuration does not properly describe the ResourceManager address and port. - */ - public YarnIntraNonHaMasterServices( - Configuration config, - org.apache.hadoop.conf.Configuration hadoopConf) throws IOException { - - super(config, hadoopConf); - - // track whether we successfully perform the initialization - boolean successful = false; - - try { - this.dispatcher = Executors.newSingleThreadExecutor(new ServicesThreadFactory()); - this.resourceManagerLeaderElectionService = new SingleLeaderElectionService(dispatcher, DEFAULT_LEADER_ID); - this.dispatcherLeaderElectionService = new SingleLeaderElectionService(dispatcher, DEFAULT_LEADER_ID); - - // all good! - successful = true; - } - finally { - if (!successful) { - // quietly undo what the parent constructor initialized - try { - super.close(); - } catch (Throwable ignored) {} - } - } - } - - // ------------------------------------------------------------------------ - // Services - // ------------------------------------------------------------------------ - - @Override - public LeaderRetrievalService getResourceManagerLeaderRetriever() { - enter(); - try { - return resourceManagerLeaderElectionService.createLeaderRetrievalService(); - } - finally { - exit(); - } - } - - @Override - public LeaderRetrievalService getDispatcherLeaderRetriever() { - enter(); - - try { - return dispatcherLeaderElectionService.createLeaderRetrievalService(); - } finally { - exit(); - } - } - - @Override - public LeaderElectionService getResourceManagerLeaderElectionService() { - enter(); - try { - return resourceManagerLeaderElectionService; - } - finally { - exit(); - } - } - - @Override - public LeaderElectionService getDispatcherLeaderElectionService() { - enter(); - try { - return dispatcherLeaderElectionService; - } finally { - exit(); - } - } - - @Override - public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) { - enter(); - try { - throw new UnsupportedOperationException("needs refactoring to accept default address"); - } - finally { - exit(); - } - } - - @Override - public LeaderElectionService getClusterRestEndpointLeaderElectionService() { - enter(); - try { - throw new UnsupportedOperationException(); - } - finally { - exit(); - } - } - - @Override - public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) { - enter(); - try { - throw new UnsupportedOperationException("needs refactoring to accept default address"); - } - finally { - exit(); - } - } - - @Override - public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) { - enter(); - try { - throw new UnsupportedOperationException("needs refactoring to accept default address"); - } - finally { - exit(); - } - } - - @Override - public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() { - enter(); - try { - throw new UnsupportedOperationException(); - } - finally { - exit(); - } - } - - // ------------------------------------------------------------------------ - // shutdown - // ------------------------------------------------------------------------ - - @Override - public void close() throws Exception { - if (enterUnlessClosed()) { - try { - try { - // this class' own cleanup logic - resourceManagerLeaderElectionService.shutdown(); - dispatcher.shutdownNow(); - } - finally { - // in any case must we call the parent cleanup logic - super.close(); - } - } - finally { - exit(); - } - } - } -} diff --git a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java b/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java deleted file mode 100644 index f80c6b17c73bd150a1eefe4ab83772c880868715..0000000000000000000000000000000000000000 --- a/flink-yarn/src/main/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterNonHaServices.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.yarn.highavailability; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.IllegalConfigurationException; -import org.apache.flink.runtime.dispatcher.Dispatcher; -import org.apache.flink.runtime.highavailability.HighAvailabilityServices; -import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; -import org.apache.flink.runtime.leaderelection.LeaderElectionService; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; -import org.apache.flink.runtime.leaderretrieval.StandaloneLeaderRetrievalService; -import org.apache.flink.runtime.resourcemanager.ResourceManager; -import org.apache.flink.runtime.rpc.akka.AkkaRpcServiceUtils; -import org.apache.flink.yarn.configuration.YarnConfigOptions; - -import java.io.IOException; - -/** - * These YarnHighAvailabilityServices are for use by the TaskManager in setups, - * where there is one ResourceManager that is statically configured in the Flink configuration. - * - *

    Handled failure types

    - *
      - *
    • User code & operator failures: Failed operators are recovered from checkpoints.
      • - *
    • Task Manager Failures: Failed Task Managers are restarted and their tasks are - * recovered from checkpoints.
      • - *
    - * - *

    Non-recoverable failure types

    - *
      - *
    • Application Master failures: These failures cannot be recovered, because TaskManagers - * have no way to discover the new Application Master's address.
      • - *
    - * - *

    Internally, these services put their recovery data into YARN's working directory, - * except for checkpoints, which are in the configured checkpoint directory. That way, - * checkpoints can be resumed with a new job/application, even if the complete YARN application - * is killed and cleaned up. - * - *

    A typical YARN setup that uses these HA services first starts the ResourceManager - * inside the ApplicationMaster and puts its RPC endpoint address into the configuration with which - * the TaskManagers are started. Because of this static addressing scheme, the setup cannot handle failures - * of the JobManager and ResourceManager, which are running as part of the Application Master. - * - * @see HighAvailabilityServices - */ -public class YarnPreConfiguredMasterNonHaServices extends AbstractYarnNonHaServices { - - /** The RPC URL under which the single ResourceManager can be reached while available. */ - private final String resourceManagerRpcUrl; - - /** The RPC URL under which the single Dispatcher can be reached while available. */ - private final String dispatcherRpcUrl; - - // ------------------------------------------------------------------------ - - /** - * Creates new YarnPreConfiguredMasterHaServices for the given Flink and YARN configuration. - * This constructor parses the ResourceManager address from the Flink configuration and sets - * up the HDFS access to store recovery data in the YARN application's working directory. - * - * @param config The Flink configuration of this component / process. - * @param hadoopConf The Hadoop configuration for the YARN cluster. - * - * @throws IOException - * Thrown, if the initialization of the Hadoop file system used by YARN fails. - * @throws IllegalConfigurationException - * Thrown, if the Flink configuration does not properly describe the ResourceManager address and port. - */ - public YarnPreConfiguredMasterNonHaServices( - Configuration config, - org.apache.hadoop.conf.Configuration hadoopConf, - HighAvailabilityServicesUtils.AddressResolution addressResolution) throws IOException { - - super(config, hadoopConf); - - // track whether we successfully perform the initialization - boolean successful = false; - - try { - // extract the hostname and port of the resource manager - final String rmHost = config.getString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS); - final int rmPort = config.getInteger(YarnConfigOptions.APP_MASTER_RPC_PORT); - - if (rmHost == null) { - throw new IllegalConfigurationException("Config parameter '" + - YarnConfigOptions.APP_MASTER_RPC_ADDRESS.key() + "' is missing."); - } - if (rmPort < 0) { - throw new IllegalConfigurationException("Config parameter '" + - YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' is missing."); - } - if (rmPort <= 0 || rmPort >= 65536) { - throw new IllegalConfigurationException("Invalid value for '" + - YarnConfigOptions.APP_MASTER_RPC_PORT.key() + "' - port must be in [1, 65535]"); - } - - this.resourceManagerRpcUrl = AkkaRpcServiceUtils.getRpcUrl( - rmHost, - rmPort, - ResourceManager.RESOURCE_MANAGER_NAME, - addressResolution, - config); - - this.dispatcherRpcUrl = AkkaRpcServiceUtils.getRpcUrl( - rmHost, - rmPort, - Dispatcher.DISPATCHER_NAME, - addressResolution, - config); - - // all well! - successful = true; - } - finally { - if (!successful) { - // quietly undo what the parent constructor initialized - try { - super.close(); - } catch (Throwable ignored) {} - } - } - } - - // ------------------------------------------------------------------------ - // Services - // ------------------------------------------------------------------------ - - @Override - public LeaderRetrievalService getResourceManagerLeaderRetriever() { - enter(); - try { - return new StandaloneLeaderRetrievalService(resourceManagerRpcUrl, DEFAULT_LEADER_ID); - } - finally { - exit(); - } - } - - @Override - public LeaderRetrievalService getDispatcherLeaderRetriever() { - enter(); - - try { - return new StandaloneLeaderRetrievalService(dispatcherRpcUrl, DEFAULT_LEADER_ID); - } finally { - exit(); - } - } - - @Override - public LeaderElectionService getResourceManagerLeaderElectionService() { - enter(); - try { - throw new UnsupportedOperationException("Not supported on the TaskManager side"); - } - finally { - exit(); - } - } - - @Override - public LeaderElectionService getDispatcherLeaderElectionService() { - enter(); - try { - throw new UnsupportedOperationException("Not supported on the TaskManager side"); - } finally { - exit(); - } - } - - @Override - public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) { - enter(); - try { - throw new UnsupportedOperationException("needs refactoring to accept default address"); - } - finally { - exit(); - } - } - - @Override - public LeaderElectionService getClusterRestEndpointLeaderElectionService() { - enter(); - try { - throw new UnsupportedOperationException(); - } - finally { - exit(); - } - } - - @Override - public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) { - enter(); - try { - throw new UnsupportedOperationException("needs refactoring to accept default address"); - } - finally { - exit(); - } - } - - @Override - public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) { - enter(); - try { - return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID); - } finally { - exit(); - } - } - - @Override - public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() { - enter(); - try { - throw new UnsupportedOperationException(); - } - finally { - exit(); - } - } -} diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java index 8ee08db1bb38909d2ea9cdbef655dd883a67f143..9009a53b877100ecda476d2053010b783116ea99 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/AbstractYarnClusterTest.java @@ -18,12 +18,8 @@ package org.apache.flink.yarn; -import org.apache.flink.client.deployment.ClusterDeploymentException; import org.apache.flink.client.deployment.ClusterRetrieveException; -import org.apache.flink.client.deployment.ClusterSpecification; -import org.apache.flink.client.program.ClusterClient; import org.apache.flink.configuration.Configuration; -import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.util.Preconditions; import org.apache.flink.util.TestLogger; @@ -45,7 +41,7 @@ import java.util.Collections; import java.util.Map; /** - * Tests for the {@link AbstractYarnClusterDescriptor}. + * Tests for the {@link YarnClusterDescriptor}. */ public class AbstractYarnClusterTest extends TestLogger { @@ -68,7 +64,7 @@ public class AbstractYarnClusterTest extends TestLogger { yarnClient.init(yarnConfiguration); yarnClient.start(); - final TestingAbstractYarnClusterDescriptor clusterDescriptor = new TestingAbstractYarnClusterDescriptor( + final YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor( new Configuration(), yarnConfiguration, temporaryFolder.newFolder().getAbsolutePath(), @@ -126,36 +122,4 @@ public class AbstractYarnClusterTest extends TestLogger { } } } - - private static final class TestingAbstractYarnClusterDescriptor extends AbstractYarnClusterDescriptor { - - private TestingAbstractYarnClusterDescriptor( - Configuration flinkConfiguration, - YarnConfiguration yarnConfiguration, - String configurationDirectory, - YarnClient yarnClient, - boolean sharedYarnClient) { - super(flinkConfiguration, yarnConfiguration, configurationDirectory, yarnClient, sharedYarnClient); - } - - @Override - protected String getYarnSessionClusterEntrypoint() { - throw new UnsupportedOperationException("Not needed for testing"); - } - - @Override - protected String getYarnJobClusterEntrypoint() { - throw new UnsupportedOperationException("Not needed for testing"); - } - - @Override - protected ClusterClient createYarnClusterClient(AbstractYarnClusterDescriptor descriptor, int numberTaskManagers, int slotsPerTaskManager, ApplicationReport report, Configuration flinkConfiguration, boolean perJobCluster) throws Exception { - throw new UnsupportedOperationException("Not needed for testing"); - } - - @Override - public ClusterClient deployJobCluster(ClusterSpecification clusterSpecification, JobGraph jobGraph, boolean detached) throws ClusterDeploymentException { - throw new UnsupportedOperationException("Not needed for testing"); - } - } } diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java index a71f139b58c8f32307054f13a74e391cd9bf1020..23388751133aca5ecf2937d9d78be025b42009aa 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/FlinkYarnSessionCliTest.java @@ -85,7 +85,7 @@ public class FlinkYarnSessionCliTest extends TestLogger { CommandLine cmd = parser.parse(options, new String[]{"run", "-j", "fake.jar", "-D", "akka.ask.timeout=5 min", "-D", "env.java.opts=-DappName=foobar", "-D", "security.ssl.internal.key-password=changeit"}); - AbstractYarnClusterDescriptor flinkYarnDescriptor = cli.createClusterDescriptor(cmd); + YarnClusterDescriptor flinkYarnDescriptor = cli.createClusterDescriptor(cmd); Assert.assertNotNull(flinkYarnDescriptor); @@ -110,7 +110,7 @@ public class FlinkYarnSessionCliTest extends TestLogger { final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true); - AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); + YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine); @@ -132,7 +132,7 @@ public class FlinkYarnSessionCliTest extends TestLogger { final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true); - AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); + YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); // each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased. assertTrue(descriptor.isDetachedMode()); @@ -152,7 +152,7 @@ public class FlinkYarnSessionCliTest extends TestLogger { CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true); - AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); + YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); assertEquals(zkNamespaceCliInput, descriptor.getZookeeperNamespace()); } @@ -171,7 +171,7 @@ public class FlinkYarnSessionCliTest extends TestLogger { CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true); - AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); + YarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); assertEquals(nodeLabelCliInput, descriptor.getNodeLabel()); } @@ -246,7 +246,7 @@ public class FlinkYarnSessionCliTest extends TestLogger { final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true); - final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine); + final YarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine); final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration(); @@ -267,7 +267,7 @@ public class FlinkYarnSessionCliTest extends TestLogger { final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true); - final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine); + final YarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine); final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration(); @@ -462,9 +462,9 @@ public class FlinkYarnSessionCliTest extends TestLogger { final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); - AbstractYarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine); + YarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine); - assertEquals(2, flinkYarnDescriptor.shipFiles.size()); + assertEquals(2, flinkYarnDescriptor.getShipFiles().size()); } diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java index 4940f12bd18f358b45ae5a2bf9ad538c97382128..143749b049a1b5e8340ba4bda66c52789079ac51 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnClusterDescriptorTest.java @@ -426,8 +426,8 @@ public class YarnClusterDescriptorTest extends TestLogger { File libFile = temporaryFolder.newFile("libFile.jar"); File libFolder = temporaryFolder.newFolder().getAbsoluteFile(); - Assert.assertFalse(descriptor.shipFiles.contains(libFile)); - Assert.assertFalse(descriptor.shipFiles.contains(libFolder)); + Assert.assertFalse(descriptor.getShipFiles().contains(libFile)); + Assert.assertFalse(descriptor.getShipFiles().contains(libFolder)); List shipFiles = new ArrayList<>(); shipFiles.add(libFile); @@ -435,17 +435,17 @@ public class YarnClusterDescriptorTest extends TestLogger { descriptor.addShipFiles(shipFiles); - Assert.assertTrue(descriptor.shipFiles.contains(libFile)); - Assert.assertTrue(descriptor.shipFiles.contains(libFolder)); + Assert.assertTrue(descriptor.getShipFiles().contains(libFile)); + Assert.assertTrue(descriptor.getShipFiles().contains(libFolder)); // only execute part of the deployment to test for shipped files Set effectiveShipFiles = new HashSet<>(); descriptor.addEnvironmentFoldersToShipFiles(effectiveShipFiles); Assert.assertEquals(0, effectiveShipFiles.size()); - Assert.assertEquals(2, descriptor.shipFiles.size()); - Assert.assertTrue(descriptor.shipFiles.contains(libFile)); - Assert.assertTrue(descriptor.shipFiles.contains(libFolder)); + Assert.assertEquals(2, descriptor.getShipFiles().size()); + Assert.assertTrue(descriptor.getShipFiles().contains(libFile)); + Assert.assertTrue(descriptor.getShipFiles().contains(libFolder)); } } @@ -481,8 +481,8 @@ public class YarnClusterDescriptorTest extends TestLogger { // only add the ship the folder, not the contents Assert.assertFalse(effectiveShipFiles.contains(libFile)); Assert.assertTrue(effectiveShipFiles.contains(libFolder)); - Assert.assertFalse(descriptor.shipFiles.contains(libFile)); - Assert.assertFalse(descriptor.shipFiles.contains(libFolder)); + Assert.assertFalse(descriptor.getShipFiles().contains(libFile)); + Assert.assertFalse(descriptor.getShipFiles().contains(libFolder)); } } diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTest.java index d0fad6dd4cab2c3bc90deb843c15b18fb2f487c3..13d296dfeb8feb7271667239afdbcc8ed2afb265 100644 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTest.java @@ -179,7 +179,7 @@ public class YarnFileStageTest extends TestLogger { try { List remotePaths = new ArrayList<>(); HashMap localResources = new HashMap<>(); - final List classpath = AbstractYarnClusterDescriptor.uploadAndRegisterFiles( + final List classpath = YarnClusterDescriptor.uploadAndRegisterFiles( Collections.singletonList(new File(srcPath.toUri().getPath())), targetFileSystem, targetDir, diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java index df7d85b892b61cdad2f689d63efc89fa637a5845..a4e681235b7e0c3f55a43ab6c9745c65054bc0fa 100755 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java +++ b/flink-yarn/src/test/java/org/apache/flink/yarn/YarnResourceManagerTest.java @@ -32,13 +32,13 @@ import org.apache.flink.runtime.heartbeat.HeartbeatServices; import org.apache.flink.runtime.highavailability.HighAvailabilityServices; import org.apache.flink.runtime.instance.HardwareDescription; import org.apache.flink.runtime.messages.Acknowledge; -import org.apache.flink.runtime.metrics.MetricRegistry; -import org.apache.flink.runtime.metrics.groups.JobManagerMetricGroup; +import org.apache.flink.runtime.metrics.groups.ResourceManagerMetricGroup; import org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups; import org.apache.flink.runtime.registration.RegistrationResponse; import org.apache.flink.runtime.resourcemanager.JobLeaderIdService; import org.apache.flink.runtime.resourcemanager.ResourceManagerGateway; import org.apache.flink.runtime.resourcemanager.SlotRequest; +import org.apache.flink.runtime.resourcemanager.exceptions.ResourceManagerException; import org.apache.flink.runtime.resourcemanager.slotmanager.SlotManager; import org.apache.flink.runtime.resourcemanager.utils.MockResourceManagerRuntimeServices; import org.apache.flink.runtime.rpc.FatalErrorHandler; @@ -49,6 +49,7 @@ import org.apache.flink.runtime.taskexecutor.SlotStatus; import org.apache.flink.runtime.taskexecutor.TaskExecutorGateway; import org.apache.flink.runtime.taskexecutor.TaskExecutorRegistrationSuccess; import org.apache.flink.runtime.util.TestingFatalErrorHandler; +import org.apache.flink.util.ExceptionUtils; import org.apache.flink.util.TestLogger; import org.apache.flink.util.function.RunnableWithException; @@ -84,6 +85,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; import static org.apache.flink.yarn.YarnConfigKeys.ENV_APP_ID; import static org.apache.flink.yarn.YarnConfigKeys.ENV_CLIENT_HOME_DIR; @@ -92,7 +94,9 @@ import static org.apache.flink.yarn.YarnConfigKeys.ENV_FLINK_CLASSPATH; import static org.apache.flink.yarn.YarnConfigKeys.ENV_HADOOP_USER_NAME; import static org.apache.flink.yarn.YarnConfigKeys.FLINK_JAR_PATH; import static org.apache.flink.yarn.YarnConfigKeys.FLINK_YARN_FILES; +import static org.apache.flink.yarn.YarnResourceManager.ERROR_MASSAGE_ON_SHUTDOWN_REQUEST; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; @@ -167,14 +171,13 @@ public class YarnResourceManagerTest extends TestLogger { HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, SlotManager slotManager, - MetricRegistry metricRegistry, JobLeaderIdService jobLeaderIdService, ClusterInformation clusterInformation, FatalErrorHandler fatalErrorHandler, @Nullable String webInterfaceUrl, AMRMClientAsync mockResourceManagerClient, NMClient mockNMClient, - JobManagerMetricGroup jobManagerMetricGroup) { + ResourceManagerMetricGroup resourceManagerMetricGroup) { super( rpcService, resourceManagerEndpointId, @@ -184,12 +187,11 @@ public class YarnResourceManagerTest extends TestLogger { highAvailabilityServices, heartbeatServices, slotManager, - metricRegistry, jobLeaderIdService, clusterInformation, fatalErrorHandler, webInterfaceUrl, - jobManagerMetricGroup); + resourceManagerMetricGroup); this.mockNMClient = mockNMClient; this.mockResourceManagerClient = mockResourceManagerClient; } @@ -246,9 +248,6 @@ public class YarnResourceManagerTest extends TestLogger { @SuppressWarnings("unchecked") public AMRMClientAsync mockResourceManagerClient = mock(AMRMClientAsync.class); - public JobManagerMetricGroup mockJMMetricGroup = - UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(); - /** * Create mock RM dependencies. */ @@ -272,14 +271,13 @@ public class YarnResourceManagerTest extends TestLogger { rmServices.highAvailabilityServices, rmServices.heartbeatServices, rmServices.slotManager, - rmServices.metricRegistry, rmServices.jobLeaderIdService, new ClusterInformation("localhost", 1234), testingFatalErrorHandler, null, mockResourceManagerClient, mockNMClient, - mockJMMetricGroup); + UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup()); } /** @@ -343,6 +341,21 @@ public class YarnResourceManagerTest extends TestLogger { return mockContainerStatus; } + @Test + public void testShutdownRequestCausesFatalError() throws Exception { + new Context() {{ + runTest(() -> { + resourceManager.onShutdownRequest(); + + Throwable t = testingFatalErrorHandler.getErrorFuture().get(2000L, TimeUnit.MILLISECONDS); + assertThat(ExceptionUtils.findThrowable(t, ResourceManagerException.class).isPresent(), is(true)); + assertThat(ExceptionUtils.findThrowableWithMessage(t, ERROR_MASSAGE_ON_SHUTDOWN_REQUEST).isPresent(), is(true)); + + testingFatalErrorHandler.clearError(); + }); + }}; + } + @Test public void testStopWorker() throws Exception { new Context() {{ diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServicesTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServicesTest.java deleted file mode 100644 index bdc7863c79a63d041d669ce42e0d6477d2719ff5..0000000000000000000000000000000000000000 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnIntraNonHaMasterServicesTest.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.yarn.highavailability; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.core.fs.Path; -import org.apache.flink.runtime.leaderelection.LeaderContender; -import org.apache.flink.runtime.leaderelection.LeaderElectionService; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalListener; -import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; -import org.apache.flink.util.OperatingSystem; -import org.apache.flink.util.StringUtils; -import org.apache.flink.util.TestLogger; - -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import java.io.File; -import java.util.Random; -import java.util.UUID; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * Tests for YarnIntraNonHaMasterServices. - */ -public class YarnIntraNonHaMasterServicesTest extends TestLogger { - - private static final Random RND = new Random(); - - @ClassRule - public static final TemporaryFolder TEMP_DIR = new TemporaryFolder(); - - private static MiniDFSCluster hdfsCluster; - - private static Path hdfsRootPath; - - private org.apache.hadoop.conf.Configuration hadoopConfig; - - // ------------------------------------------------------------------------ - // Test setup and shutdown - // ------------------------------------------------------------------------ - - @BeforeClass - public static void createHDFS() throws Exception { - Assume.assumeTrue(!OperatingSystem.isWindows()); - - final File tempDir = TEMP_DIR.newFolder(); - - org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration(); - hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath()); - - MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); - hdfsCluster = builder.build(); - hdfsRootPath = new Path(hdfsCluster.getURI()); - } - - @AfterClass - public static void destroyHDFS() { - if (hdfsCluster != null) { - hdfsCluster.shutdown(); - } - hdfsCluster = null; - hdfsRootPath = null; - } - - @Before - public void initConfig() { - hadoopConfig = new org.apache.hadoop.conf.Configuration(); - hadoopConfig.set(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, hdfsRootPath.toString()); - } - - // ------------------------------------------------------------------------ - // Tests - // ------------------------------------------------------------------------ - - @Test - public void testRepeatedClose() throws Exception { - final Configuration flinkConfig = new Configuration(); - - final YarnHighAvailabilityServices services = new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig); - services.closeAndCleanupAllData(); - - // this should not throw an exception - services.close(); - } - - @Test - public void testClosingReportsToLeader() throws Exception { - final Configuration flinkConfig = new Configuration(); - - try (YarnHighAvailabilityServices services = new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig)) { - final LeaderElectionService elector = services.getResourceManagerLeaderElectionService(); - final LeaderRetrievalService retrieval = services.getResourceManagerLeaderRetriever(); - final LeaderContender contender = mockContender(elector); - final LeaderRetrievalListener listener = mock(LeaderRetrievalListener.class); - - elector.start(contender); - retrieval.start(listener); - - // wait until the contender has become the leader - verify(listener, timeout(1000L).times(1)).notifyLeaderAddress(anyString(), any(UUID.class)); - - // now we can close the election service - services.close(); - - verify(contender, timeout(1000L).times(1)).handleError(any(Exception.class)); - } - } - - // ------------------------------------------------------------------------ - // utilities - // ------------------------------------------------------------------------ - - private static LeaderContender mockContender(final LeaderElectionService service) { - String address = StringUtils.getRandomString(RND, 5, 10, 'a', 'z'); - return mockContender(service, address); - } - - private static LeaderContender mockContender(final LeaderElectionService service, final String address) { - LeaderContender mockContender = mock(LeaderContender.class); - - when(mockContender.getAddress()).thenReturn(address); - - doAnswer(new Answer() { - @Override - public Void answer(InvocationOnMock invocation) throws Throwable { - final UUID uuid = (UUID) invocation.getArguments()[0]; - service.confirmLeaderSessionID(uuid); - return null; - } - }).when(mockContender).grantLeadership(any(UUID.class)); - - return mockContender; - } -} diff --git a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java b/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java deleted file mode 100644 index 67186e985b32a310d7da8b620228b8ca3d0c6f4e..0000000000000000000000000000000000000000 --- a/flink-yarn/src/test/java/org/apache/flink/yarn/highavailability/YarnPreConfiguredMasterHaServicesTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.yarn.highavailability; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.IllegalConfigurationException; -import org.apache.flink.core.fs.FileSystem; -import org.apache.flink.core.fs.Path; -import org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils; -import org.apache.flink.util.OperatingSystem; -import org.apache.flink.util.TestLogger; -import org.apache.flink.yarn.configuration.YarnConfigOptions; - -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.junit.AfterClass; -import org.junit.Assume; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.FileNotFoundException; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * Tests for YarnPreConfiguredMasterNonHaServices. - */ -public class YarnPreConfiguredMasterHaServicesTest extends TestLogger { - - @ClassRule - public static final TemporaryFolder TEMP_DIR = new TemporaryFolder(); - - private static MiniDFSCluster hdfsCluster; - - private static Path hdfsRootPath; - - private org.apache.hadoop.conf.Configuration hadoopConfig; - - // ------------------------------------------------------------------------ - // Test setup and shutdown - // ------------------------------------------------------------------------ - - @BeforeClass - public static void createHDFS() throws Exception { - Assume.assumeTrue(!OperatingSystem.isWindows()); - - final File tempDir = TEMP_DIR.newFolder(); - - org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration(); - hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tempDir.getAbsolutePath()); - - MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); - hdfsCluster = builder.build(); - hdfsRootPath = new Path(hdfsCluster.getURI()); - } - - @AfterClass - public static void destroyHDFS() { - if (hdfsCluster != null) { - hdfsCluster.shutdown(); - } - hdfsCluster = null; - hdfsRootPath = null; - } - - @Before - public void initConfig() { - hadoopConfig = new org.apache.hadoop.conf.Configuration(); - hadoopConfig.set(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, hdfsRootPath.toString()); - } - - // ------------------------------------------------------------------------ - // Tests - // ------------------------------------------------------------------------ - - @Test - public void testMissingRmConfiguration() throws Exception { - final Configuration flinkConfig = new Configuration(); - - // missing resource manager address - try { - new YarnPreConfiguredMasterNonHaServices( - flinkConfig, - hadoopConfig, - HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION); - fail(); - } catch (IllegalConfigurationException e) { - // expected - } - - flinkConfig.setString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS, "localhost"); - - // missing resource manager port - try { - new YarnPreConfiguredMasterNonHaServices( - flinkConfig, - hadoopConfig, - HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION); - fail(); - } catch (IllegalConfigurationException e) { - // expected - } - - flinkConfig.setInteger(YarnConfigOptions.APP_MASTER_RPC_PORT, 1427); - - // now everything is good ;-) - new YarnPreConfiguredMasterNonHaServices( - flinkConfig, - hadoopConfig, - HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION).closeAndCleanupAllData(); - } - - @Test - public void testCloseAndCleanup() throws Exception { - final Configuration flinkConfig = new Configuration(); - flinkConfig.setString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS, "localhost"); - flinkConfig.setInteger(YarnConfigOptions.APP_MASTER_RPC_PORT, 1427); - - // create the services - YarnHighAvailabilityServices services = new YarnPreConfiguredMasterNonHaServices( - flinkConfig, - hadoopConfig, - HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION); - services.closeAndCleanupAllData(); - - final FileSystem fileSystem = hdfsRootPath.getFileSystem(); - final Path workDir = new Path(hdfsCluster.getFileSystem().getWorkingDirectory().toString()); - - try { - fileSystem.getFileStatus(new Path(workDir, YarnHighAvailabilityServices.FLINK_RECOVERY_DATA_DIR)); - fail("Flink recovery data directory still exists"); - } - catch (FileNotFoundException e) { - // expected, because the directory should have been cleaned up - } - - assertTrue(services.isClosed()); - - // doing another cleanup when the services are closed should fail - try { - services.closeAndCleanupAllData(); - fail("should fail with an IllegalStateException"); - } catch (IllegalStateException e) { - // expected - } - } - - @Test - public void testCallsOnClosedServices() throws Exception { - final Configuration flinkConfig = new Configuration(); - flinkConfig.setString(YarnConfigOptions.APP_MASTER_RPC_ADDRESS, "localhost"); - flinkConfig.setInteger(YarnConfigOptions.APP_MASTER_RPC_PORT, 1427); - - YarnHighAvailabilityServices services = new YarnPreConfiguredMasterNonHaServices( - flinkConfig, - hadoopConfig, - HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION); - - // this method is not supported - try { - services.getJobGraphStore(); - fail(); - } catch (UnsupportedOperationException ignored) {} - - services.close(); - - // all these methods should fail now - - try { - services.createBlobStore(); - fail(); - } catch (IllegalStateException ignored) {} - - try { - services.getCheckpointRecoveryFactory(); - fail(); - } catch (IllegalStateException ignored) {} - - try { - services.getJobManagerLeaderElectionService(new JobID()); - fail(); - } catch (IllegalStateException ignored) {} - - try { - services.getJobManagerLeaderRetriever(new JobID()); - fail(); - } catch (IllegalStateException ignored) {} - - try { - services.getRunningJobsRegistry(); - fail(); - } catch (IllegalStateException ignored) {} - - try { - services.getResourceManagerLeaderElectionService(); - fail(); - } catch (IllegalStateException ignored) {} - - try { - services.getResourceManagerLeaderRetriever(); - fail(); - } catch (IllegalStateException ignored) {} - } -} diff --git a/licenses-binary/LICENSE-hdrhistogram b/licenses-binary/LICENSE-hdrhistogram deleted file mode 100644 index 09c38ea51608e0156754ec01cce6246d8bc0f0fe..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE-hdrhistogram +++ /dev/null @@ -1,125 +0,0 @@ -The code was Written by Gil Tene, Michael Barker, and Matt Warren, -and released to the public domain, as explained at -http://creativecommons.org/publicdomain/zero/1.0/ - -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. diff --git a/licenses-binary/LICENSE-re2j b/licenses-binary/LICENSE-re2j deleted file mode 100644 index b620ae68fe332112e50d1fc5da4a32cd6191681a..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE-re2j +++ /dev/null @@ -1,32 +0,0 @@ -This is a work derived from Russ Cox's RE2 in Go, whose license -http://golang.org/LICENSE is as follows: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - - * Neither the name of Google Inc. nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE-stax2api b/licenses-binary/LICENSE-stax2api deleted file mode 100644 index 0ed6361699658bda41a64e6a1df34901fe3fd1f5..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE-stax2api +++ /dev/null @@ -1,22 +0,0 @@ -Copyright woodstox stax2api contributors. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, -INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, -BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE-xmlenc b/licenses-binary/LICENSE-xmlenc deleted file mode 100644 index a1256bdf6f8fd625193e649dbb0789fd3d588b27..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE-xmlenc +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2003-2005, Ernst de Haan -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/licenses-binary/LICENSE.@angular b/licenses-binary/LICENSE.@angular deleted file mode 100644 index 828d3633f735937ffecece33687bf31a30572ea5..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.@angular +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License - -Copyright (c) 2010-2019 Google LLC. http://angular.io/license - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/licenses-binary/LICENSE.angular b/licenses-binary/LICENSE.angular deleted file mode 100644 index 91f0644936811dc7e59e1cbc258022cdb770b8af..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.angular +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License - -Copyright (c) 2010-2018 Google, Inc. http://angularjs.org - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - diff --git a/licenses-binary/LICENSE.angular-drag-and-drop-list b/licenses-binary/LICENSE.angular-drag-and-drop-list deleted file mode 100644 index 1787fc65befe1678b6e5159d83a39bbf9d1ad2ec..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.angular-drag-and-drop-list +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Marcel Juenemann -Copyright (c) 2014-2016 Google Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.angular-moment b/licenses-binary/LICENSE.angular-moment deleted file mode 100644 index fe9db79b9113a7d0034394d29559aef67ea92c9d..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.angular-moment +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2013-2016 Uri Shaked and contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/licenses-binary/LICENSE.angular-ui-router b/licenses-binary/LICENSE.angular-ui-router deleted file mode 100644 index 6413b092d70f7bb0866e78d3c9f9f841a8402966..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.angular-ui-router +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License - -Copyright (c) 2013-2015 The AngularUI Team, Karsten Sperling - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/licenses-binary/LICENSE.ant-design-palettes b/licenses-binary/LICENSE.ant-design-palettes deleted file mode 100644 index dfe5c2183e82554ef991024afb81baac82c7e10a..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.ant-design-palettes +++ /dev/null @@ -1,22 +0,0 @@ -MIT LICENSE - -Copyright (c) 2018-present Ant UED, https://xtech.antfin.com/ - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.asm b/licenses-binary/LICENSE.asm deleted file mode 100644 index 62ffbccb6837a25a3baca7f86249d08acff91cf4..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.asm +++ /dev/null @@ -1,31 +0,0 @@ -ASM: a very small and fast Java bytecode manipulation framework - -Copyright (c) 2000-2011 INRIA, France Telecom -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holders nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF -THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.base64 b/licenses-binary/LICENSE.base64 deleted file mode 100644 index 31ebc840539c1a88450cd6999d7b0d6d0a59c2ed..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.base64 +++ /dev/null @@ -1,26 +0,0 @@ -The person or persons who have associated work with this document (the -"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of -his knowledge, the work of authorship identified is in the public domain of -the country from which the work is published, or (b) hereby dedicates whatever -copyright the dedicators holds in the work of authorship identified below (the -"Work") to the public domain. A certifier, moreover, dedicates any copyright -interest he may have in the associated work, and for these purposes, is -described as a "dedicator" below. - -A certifier has taken reasonable steps to verify the copyright status of this -work. Certifier recognizes that his good faith efforts may not shield him from -liability if in fact the work certified is not in the public domain. - -Dedicator makes this dedication for the benefit of the public at large and to -the detriment of the Dedicator's heirs and successors. Dedicator intends this -dedication to be an overt act of relinquishment in perpetuate of all present -and future rights under copyright law, whether vested or contingent, in the -Work. Dedicator understands that such relinquishment of all rights includes -the relinquishment of all rights to enforce (by lawsuit or otherwise) those -copyrights in the Work. - -Dedicator recognizes that, once placed in the public domain, the Work may be -freely reproduced, distributed, transmitted, used, modified, built upon, or -otherwise exploited by anyone for any purpose, commercial or non-commercial, -and in any way, including by methods that have not yet been invented or -conceived. diff --git a/licenses-binary/LICENSE.bootstrap b/licenses-binary/LICENSE.bootstrap deleted file mode 100644 index 6ca0ceecc5d792951d4b5ab2743066b48a88857b..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.bootstrap +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2011-2018 Twitter, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/licenses-binary/LICENSE.core-js b/licenses-binary/LICENSE.core-js deleted file mode 100644 index 8cc76f6c185d94bcaaae9c850c93fc8d46cd1c35..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.core-js +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License - -Copyright (c) 2019 Google LLC. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/licenses-binary/LICENSE.d3 b/licenses-binary/LICENSE.d3 deleted file mode 100644 index 1d9d875edb469786a9c6dbae5aaeef9a3bcb2c33..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.d3 +++ /dev/null @@ -1,27 +0,0 @@ -Copyright 2010-2017 Mike Bostock -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the author nor the names of contributors may be used to - endorse or promote products derived from this software without specific prior - written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.dagre b/licenses-binary/LICENSE.dagre deleted file mode 100644 index 7d7dd94248124266417a4438f2e8900578b637a8..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.dagre +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2012-2014 Chris Pettitt - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/licenses-binary/LICENSE.dagre-d3 b/licenses-binary/LICENSE.dagre-d3 deleted file mode 100644 index 1d64ed68ce64d9ab5a1d6fc3d3ee0b26a0dadab8..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.dagre-d3 +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2013 Chris Pettitt - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/licenses-binary/LICENSE.ev-emitter b/licenses-binary/LICENSE.ev-emitter deleted file mode 100644 index 3e30b1aeff26a9130b03305fc0df8845cdff1bf7..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.ev-emitter +++ /dev/null @@ -1,7 +0,0 @@ -Copyright © 2018 David DeSandro - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.font-awesome b/licenses-binary/LICENSE.font-awesome deleted file mode 100644 index b9fb2c6e6ef062be3ce26c6191adb30978916737..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.font-awesome +++ /dev/null @@ -1,97 +0,0 @@ -The MIT license - -Copyright - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - - SIL OPEN FONT LICENSE - -Version 1.1 - 26 February 2007 -PREAMBLE - -The goals of the Open Font License (OFL) are to stimulate worldwide -development of collaborative font projects, to support the font creation -efforts of academic and linguistic communities, and to provide a free and -open framework in which fonts may be shared and improved in partnership -with others. - -The OFL allows the licensed fonts to be used, studied, modified and -redistributed freely as long as they are not sold by themselves. The -fonts, including any derivative works, can be bundled, embedded, -redistributed and/or sold with any software provided that any reserved -names are not used by derivative works. The fonts and derivatives, -however, cannot be released under any other type of license. The -requirement for fonts to remain under this license does not apply -to any document created using the fonts or their derivatives. -DEFINITIONS - -"Font Software" refers to the set of files released by the Copyright -Holder(s) under this license and clearly marked as such. This may -include source files, build scripts and documentation. - -"Reserved Font Name" refers to any names specified as such after the -copyright statement(s). - -"Original Version" refers to the collection of Font Software components as -distributed by the Copyright Holder(s). - -"Modified Version" refers to any derivative made by adding to, deleting, -or substituting — in part or in whole — any of the components of the -Original Version, by changing formats or by porting the Font Software to a -new environment. - -"Author" refers to any designer, engineer, programmer, technical -writer or other person who contributed to the Font Software. -PERMISSION & CONDITIONS - -Permission is hereby granted, free of charge, to any person obtaining -a copy of the Font Software, to use, study, copy, merge, embed, modify, -redistribute, and sell modified and unmodified copies of the Font -Software, subject to the following conditions: - -1) Neither the Font Software nor any of its individual components, -in Original or Modified Versions, may be sold by itself. - -2) Original or Modified Versions of the Font Software may be bundled, -redistributed and/or sold with any software, provided that each copy -contains the above copyright notice and this license. These can be -included either as stand-alone text files, human-readable headers or -in the appropriate machine-readable metadata fields within text or -binary files as long as those fields can be easily viewed by the user. - -3) No Modified Version of the Font Software may use the Reserved Font -Name(s) unless explicit written permission is granted by the corresponding -Copyright Holder. This restriction only applies to the primary font name as -presented to the users. - -4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font -Software shall not be used to promote, endorse or advertise any -Modified Version, except to acknowledge the contribution(s) of the -Copyright Holder(s) and the Author(s) or with their explicit written -permission. - -5) The Font Software, modified or unmodified, in part or in whole, -must be distributed entirely under this license, and must not be -distributed under any other license. The requirement for fonts to -remain under this license does not apply to any document created -using the Font Software. -TERMINATION - -This license becomes null and void if any of the above conditions are -not met. -DISCLAIMER - -THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT -OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE -COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL -DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM -OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/licenses-binary/LICENSE.google-auth-library-credentials b/licenses-binary/LICENSE.google-auth-library-credentials deleted file mode 100644 index 12edf23c6711ffcd3bfbfbad8606fcda542c9931..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.google-auth-library-credentials +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2014, Google Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.graphlib b/licenses-binary/LICENSE.graphlib deleted file mode 100644 index e3c8f95557dbb304777cfd171821b9618231f183..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.graphlib +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (c) 2012-2014 Chris Pettitt - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.grizzled-slf4j b/licenses-binary/LICENSE.grizzled-slf4j deleted file mode 100644 index 6601c0131eeac14353f6857251a1e40fd95880e6..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.grizzled-slf4j +++ /dev/null @@ -1,11 +0,0 @@ -Copyright © 2010-2012 Brian M. Clapper. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -Neither the name “clapper.org” nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.imagesloaded b/licenses-binary/LICENSE.imagesloaded deleted file mode 100644 index 3e30b1aeff26a9130b03305fc0df8845cdff1bf7..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.imagesloaded +++ /dev/null @@ -1,7 +0,0 @@ -Copyright © 2018 David DeSandro - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses-binary/LICENSE.influx b/licenses-binary/LICENSE.influx deleted file mode 100644 index f21351ced0ec67b1f256b01f052f21aaf8ad5605..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.influx +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014-2017 Stefan Majer - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/licenses-binary/LICENSE.janino b/licenses-binary/LICENSE.janino deleted file mode 100644 index ef871e242621862a6466a90ac26c883041e277b3..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.janino +++ /dev/null @@ -1,31 +0,0 @@ -Janino - An embedded Java[TM] compiler - -Copyright (c) 2001-2016, Arno Unkrig -Copyright (c) 2015-2016 TIBCO Software Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials - provided with the distribution. - 3. Neither the name of JANINO nor the names of its contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER -IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN -IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.jline b/licenses-binary/LICENSE.jline deleted file mode 100644 index e34763968c262de5bfa4cf0e0bc33d6edaddc9be..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.jline +++ /dev/null @@ -1,7 +0,0 @@ -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.jquery b/licenses-binary/LICENSE.jquery deleted file mode 100644 index e3dbacb999cef3718b1c4cc83d9aa45f869256a6..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.jquery +++ /dev/null @@ -1,20 +0,0 @@ -Copyright JS Foundation and other contributors, https://js.foundation/ - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.jsr166y b/licenses-binary/LICENSE.jsr166y deleted file mode 100644 index b1c292b54cb2b24b1b0b9481cc59ba45bd4c7a7e..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.jsr166y +++ /dev/null @@ -1,26 +0,0 @@ -The person or persons who have associated work with this document (the -"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of -his knowledge, the work of authorship identified is in the public domain of -the country from which the work is published, or (b) hereby dedicates whatever -copyright the dedicators holds in the work of authorship identified below (the -"Work") to the public domain. A certifier, moreover, dedicates any copyright -interest he may have in the associated work, and for these purposes, is -described as a "dedicator" below. - -A certifier has taken reasonable steps to verify the copyright status of this -work. Certifier recognizes that his good faith efforts may not shield him from -liability if in fact the work certified is not in the public domain. - -Dedicator makes this dedication for the benefit of the public at large and to -the detriment of the Dedicator's heirs and successors. Dedicator intends this -dedication to be an overt act of relinquishment in perpetuity of all present -and future rights under copyright law, whether vested or contingent, in the -Work. Dedicator understands that such relinquishment of all rights includes -the relinquishment of all rights to enforce (by lawsuit or otherwise) those -copyrights in the Work. - -Dedicator recognizes that, once placed in the public domain, the Work may be -freely reproduced, distributed, transmitted, used, modified, built upon, or -otherwise exploited by anyone for any purpose, commercial or non-commercial, -and in any way, including by methods that have not yet been invented or -conceived. diff --git a/licenses-binary/LICENSE.jzlib b/licenses-binary/LICENSE.jzlib deleted file mode 100644 index 29ad562af0a5ed0b2c0171dc77ae1e7dfced01de..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.jzlib +++ /dev/null @@ -1,26 +0,0 @@ -Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the distribution. - - 3. The names of the authors may not be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT, -INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, -OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, -EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/licenses-binary/LICENSE.kryo b/licenses-binary/LICENSE.kryo deleted file mode 100644 index e1cd88478edf78dd125ece54aec34d88a5284391..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.kryo +++ /dev/null @@ -1,10 +0,0 @@ -Copyright (c) 2008, Nathan Sweet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.lodash b/licenses-binary/LICENSE.lodash deleted file mode 100644 index e3a30b4eafbab15692e1b8d2cdf6dfa93a912452..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.lodash +++ /dev/null @@ -1,10 +0,0 @@ -The MIT License - -Copyright JS Foundation and other contributors - -Based on Underscore.js, copyright Jeremy Ashkenas, -DocumentCloud and Investigative Reporters & Editors - -This software consists of voluntary contributions made by many -individuals. For exact contribution history, see the revision history -available at https://github.com/lodash/lodash \ No newline at end of file diff --git a/licenses-binary/LICENSE.minlog b/licenses-binary/LICENSE.minlog deleted file mode 100644 index e1cd88478edf78dd125ece54aec34d88a5284391..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.minlog +++ /dev/null @@ -1,10 +0,0 @@ -Copyright (c) 2008, Nathan Sweet -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name of Esoteric Software nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.moment b/licenses-binary/LICENSE.moment deleted file mode 100644 index 8618b7333d6f520594c2f7b7c3c884e766f3e782..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.moment +++ /dev/null @@ -1,22 +0,0 @@ -Copyright (c) JS Foundation and other contributors - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.moment-duration-format b/licenses-binary/LICENSE.moment-duration-format deleted file mode 100644 index 06ec6fbe07368dae0dad183ebd58cf53838b04f0..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.moment-duration-format +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2017 vin-car - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/licenses-binary/LICENSE.monaco-editor b/licenses-binary/LICENSE.monaco-editor deleted file mode 100644 index 862172d90e8fe4158d6fe0956c11d519c2b2b057..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.monaco-editor +++ /dev/null @@ -1,9 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2019 Microsoft Corporation - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.ng-zorro-antd b/licenses-binary/LICENSE.ng-zorro-antd deleted file mode 100644 index 2dae87549ebcdd14cac98b841451eb699531e71c..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.ng-zorro-antd +++ /dev/null @@ -1,22 +0,0 @@ -MIT LICENSE - -Copyright (c) 2017-present Alibaba.com - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.py4j b/licenses-binary/LICENSE.py4j deleted file mode 100644 index 0f45e3e464c1eaa0b965da6a87d79e5c9e065aac..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.py4j +++ /dev/null @@ -1,26 +0,0 @@ -Copyright (c) 2009-2018, Barthelemy Dagenais and individual contributors. All -rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -- Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -- Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -- The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.pyrolite b/licenses-binary/LICENSE.pyrolite deleted file mode 100644 index ad923a6ea4c9ca189213b242b19db2c18390b0b3..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.pyrolite +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) by Irmen de Jong - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/licenses-binary/LICENSE.qtip2 b/licenses-binary/LICENSE.qtip2 deleted file mode 100644 index 0cfb790dc91d39381c53b35a3142299359a69faa..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.qtip2 +++ /dev/null @@ -1,22 +0,0 @@ -Copyright (c) 2012 Craig Michael Thompson - -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without -restriction, including without limitation the rights to use, -copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.rxjs b/licenses-binary/LICENSE.rxjs deleted file mode 100644 index 65afc31116868019ec7c8dbe9b5d50f40161312e..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.rxjs +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright (c) 2015-2018 Google, Inc., Netflix, Inc., Microsoft Corp. and contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/licenses-binary/LICENSE.scala b/licenses-binary/LICENSE.scala deleted file mode 100644 index 6d8bdabbbe0fbf2b0ca84be143852e353965668a..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.scala +++ /dev/null @@ -1,11 +0,0 @@ -Copyright (c) 2002- EPFL -Copyright (c) 2011- Lightbend, Inc. - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -Neither the name of the EPFL nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses-binary/LICENSE.scopt b/licenses-binary/LICENSE.scopt deleted file mode 100644 index 6d6a875878f4b59bcde147f7f2e170489dcd58e0..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.scopt +++ /dev/null @@ -1,21 +0,0 @@ -This project is licensed under the [MIT license](https://en.wikipedia.org/wiki/MIT_License). - -Copyright (c) scopt contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.slf4j b/licenses-binary/LICENSE.slf4j deleted file mode 100644 index 93119e70ed5e73202ed8e9ef4035439cf1d3efc7..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.slf4j +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2017 QOS.ch - All rights reserved. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.slf4j-api b/licenses-binary/LICENSE.slf4j-api deleted file mode 100644 index 93119e70ed5e73202ed8e9ef4035439cf1d3efc7..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.slf4j-api +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (c) 2004-2017 QOS.ch - All rights reserved. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.tinycolor2 b/licenses-binary/LICENSE.tinycolor2 deleted file mode 100644 index 38debb194c6ded22bc145f377185a94f5459e7a8..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.tinycolor2 +++ /dev/null @@ -1,21 +0,0 @@ -MIT -Copyright (c), Brian Grinstead, http://briangrinstead.com - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses-binary/LICENSE.tslib b/licenses-binary/LICENSE.tslib deleted file mode 100644 index dfcd5509eddce60b708f68e3855cd705a70c9c6b..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.tslib +++ /dev/null @@ -1,57 +0,0 @@ -Apache-2.0 -Apache License - -Version 2.0, January 2004 - -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. - -"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. - -2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. - -3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of this License; and - -You must cause any modified files to carry prominent notices stating that You changed the files; and - -You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and - -If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. - -5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. - -6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. - -8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - diff --git a/licenses-binary/LICENSE.webbit b/licenses-binary/LICENSE.webbit deleted file mode 100644 index ec5f348998bd444aa75e05606afa5f75b16d922a..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.webbit +++ /dev/null @@ -1,38 +0,0 @@ -(BSD License: http://www.opensource.org/licenses/bsd-license) - -Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or -without modification, are permitted provided that the -following conditions are met: - -* Redistributions of source code must retain the above - copyright notice, this list of conditions and the - following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the - following disclaimer in the documentation and/or other - materials provided with the distribution. - -* Neither the name of the Webbit nor the names of - its contributors may be used to endorse or promote products - derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE -GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR -BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - diff --git a/licenses-binary/LICENSE.zone b/licenses-binary/LICENSE.zone deleted file mode 100644 index c5de4726c6fbe3bbaff2529117fc6be7389be766..0000000000000000000000000000000000000000 --- a/licenses-binary/LICENSE.zone +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License - -Copyright (c) 2016-2018 Google, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/licenses-binary/LICENSE.protobuf b/licenses/LICENSE.cloudpickle similarity index 30% rename from licenses-binary/LICENSE.protobuf rename to licenses/LICENSE.cloudpickle index 19b305b00060a774a9180fb916c14b49edb2008f..d112c4806aa90b4c911b360fa0ce1d0ee7c031ce 100644 --- a/licenses-binary/LICENSE.protobuf +++ b/licenses/LICENSE.cloudpickle @@ -1,32 +1,32 @@ -Copyright 2008 Google Inc. All rights reserved. +This module was extracted from the `cloud` package, developed by +PiCloud, Inc. -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +Copyright (c) 2015, Cloudpickle contributors. +Copyright (c) 2012, Regents of the University of California. +Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the University of California, Berkeley nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pom.xml b/pom.xml index 5a8b5983917b0bc6c9b6dd79000be73f517df270..4b49170c0ff9247600f4da125358c21c765af1fa 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ under the License. Starting Hadoop 3, org.apache.kerby will be used instead of MiniKDC. We may have to revisit the impact at that time. --> - 2.7.2 + 3.2.0 ./docs/_includes/generated 2.3.4 Apache Flink + UTF-8 @@ -1685,7 +1686,7 @@ under the License. org.apache.maven.plugins maven-shade-plugin - 3.2.1 + 3.1.1 diff --git a/tools/releasing/collect_license_files.sh b/tools/releasing/collect_license_files.sh index 5aa7585410f73d3ce941692f127aa331790380fa..754b5e8043271192b5d905de33bd333f3bfc290a 100755 --- a/tools/releasing/collect_license_files.sh +++ b/tools/releasing/collect_license_files.sh @@ -49,15 +49,14 @@ do (cd "${DIR}" && jar xf ${JAR} META-INF/NOTICE META-INF/licenses) done -NOTICE="${DST}/NOTICE-binary" +NOTICE="${DST}/NOTICE" [ -f "${NOTICE}" ] && rm "${NOTICE}" cp "${NOTICE_BINARY_PREAMBLE}" "${NOTICE}" (export LC_ALL=C; find "${TMP}" -name "NOTICE" | sort | xargs cat >> "${NOTICE}") -LICENSES="${DST}/licenses-binary" +LICENSES="${DST}/licenses" [ -f "${LICENSES}" ] && rm -r "${LICENSES}" find "${TMP}" -name "licenses" -type d -exec cp -r -- "{}" "${DST}" \; -mv "${DST}/licenses" "${LICENSES}" cp "${SLF4J_LICENSE}" "${LICENSES}" rm -r "${TMP}" diff --git a/tools/releasing/create_binary_release.sh b/tools/releasing/create_binary_release.sh index 60a3cca85d865fcef86a886a1ca5332094c31d5b..6630106638a94905d012c3cda576e847d5552aaf 100755 --- a/tools/releasing/create_binary_release.sh +++ b/tools/releasing/create_binary_release.sh @@ -74,7 +74,8 @@ make_binary_release() { # enable release profile here (to check for the maven version) $MVN clean package $FLAGS -Prelease -pl flink-dist -am -Dgpg.skip -Dcheckstyle.skip=true -DskipTests - cd flink-dist/target/flink-*-bin/ + cd flink-dist/target/flink-${RELEASE_VERSION}-bin + ${FLINK_DIR}/tools/releasing/collect_license_files.sh ./flink-${RELEASE_VERSION} ./flink-${RELEASE_VERSION} tar czf "${dir_name}.tgz" flink-* cp flink-*.tgz ${RELEASE_DIR} diff --git a/tools/travis/splits/split_misc.sh b/tools/travis/splits/split_misc.sh index 97f811d18616a580a246a985b52030498144d59d..834c8638503be7a872a554e4fc317906d3ce8b4a 100755 --- a/tools/travis/splits/split_misc.sh +++ b/tools/travis/splits/split_misc.sh @@ -57,14 +57,17 @@ run_test "Streaming File Sink end-to-end test" "$END_TO_END_DIR/test-scripts/tes run_test "Streaming File Sink s3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_file_sink.sh s3" "skip_check_exceptions" run_test "Stateful stream job upgrade end-to-end test" "$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4" -run_test "Elasticsearch (v2.3.5) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 2 https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.5/elasticsearch-2.3.5.tar.gz" +if [[ ${PROFILE} != *"jdk11"* ]]; then + run_test "Elasticsearch (v2.3.5) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 2 https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.5/elasticsearch-2.3.5.tar.gz" +fi run_test "Elasticsearch (v5.1.2) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 5 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.1.2.tar.gz" run_test "Elasticsearch (v6.3.1) sink end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_elasticsearch.sh 6 https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.tar.gz" run_test "Quickstarts Java nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_quickstarts.sh java" run_test "Quickstarts Scala nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_quickstarts.sh scala" -run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_confluent_schema_registry.sh" +# Disabled until FLINK-13567 has been fixed +#run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_confluent_schema_registry.sh" run_test "State TTL Heap backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh file" run_test "State TTL RocksDb backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh rocks" diff --git a/tools/travis/splits/split_misc_hadoopfree.sh b/tools/travis/splits/split_misc_hadoopfree.sh index 193ad0b13a83e5d02bd7b2931a6ceda0837b6058..e4a36a8f969fcb75b07a5733f89d7b5a0e351afb 100755 --- a/tools/travis/splits/split_misc_hadoopfree.sh +++ b/tools/travis/splits/split_misc_hadoopfree.sh @@ -63,7 +63,8 @@ run_test "Elasticsearch (v6.3.1) sink end-to-end test" "$END_TO_END_DIR/test-scr run_test "Quickstarts Java nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_quickstarts.sh java" run_test "Quickstarts Scala nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_quickstarts.sh scala" -run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_confluent_schema_registry.sh" +# Disabled until FLINK-13567 has been fixed +#run_test "Avro Confluent Schema Registry nightly end-to-end test" "$END_TO_END_DIR/test-scripts/test_confluent_schema_registry.sh" run_test "State TTL Heap backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh file" run_test "State TTL RocksDb backend end-to-end test" "$END_TO_END_DIR/test-scripts/test_stream_state_ttl.sh rocks" diff --git a/tools/travis/stage.sh b/tools/travis/stage.sh index 6676b46e8ef71a8948d10d6a445cd7f017afd2ad..b0f4456d324cdf46e819c614e662bc5ec379d59e 100644 --- a/tools/travis/stage.sh +++ b/tools/travis/stage.sh @@ -89,7 +89,9 @@ flink-connectors/flink-connector-cassandra,\ flink-connectors/flink-connector-elasticsearch2,\ flink-connectors/flink-connector-elasticsearch5,\ flink-connectors/flink-connector-elasticsearch6,\ +flink-connectors/flink-connector-elasticsearch7,\ flink-connectors/flink-sql-connector-elasticsearch6,\ +flink-connectors/flink-sql-connector-elasticsearch7,\ flink-connectors/flink-connector-elasticsearch-base,\ flink-connectors/flink-connector-filesystem,\ flink-connectors/flink-connector-kafka-0.9,\ diff --git a/tools/travis_controller.sh b/tools/travis_controller.sh index 91392f658d40bc44a5bd940e4e938830f4e055a6..06d42ff66774f032dd09ed8b70bbc6606b2cda71 100755 --- a/tools/travis_controller.sh +++ b/tools/travis_controller.sh @@ -126,39 +126,6 @@ if [ $STAGE == "$STAGE_COMPILE" ]; then echo "==============================================================================" fi - if [ $EXIT_CODE == 0 ]; then - if [[ $PROFILE == *"scala-2.11"* ]]; then - ./tools/releasing/collect_license_files.sh ./build-target - diff "NOTICE-binary" "licenses-output/NOTICE-binary" - EXIT_CODE=$(($EXIT_CODE+$?)) - diff -r "licenses-binary" "licenses-output/licenses-binary" - EXIT_CODE=$(($EXIT_CODE+$?)) - - if [ $EXIT_CODE != 0 ]; then - echo "==============================================================================" - echo "ERROR: binary licensing is out-of-date." - echo "Please update NOTICE-binary and licenses-binary:" - echo "Step 1: Rebuild flink" - echo "Step 2: Run 'tools/releasing/collect_license_files.sh build-target'" - echo " This extracts all the licensing files from the distribution, and puts them in 'licenses-output'." - echo " If the build-target symlink does not exist after building flink, point the tool to 'flink-dist/target/flink--bin/flink-' instead." - echo "Step 3: Replace existing licensing" - echo " Delete NOTICE-binary and the entire licenses-binary directory." - echo " Copy the contents in 'licenses-output' into the root directory of the Flink project." - echo "Step 4: Remember to commit the changes!" - echo "==============================================================================" - fi - else - echo "==============================================================================" - echo "Ignoring the license file check because built uses different Scala version than 2.11. See FLINK-14008." - echo "==============================================================================" - fi - else - echo "==============================================================================" - echo "Previous build failure detected, skipping licensing check." - echo "==============================================================================" - fi - if [ $EXIT_CODE == 0 ]; then echo "Creating cache build directory $CACHE_FLINK_DIR" mkdir -p "$CACHE_FLINK_DIR"