diff --git a/0002-Upgrade-os-maven-plugin-to-1.7.1.patch b/0002-Upgrade-os-maven-plugin-to-1.7.1.patch new file mode 100644 index 0000000000000000000000000000000000000000..cad50c53e116d2e281e938eea880debddb1d00e2 --- /dev/null +++ b/0002-Upgrade-os-maven-plugin-to-1.7.1.patch @@ -0,0 +1,22 @@ +From 958468fc5154399f1a256097502b21a8f256b2f3 Mon Sep 17 00:00:00 2001 +From: Dingli Zhang +Date: Mon, 1 Jul 2024 05:21:50 +0000 +Subject: [PATCH] Upgrade os-maven-plugin to 1.7.1 + + +diff --git a/connector/connect/common/pom.xml b/connector/connect/common/pom.xml +index c78c5445..57515a8f 100644 +--- a/connector/connect/common/pom.xml ++++ b/connector/connect/common/pom.xml +@@ -116,7 +116,7 @@ + + kr.motd.maven + os-maven-plugin +- 1.6.2 ++ 1.7.1 + + + target/scala-${scala.binary.version}/classes +-- +2.41.0 + diff --git a/1000-Added-support-for-building-the-riscv64-protoc-binari.patch b/1000-Added-support-for-building-the-riscv64-protoc-binari.patch new file mode 100644 index 0000000000000000000000000000000000000000..03dee7f9fa28265a5a8c2a64f3342b7186ec90e1 --- /dev/null +++ b/1000-Added-support-for-building-the-riscv64-protoc-binari.patch @@ -0,0 +1,56 @@ +From 8b626946fcabc659a41de3b15cbe6f5d45bd596e Mon Sep 17 00:00:00 2001 +From: Dingli Zhang +Date: Tue, 7 May 2024 13:56:04 +0800 +Subject: [PATCH] Added support for building the riscv64 protoc binaries with + Kokoro + + +diff --git a/kokoro/release/protoc/linux/build.sh b/kokoro/release/protoc/linux/build.sh +index 44a95aab3..13be7b2ed 100755 +--- a/kokoro/release/protoc/linux/build.sh ++++ b/kokoro/release/protoc/linux/build.sh +@@ -22,6 +22,9 @@ protoc-artifacts/build-protoc.sh linux aarch_64 protoc + sudo apt install -y g++-powerpc64le-linux-gnu + protoc-artifacts/build-protoc.sh linux ppcle_64 protoc + ++sudo apt install -y g++-riscv64-linux-gnu ++protoc-artifacts/build-protoc.sh linux riscv64 protoc ++ + sudo apt install -y g++-s390x-linux-gnu + protoc-artifacts/build-protoc.sh linux s390_64 protoc + +diff --git a/protoc-artifacts/build-protoc.sh b/protoc-artifacts/build-protoc.sh +index 460284262..20ca05739 100755 +--- a/protoc-artifacts/build-protoc.sh ++++ b/protoc-artifacts/build-protoc.sh +@@ -93,6 +93,8 @@ checkArch () + assertEq $format "elf64-x86-64" $LINENO + elif [[ "$ARCH" == aarch_64 ]]; then + assertEq $format "elf64-little" $LINENO ++ elif [[ "$ARCH" == riscv64 ]]; then ++ assertEq $format "elf64-littleriscv" $LINENO + elif [[ "$ARCH" == s390_64 ]]; then + if [[ $host_machine == s390x ]];then + assertEq $format "elf64-s390" $LINENO +@@ -162,6 +164,9 @@ checkDependencies () + elif [[ "$ARCH" == aarch_64 ]]; then + dump_cmd='objdump -p '"$1"' | grep NEEDED' + white_list="libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-aarch64\.so\.1" ++ elif [[ "$ARCH" == riscv64 ]]; then ++ dump_cmd='objdump -p '"$1"' | grep NEEDED' ++ white_list="libz\.so\.1\|libm\.so\.6\|libc\.so\.6\|ld-linux-riscv64-lp64d\.so\.1" + fi + elif [[ "$OS" == osx ]]; then + dump_cmd='otool -L '"$1"' | fgrep dylib' +@@ -226,6 +231,8 @@ elif [[ "$(uname)" == Linux* ]]; then + elif [[ "$ARCH" == ppcle_64 ]]; then + CXXFLAGS="$CXXFLAGS -m64" + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=powerpc64le-linux-gnu" ++ elif [[ "$ARCH" == riscv64 ]]; then ++ CONFIGURE_ARGS="$CONFIGURE_ARGS --host=riscv64-linux-gnu" + elif [[ "$ARCH" == s390_64 ]]; then + CXXFLAGS="$CXXFLAGS -m64" + CONFIGURE_ARGS="$CONFIGURE_ARGS --host=s390x-linux-gnu" +-- +2.41.0 + diff --git a/1001-Add-protoc-java-support-for-riscv64.patch b/1001-Add-protoc-java-support-for-riscv64.patch new file mode 100644 index 0000000000000000000000000000000000000000..3628aad37bbc6a8434cc4f46e2c5578f72f44800 --- /dev/null +++ b/1001-Add-protoc-java-support-for-riscv64.patch @@ -0,0 +1,23 @@ +From 2c52801962444baa444401b8fafd09bd6cfab92b Mon Sep 17 00:00:00 2001 +From: Dingli Zhang +Date: Thu, 9 May 2024 00:02:35 +0800 +Subject: [PATCH] Add support for riscv64 + +--- + src/main/java/com/github/os72/protocjar/PlatformDetector.java | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/main/java/com/github/os72/protocjar/PlatformDetector.java b/src/main/java/com/github/os72/protocjar/PlatformDetector.java +index 7a99348..b8f08e5 100644 +--- a/src/main/java/com/github/os72/protocjar/PlatformDetector.java ++++ b/src/main/java/com/github/os72/protocjar/PlatformDetector.java +@@ -211,6 +211,9 @@ private static String normalizeArch(String value) { + if ("ppc64le".equals(value)) { + return "ppcle_64"; + } ++ if ("riscv64".equals(value)) { ++ return "riscv64"; ++ } + if ("s390".equals(value)) { + return "s390_32"; + } diff --git a/1002-Added-support-for-building-the-riscv64-protoc-gen-gr.patch b/1002-Added-support-for-building-the-riscv64-protoc-gen-gr.patch new file mode 100644 index 0000000000000000000000000000000000000000..e399fa163b96ad2f189cf3e9e31b14693cf0c061 --- /dev/null +++ b/1002-Added-support-for-building-the-riscv64-protoc-gen-gr.patch @@ -0,0 +1,147 @@ +From 11c8f77c101385119e34d0961e4203c6eb27c52d Mon Sep 17 00:00:00 2001 +From: Dingli Zhang +Date: Fri, 5 Jul 2024 03:48:20 +0000 +Subject: [PATCH] Added support for building the riscv64 protoc-gen-grpc-java + binaries with Kokoro + + +diff --git a/build.gradle b/build.gradle +index ed4baab82..c5bed1026 100644 +--- a/build.gradle ++++ b/build.gradle +@@ -23,6 +23,8 @@ subprojects { + version = "1.56.0" // CURRENT_GRPC_VERSION + + repositories { ++ maven { url 'file://@HOME@/.m2/repository'} ++ maven { url "https://mirrors.huaweicloud.com/repository/maven/"} + maven { // The google mirror is less flaky than mavenCentral() + url "https://maven-central.storage-download.googleapis.com/maven2/" } + mavenCentral() +diff --git a/buildscripts/kokoro/linux_artifacts.sh b/buildscripts/kokoro/linux_artifacts.sh +index c80d3436f..49a15a457 100755 +--- a/buildscripts/kokoro/linux_artifacts.sh ++++ b/buildscripts/kokoro/linux_artifacts.sh +@@ -56,6 +56,8 @@ cp -r "$LOCAL_MVN_TEMP"/* "$MVN_ARTIFACT_DIR"/ + SKIP_TESTS=true ARCH=aarch_64 /grpc-java/buildscripts/kokoro/unix.sh + "$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-multiarch env \ + SKIP_TESTS=true ARCH=ppcle_64 /grpc-java/buildscripts/kokoro/unix.sh ++"$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-multiarch env \ ++ SKIP_TESTS=true ARCH=risv64 /grpc-java/buildscripts/kokoro/unix.sh + # Use a newer GCC version. GCC 7 in multiarch has a bug: + # internal compiler error: output_operand: invalid %-code + "$GRPC_JAVA_DIR"/buildscripts/run_in_docker.sh grpc-java-artifacts-ubuntu2004 env \ +diff --git a/buildscripts/kokoro/unix.sh b/buildscripts/kokoro/unix.sh +index 1e6af6cfc..048aef9e8 100755 +--- a/buildscripts/kokoro/unix.sh ++++ b/buildscripts/kokoro/unix.sh +@@ -48,7 +48,7 @@ GRADLE_FLAGS+=" -Pcheckstyle.ignoreFailures=false" + GRADLE_FLAGS+=" -PfailOnWarnings=true" + GRADLE_FLAGS+=" -PerrorProne=true" + GRADLE_FLAGS+=" -PskipAndroid=true" +-GRADLE_FLAGS+=" -Dorg.gradle.parallel=true" ++GRADLE_FLAGS+=" -Dorg.gradle.parallel=false" + export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" + + # Make protobuf discoverable by :grpc-compiler +@@ -90,7 +90,7 @@ fi + LOCAL_MVN_TEMP=$(mktemp -d) + # Note that this disables parallel=true from GRADLE_FLAGS + if [[ -z "${ALL_ARTIFACTS:-}" ]]; then +- if [[ "$ARCH" = "aarch_64" || "$ARCH" = "ppcle_64" || "$ARCH" = "s390_64" ]]; then ++ if [[ $ARCH == "aarch_64" || $ARCH == "riscv64" ]]; then + GRADLE_FLAGS+=" -x grpc-compiler:generateTestProto -x grpc-compiler:generateTestLiteProto" + GRADLE_FLAGS+=" -x grpc-compiler:testGolden -x grpc-compiler:testLiteGolden" + GRADLE_FLAGS+=" -x grpc-compiler:testDeprecatedGolden -x grpc-compiler:testDeprecatedLiteGolden" +diff --git a/buildscripts/kokoro/upload_artifacts.sh b/buildscripts/kokoro/upload_artifacts.sh +index 39e27eff5..bc47d4b50 100644 +--- a/buildscripts/kokoro/upload_artifacts.sh ++++ b/buildscripts/kokoro/upload_artifacts.sh +@@ -34,6 +34,9 @@ LOCAL_OTHER_ARTIFACTS="$KOKORO_GFILE_DIR"/github/grpc-java/artifacts/ + # for linux aarch64 platform + [[ "$(find "$LOCAL_MVN_ARTIFACTS" -type f -iname 'protoc-gen-grpc-java-*-linux-aarch_64.exe' | wc -l)" != '0' ]] + ++# for linux riscv64 platform ++[[ "$(find "$LOCAL_MVN_ARTIFACTS" -type f -iname 'protoc-gen-grpc-java-*-linux-riscv64.exe' | wc -l)" != '0' ]] ++ + # for linux ppc64le platform + [[ "$(find "$LOCAL_MVN_ARTIFACTS" -type f -iname 'protoc-gen-grpc-java-*-linux-ppcle_64.exe' | wc -l)" != '0' ]] + +diff --git a/buildscripts/make_dependencies.sh b/buildscripts/make_dependencies.sh +index 3d02a72f4..0427d47e1 100755 +--- a/buildscripts/make_dependencies.sh ++++ b/buildscripts/make_dependencies.sh +@@ -7,8 +7,8 @@ PROTOBUF_VERSION=21.7 + + # ARCH is x86_64 bit unless otherwise specified. + ARCH="${ARCH:-x86_64}" +-DOWNLOAD_DIR=/tmp/source +-INSTALL_DIR="/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH" ++DOWNLOAD_DIR=/var/tmp/source ++INSTALL_DIR="/var/tmp/protobuf-cache/$PROTOBUF_VERSION/$(uname -s)-$ARCH" + mkdir -p $DOWNLOAD_DIR + + # Start with a sane default +@@ -36,6 +36,8 @@ else + --prefix="$INSTALL_DIR" + elif [[ "$ARCH" == aarch* ]]; then + ./configure --disable-shared --host=aarch64-linux-gnu --prefix="$INSTALL_DIR" ++ elif [[ "$ARCH" == riscv* ]]; then ++ ./configure --disable-shared --host=riscv64-linux-gnu --prefix="$INSTALL_DIR" + elif [[ "$ARCH" == ppc* ]]; then + ./configure --disable-shared --host=powerpc64le-linux-gnu --prefix="$INSTALL_DIR" + elif [[ "$ARCH" == s390* ]]; then +diff --git a/compiler/build.gradle b/compiler/build.gradle +index 12798208c..7960642cd 100644 +--- a/compiler/build.gradle ++++ b/compiler/build.gradle +@@ -58,6 +58,7 @@ model { + cppCompiler.executable = 'aarch64-linux-gnu-g++' + linker.executable = 'aarch64-linux-gnu-g++' + } ++ target("riscv64") + target("s390_64") { + cppCompiler.executable = 's390x-linux-gnu-g++' + linker.executable = 's390x-linux-gnu-g++' +@@ -73,6 +74,7 @@ model { + x86_64 { architecture "x86_64" } + ppcle_64 { architecture "ppcle_64" } + aarch_64 { architecture "aarch_64" } ++ riscv64 { architecture "riscv64" } + s390_64 { architecture "s390_64" } + loongarch_64 { architecture "loongarch_64" } + } +@@ -84,6 +86,7 @@ model { + 'x86_64', + 'ppcle_64', + 'aarch_64', ++ 'riscv64', + 's390_64', + 'loongarch_64' + ]) { +diff --git a/compiler/check-artifact.sh b/compiler/check-artifact.sh +index a5f33a35b..ebe9dc4a1 100755 +--- a/compiler/check-artifact.sh ++++ b/compiler/check-artifact.sh +@@ -61,6 +61,8 @@ checkArch () + assertEq "$format" "elf64-x86-64" $LINENO + elif [[ "$ARCH" == aarch_64 ]]; then + assertEq "$format" "elf64-little" $LINENO ++ elif [[ "$ARCH" == riscv64 ]]; then ++ assertEq $format "elf64-littleriscv" $LINENO + elif [[ "$ARCH" == loongarch_64 ]]; then + echo $format + assertEq "$format" "elf64-loongarch" $LINENO +@@ -119,6 +121,9 @@ checkDependencies () + elif [[ "$ARCH" == aarch_64 ]]; then + dump_cmd='aarch64-linux-gnu-objdump -x '"$1"' |grep "NEEDED"' + white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld-linux-aarch64\.so\.1" ++ elif [[ "$ARCH" == riscv64 ]]; then ++ dump_cmd='objdump -p '"$1"' | grep NEEDED' ++ white_list="libm\.so\.6\|libc\.so\.6\|ld-linux-riscv64-lp64d\.so\.1" + elif [[ "$ARCH" == loongarch_64 ]]; then + dump_cmd='objdump -x '"$1"' | grep NEEDED' + white_list="linux-vdso\.so\.1\|libpthread\.so\.0\|libm\.so\.6\|libc\.so\.6\|ld\.so\.1" +-- +2.41.0 + diff --git a/gradle-7.6-bin.zip b/gradle-7.6-bin.zip new file mode 100644 index 0000000000000000000000000000000000000000..d2f675a26739180a39753464bf0d48249e7b8bce Binary files /dev/null and b/gradle-7.6-bin.zip differ diff --git a/protobuf-all-21.7.tar.gz b/protobuf-all-21.7.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..aeab399f8cc4c44da94956e126597d1c9e3d4761 Binary files /dev/null and b/protobuf-all-21.7.tar.gz differ diff --git a/spark.spec b/spark.spec index ef4e10c05dea881b5a0625ac532234870165dbd0..048cbee8ca9504cf3c6b1b3367c2812f2e4747c3 100644 --- a/spark.spec +++ b/spark.spec @@ -4,28 +4,88 @@ Summary: A unified analytics engine for large-scale data processing. Name: spark Version: 3.5.0 -Release: 2 +Release: 3 License: Apache 2.0 URL: http://spark.apache.org/ Source0: https://github.com/apache/spark/archive/v%{version}.tar.gz - +Source1: https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.21.7.tar.gz +Source2: https://github.com/os72/protoc-jar/archive/refs/tags/v3.11.4.tar.gz +Source3: https://github.com/os72/protoc-jar-maven-plugin/archive/refs/tags/v3.11.4.zip +Source4: https://services.gradle.org/distributions/gradle-7.6-bin.zip +Source5: https://github.com/google/protobuf/releases/download/v21.7/protobuf-all-21.7.tar.gz +Source6: https://github.com/grpc/grpc-java/archive/refs/tags/v1.56.0.tar.gz Patch0001: 0001-change-mvn-scalafmt.patch - +Patch0002: 0002-Upgrade-os-maven-plugin-to-1.7.1.patch + +%ifarch riscv64 +BuildRequires: protobuf-devel protobuf-compiler +BuildRequires: autoconf automake libtool pkgconfig zlib-devel libstdc++-static gcc-c++ +Patch1000: 1000-Added-support-for-building-the-riscv64-protoc-binari.patch +Patch1001: 1001-Add-protoc-java-support-for-riscv64.patch +Patch1002: 1002-Added-support-for-building-the-riscv64-protoc-gen-gr.patch +%endif BuildRequires: java-1.8.0-openjdk-devel git maven Requires: java-1.8.0-openjdk -ExclusiveArch: x86_64 aarch64 ppc64le +ExclusiveArch: x86_64 aarch64 ppc64le riscv64 %description Apache Spark achieves high performance for both batch and streaming data, using a state-of-the-art DAG scheduler, a query optimizer, and a physical execution engine. %prep +%ifarch riscv64 +mkdir -p ${HOME}/%{name}-prep_dir +export PROTOC_VERSION="3."$(protoc --version | awk '{print $NF}') +mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=${PROTOC_VERSION} -Dclassifier=linux-riscv64 -Dpackaging=exe -Dfile=/usr/bin/protoc +# protoc +tar -mxf %{SOURCE1} -C ${HOME}/%{name}-prep_dir +pushd ${HOME}/%{name}-prep_dir/protobuf-3.21.7 +%patch1000 -p1 +./autogen.sh +./protoc-artifacts/build-protoc.sh linux riscv64 protoc +mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc -Dversion=3.21.7 -Dclassifier=linux-riscv64 -Dpackaging=exe -Dfile=protoc-artifacts/target/linux/riscv64/protoc.exe +popd +# protoc-jar +tar -mxf %{SOURCE2} -C ${HOME}/%{name}-prep_dir +pushd ${HOME}/%{name}-prep_dir/protoc-jar-3.11.4 +%patch1001 -p1 +mvn clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true +popd +# protoc-jar-maven-plugin +unzip %{SOURCE3} -d ${HOME}/%{name}-prep_dir +pushd ${HOME}/%{name}-prep_dir/protoc-jar-maven-plugin-3.11.4 +mvn clean install -Dmaven.test.skip=true -Dmaven.javadoc.skip=true +popd +# prepare gradle and protobuf for build protoc-gen-grpc-java +mkdir -p %{_tmppath}/source +cp %{SOURCE4} %{_tmppath}/source +tar xzf %{SOURCE5} -C %{_tmppath}/source +# protoc-gen-grpc-java +tar -mxf %{SOURCE6} -C ${HOME}/%{name}-prep_dir +pushd ${HOME}/%{name}-prep_dir/grpc-java-1.56.0 +%patch1002 -p1 +sed -i "s,@HOME@,${HOME},g" build.gradle +sed -i 's|https\\://services.gradle.org/distributions|file://%{_tmppath}/source|g' gradle/wrapper/gradle-wrapper.properties +SKIP_TESTS=true ARCH=riscv64 ./buildscripts/kokoro/unix.sh +mvn install:install-file -DgroupId=io.grpc -DartifactId=protoc-gen-grpc-java -Dversion=1.56.0 -Dclassifier=linux-riscv64 -Dpackaging=exe -Dfile=mvn-artifacts/io/grpc/protoc-gen-grpc-java/1.56.0/protoc-gen-grpc-java-1.56.0-linux-riscv64.exe +popd +%endif + %setup -q %patch0001 -p1 +%patch0002 -p1 + +%ifarch riscv64 +sed -i -e 's/protoVersion = "3.23.4/protoVersion = "'${PROTOC_VERSION}/'' project/SparkBuild.scala +sed -i -e 's/3.23.4/'${PROTOC_VERSION}/'' pom.xml +%endif %build +%ifarch riscv64 +export MAVEN_OPTS="-Xms2048M -Xmx8000M" +%endif ./dev/make-distribution.sh --mvn mvn --name spark-3.5.0 --tgz -DskipTests -Dmaven.test.skip=true %install @@ -37,6 +97,11 @@ cp -rf ../%{name}-%{version} %{buildroot}/opt/apache-%{name}-%{version} %changelog +* Mon Jul 1 2024 Dingli Zhang - 3.5.0-3 +- Add riscv64 to ExclusiveArch +- Fix build on riscv64 +- Upgrade os-maven-plugin to 1.7.1 + * Tue Mar 12 2024 peng.zou - 3.5.0-2 - add ppc64le support diff --git a/v1.56.0.tar.gz b/v1.56.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..6f8ab8d1d0765822631b9acfc00a19525b000ea2 Binary files /dev/null and b/v1.56.0.tar.gz differ diff --git a/v3.11.4.tar.gz b/v3.11.4.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..dd17e9631e22a916d9beff8d6e9085cb125984de Binary files /dev/null and b/v3.11.4.tar.gz differ diff --git a/v3.11.4.zip b/v3.11.4.zip new file mode 100644 index 0000000000000000000000000000000000000000..5d16b9c837a61a11c226d2f79abd1e1ca808d3cc Binary files /dev/null and b/v3.11.4.zip differ diff --git a/v3.21.7.tar.gz b/v3.21.7.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..0b2706e9a1d7db96d7579166d4ed1e2464c908c3 Binary files /dev/null and b/v3.21.7.tar.gz differ