Skip to content

Commit c89ad78

Browse files
czentgrczentgr
committed
[native] Test standard runner prestocpp build
1 parent ed176b9 commit c89ad78

File tree

7 files changed

+414
-0
lines changed

7 files changed

+414
-0
lines changed

‎.github/workflows/format-check.yml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: format-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
format-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check formatting
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make format-check

‎.github/workflows/header-check.yml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: header-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
header-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check license headers
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make header-check

‎.github/workflows/linux-adapters-build.yml

+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: linux-adapters-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/scripts/**'
8+
9+
jobs:
10+
linux-build-adapters:
11+
runs-on: ubuntu-22.04
12+
container:
13+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Update submodules
21+
run: |
22+
cd presto-native-execution
23+
make submodules
24+
- name: Build all adapter dependencies
25+
run: |
26+
mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/install
27+
source /opt/rh/gcc-toolset-12/enable
28+
set -xu
29+
cd presto-native-execution
30+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh
31+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh

‎.github/workflows/linux-build-and-unit-test.yml

+239
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
2+
name: linux-build-and-unit-test
3+
4+
on:
5+
workflow_dispatch:
6+
pull_request:
7+
8+
jobs:
9+
prestocpp-linux-build-and-unit-test:
10+
runs-on: ubuntu-22.04
11+
container:
12+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
13+
env:
14+
CCACHE_DIR: "${{ github.workspace }}/ccache"
15+
steps:
16+
- uses: actions/checkout@v4
17+
- name: Fix git permissions
18+
# Usually actions/checkout does this but as we run in a container
19+
# it doesn't work
20+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
21+
22+
- name: Update velox
23+
run: |
24+
cd presto-native-execution
25+
make velox-submodule
26+
27+
- name: Install prestocpp adapter dependencies
28+
run: |
29+
mkdir -p ${HOME}/adapter-deps/install
30+
source /opt/rh/gcc-toolset-12/enable
31+
set -xu
32+
cd presto-native-execution
33+
DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
34+
35+
- name: Install github CLI for the ccache stash
36+
run: ./presto-native-execution/scripts/setup-centos.sh install_gh
37+
38+
- name: Calculate merge-base date for ccache
39+
run: |
40+
git fetch origin master
41+
git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/master HEAD) | tee merge-base-date
42+
43+
- uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
44+
with:
45+
path: '${{ env.CCACHE_DIR }}'
46+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
47+
48+
- name: Zero ccache statistics
49+
run: ccache -sz
50+
51+
- name: Build engine
52+
run: |
53+
source /opt/rh/gcc-toolset-12/enable
54+
cd presto-native-execution
55+
cmake \
56+
-B _build/release \
57+
-GNinja \
58+
-DTREAT_WARNINGS_AS_ERRORS=1 \
59+
-DENABLE_ALL_WARNINGS=1 \
60+
-DCMAKE_BUILD_TYPE=Release \
61+
-DPRESTO_ENABLE_PARQUET=ON \
62+
-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
63+
-DPRESTO_ENABLE_JWT=ON \
64+
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
65+
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
66+
-DCMAKE_PREFIX_PATH=/usr/local \
67+
-DThrift_ROOT=/usr/local \
68+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
69+
-DMAX_LINK_JOBS=2
70+
ninja -C _build/release -j 4
71+
72+
- name: Ccache after
73+
run: ccache -s
74+
75+
- uses: apache/infrastructure-actions/stash/save@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
76+
with:
77+
path: '${{ env.CCACHE_DIR }}'
78+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
79+
80+
- name: Run Unit Tests
81+
run: |
82+
# Ensure transitive dependency libboost-iostreams is found.
83+
ldconfig /usr/local/lib
84+
cd presto-native-execution/_build/release
85+
ctest -j 4 -VV --output-on-failure --exclude-regex velox.*
86+
87+
- name: Upload artifacts
88+
uses: actions/upload-artifact@v4
89+
with:
90+
name: presto-native-build
91+
path: |
92+
presto-native-execution/_build/release/presto_cpp/main/presto_server
93+
presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
94+
95+
linux-presto-e2e-tests:
96+
needs: prestocpp-linux-build-and-unit-test
97+
runs-on: ubuntu-22.04
98+
container:
99+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
100+
env:
101+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
102+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
103+
MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
104+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
105+
steps:
106+
- uses: actions/checkout@v4
107+
- name: Fix git permissions
108+
# Usually actions/checkout does this but as we run in a container
109+
# it doesn't work
110+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
111+
112+
- name: Download artifacts
113+
uses: actions/download-artifact@v4
114+
with:
115+
name: presto-native-build
116+
path: presto-native-execution/_build/release
117+
118+
- name: Install OpenJDK8
119+
uses: actions/setup-java@v4
120+
with:
121+
distribution: 'temurin'
122+
java-version: '8'
123+
124+
- name: Cache local Maven repository
125+
id: cache-maven
126+
uses: actions/cache@v4
127+
with:
128+
path: ~/.m2/repository
129+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
130+
restore-keys: |
131+
${{ runner.os }}-maven-2-
132+
133+
- name: Populate maven cache
134+
if: steps.cache-maven.outputs.cache-hit != 'true'
135+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
136+
137+
- name: Maven install
138+
env:
139+
MAVEN_OPTS: ${MAVEN_INSTALL_OPTS}
140+
#MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
141+
run: |
142+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
143+
144+
- name: Run presto-native e2e tests
145+
run: |
146+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
147+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'`
148+
# Convert file paths to comma separated class names
149+
export TESTCLASSES=
150+
for test_file in $TESTFILES
151+
do
152+
tmp=${test_file##*/}
153+
test_class=${tmp%%\.*}
154+
export TESTCLASSES="${TESTCLASSES},$test_class"
155+
done
156+
export TESTCLASSES=${TESTCLASSES#,}
157+
echo "TESTCLASSES = $TESTCLASSES"
158+
# TODO: neeed to enable remote function tests with
159+
# "-Ppresto-native-execution-remote-functions" once
160+
# > https://github.com/facebookincubator/velox/discussions/6163
161+
# is fixed.
162+
163+
mvn test \
164+
${MAVEN_TEST} \
165+
-pl 'presto-native-execution' \
166+
-Dtest="${TESTCLASSES}" \
167+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
168+
-DDATA_DIR=${RUNNER_TEMP} \
169+
-Duser.timezone=America/Bahia_Banderas \
170+
-T1C
171+
172+
linux-spark-e2e-tests:
173+
needs: prestocpp-linux-build-and-unit-test
174+
runs-on: ubuntu-22.04
175+
container:
176+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
177+
env:
178+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
179+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
180+
MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
181+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
182+
steps:
183+
- uses: actions/checkout@v4
184+
- name: Fix git permissions
185+
# Usually actions/checkout does this but as we run in a container
186+
# it doesn't work
187+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
188+
189+
- name: Download artifacts
190+
uses: actions/download-artifact@v4
191+
with:
192+
name: presto-native-build
193+
path: presto-native-execution/_build/release
194+
195+
- name: Check Java
196+
run: java --version
197+
198+
- name: Cache local Maven repository
199+
id: cache-maven
200+
uses: actions/cache@v4
201+
with:
202+
path: ~/.m2/repository
203+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
204+
restore-keys: |
205+
${{ runner.os }}-maven-2-
206+
207+
- name: Populate maven cache
208+
if: steps.cache-maven.outputs.cache-hit != 'true'
209+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
210+
211+
- name: Maven install
212+
env:
213+
MAVEN_OPTS: ${MAVEN_INSTALL_OPTS}
214+
#MAVEN_FAST_INSTALL: "-B -V --quiet -T 1C -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
215+
run: |
216+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
217+
218+
- name: Run spark e2e tests
219+
run: |
220+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
221+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'`
222+
# Convert file paths to comma separated class names
223+
export TESTCLASSES=
224+
for test_file in $TESTFILES
225+
do
226+
tmp=${test_file##*/}
227+
test_class=${tmp%%\.*}
228+
export TESTCLASSES="${TESTCLASSES},$test_class"
229+
done
230+
export TESTCLASSES=${TESTCLASSES#,}
231+
echo "TESTCLASSES = $TESTCLASSES"
232+
mvn test \
233+
${MAVEN_TEST} \
234+
-pl 'presto-native-execution' \
235+
-Dtest="${TESTCLASSES}" \
236+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
237+
-DDATA_DIR=${RUNNER_TEMP} \
238+
-Duser.timezone=America/Bahia_Banderas \
239+
-T1C

‎.github/workflows/macos-build.yml

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: macos-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
macos-build:
11+
runs-on: macos-13
12+
steps:
13+
- uses: actions/[email protected]
14+
- name: Fix git permissions
15+
# Usually actions/checkout does this but as we run in a container
16+
# it doesn't work
17+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
18+
- name: Update submodules
19+
run: |
20+
cd presto-native-execution
21+
make submodules
22+
- uses: maxim-lobanov/[email protected]
23+
with:
24+
xcode-version: 15
25+
- name: "Setup MacOS"
26+
run: |
27+
set -xu
28+
mkdir ~/deps ~/deps-src
29+
git clone --depth 1 https://github.com/Homebrew/brew ~/deps
30+
PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh
31+
# Calculate the prefix path before we delete brew's repos and taps.
32+
rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%.
33+
rm -rf ~/deps-src
34+
- name: "Build presto_cpp on MacOS"
35+
run: |
36+
clang --version
37+
export INSTALL_PREFIX=~/deps
38+
export PATH=~/deps/bin:$(brew --prefix m4)/bin:$(brew --prefix bison)/bin:${PATH}
39+
cd presto-native-execution
40+
cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
41+
ninja -C _build/debug

0 commit comments

Comments
 (0)