Skip to content

Commit a650c10

Browse files
czentgrczentgr
committed
[native] Test standard runner prestocpp build
1 parent ed176b9 commit a650c10

File tree

6 files changed

+390
-0
lines changed

6 files changed

+390
-0
lines changed

‎.github/workflows/format-check.yml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: format-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
format-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check formatting
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make format-check

‎.github/workflows/header-check.yml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: header-check
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
header-check:
11+
runs-on: ubuntu-latest
12+
container:
13+
image: public.ecr.aws/oss-presto/velox-dev:check
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Check license headers
21+
run: |
22+
git fetch origin master
23+
cd presto-native-execution
24+
make header-check

‎.github/workflows/linux-build-and-unit-test.yml

+240
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
2+
name: linux-build-and-unit-test
3+
4+
on:
5+
workflow_dispatch:
6+
pull_request:
7+
8+
jobs:
9+
linux-build-and-unit-test:
10+
runs-on: ubuntu-22.04
11+
container:
12+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
13+
env:
14+
CCACHE_DIR: "${{ github.workspace }}/ccache"
15+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
16+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
17+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
18+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
19+
steps:
20+
- uses: actions/checkout@v4
21+
- name: Fix git permissions
22+
# Usually actions/checkout does this but as we run in a container
23+
# it doesn't work
24+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
25+
26+
- name: Update velox
27+
run: |
28+
cd presto-native-execution
29+
make velox-submodule
30+
31+
- name: "Install all adapter dependencies"
32+
run: |
33+
mkdir -p ${HOME}/adapter-deps/install
34+
source /opt/rh/gcc-toolset-12/enable
35+
set -xu
36+
cd presto-native-execution
37+
DEPENDENCY_DIR=${HOME}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
38+
39+
- name: "Install github CLI for the ccache stash"
40+
run: ./presto-native-execution/scripts/setup-centos.sh install_gh
41+
42+
- name: Calculate merge-base date for CCache
43+
run: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/master HEAD) | tee merge-base-date
44+
45+
- uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
46+
with:
47+
path: '${{ env.CCACHE_DIR }}'
48+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
49+
50+
- name: "Zero Ccache Statistics"
51+
run: |
52+
ccache -sz
53+
54+
- name: Build
55+
run: |
56+
source /opt/rh/gcc-toolset-12/enable
57+
cd presto-native-execution
58+
cmake \
59+
-B _build/release \
60+
-GNinja \
61+
-DTREAT_WARNINGS_AS_ERRORS=1 \
62+
-DENABLE_ALL_WARNINGS=1 \
63+
-DCMAKE_BUILD_TYPE=Release \
64+
-DPRESTO_ENABLE_PARQUET=ON \
65+
-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
66+
-DPRESTO_ENABLE_JWT=ON \
67+
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
68+
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
69+
-DCMAKE_PREFIX_PATH=/usr/local \
70+
-DThrift_ROOT=/usr/local \
71+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
72+
-DMAX_LINK_JOBS=2
73+
ninja -C _build/release -j 4
74+
75+
- name: Ccache after
76+
run: ccache -s
77+
78+
- uses: apache/infrastructure-actions/stash/restore@4ab8682fbd4623d2b4fc1c98db38aba5091924c3
79+
with:
80+
path: '${{ env.CCACHE_DIR }}'
81+
key: ccache-linux-prestocpp-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
82+
83+
- name: Run Unit Tests
84+
run: |
85+
cd presto-native-execution/_build/debug
86+
ctest -j 4 -VV --output-on-failure --exclude-regex velox.*
87+
88+
- name: Upload artifacts
89+
uses: actions/upload-artifact@v4
90+
with:
91+
name: presto-native-build
92+
path: |
93+
presto-native-execution/_build/release/presto_cpp/main/presto_server
94+
presto-native-execution/_build/release/velox/velox/functions/remote/server/velox_functions_remote_server_main
95+
96+
linux-presto-e2e-tests:
97+
needs: linux-build-and-unit-test
98+
runs-on: ubuntu-22.04
99+
container:
100+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
101+
env:
102+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
103+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
104+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
105+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
106+
steps:
107+
- uses: actions/checkout@v4
108+
- name: Fix git permissions
109+
# Usually actions/checkout does this but as we run in a container
110+
# it doesn't work
111+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
112+
113+
- name: Download artifacts
114+
uses: actions/download-artifact@v4
115+
with:
116+
name: presto-native-build
117+
path: presto-native-execution/_build/release
118+
119+
- name: Install OpenJDK8
120+
uses: actions/setup-java@v4
121+
with:
122+
distribution: 'temurin'
123+
java-version: '8'
124+
125+
- name: Cache local Maven repository
126+
id: cache-maven
127+
uses: actions/cache@v4
128+
with:
129+
path: ~/.m2/repository
130+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
131+
restore-keys: |
132+
${{ runner.os }}-maven-2-
133+
134+
- name: Populate maven cache
135+
if: steps.cache-maven.outputs.cache-hit != 'true'
136+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
137+
138+
- name: Maven install
139+
env:
140+
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
141+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
142+
run: |
143+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
144+
145+
- name: Run presto-native e2e tests
146+
run: |
147+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
148+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoNative*.java'`
149+
# Convert file paths to comma separated class names
150+
export TESTCLASSES=
151+
for test_file in $TESTFILES
152+
do
153+
tmp=${test_file##*/}
154+
test_class=${tmp%%\.*}
155+
export TESTCLASSES="${TESTCLASSES},$test_class"
156+
done
157+
export TESTCLASSES=${TESTCLASSES#,}
158+
echo "TESTCLASSES = $TESTCLASSES"
159+
# TODO: neeed to enable remote function tests with
160+
# "-Ppresto-native-execution-remote-functions" once
161+
# > https://github.com/facebookincubator/velox/discussions/6163
162+
# is fixed.
163+
164+
mvn test \
165+
${MAVEN_TEST} \
166+
-pl 'presto-native-execution' \
167+
-Dtest="${TESTCLASSES}" \
168+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
169+
-DDATA_DIR=${RUNNER_TEMP} \
170+
-Duser.timezone=America/Bahia_Banderas \
171+
-T1C
172+
173+
linux-spark-e2e-tests:
174+
needs: linux-build-and-unit-test
175+
runs-on: ubuntu-22.04
176+
container:
177+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
178+
env:
179+
MAVEN_OPTS: "-Xmx4G -XX:+ExitOnOutOfMemoryError"
180+
MAVEN_INSTALL_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
181+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
182+
MAVEN_TEST: "-B -Dair.check.skip-all -Dmaven.javadoc.skip=true -DLogTestDurationListener.enabled=true --fail-at-end"
183+
steps:
184+
- uses: actions/checkout@v4
185+
- name: Fix git permissions
186+
# Usually actions/checkout does this but as we run in a container
187+
# it doesn't work
188+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
189+
190+
- name: Download artifacts
191+
uses: actions/download-artifact@v4
192+
with:
193+
name: presto-native-build
194+
path: presto-native-execution/_build/release
195+
196+
- name: Check Java
197+
run: java --version
198+
199+
- name: Cache local Maven repository
200+
id: cache-maven
201+
uses: actions/cache@v4
202+
with:
203+
path: ~/.m2/repository
204+
key: ${{ runner.os }}-maven-2-${{ hashFiles('**/pom.xml') }}
205+
restore-keys: |
206+
${{ runner.os }}-maven-2-
207+
208+
- name: Populate maven cache
209+
if: steps.cache-maven.outputs.cache-hit != 'true'
210+
run: ./mvnw de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
211+
212+
- name: Maven install
213+
env:
214+
MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
215+
MAVEN_FAST_INSTALL: "-B -V --quiet -T C1 -DskipTests -Dair.check.skip-all -Dmaven.javadoc.skip=true"
216+
run: |
217+
for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
218+
219+
- name: Run spark e2e tests
220+
run: |
221+
export PRESTO_SERVER_PATH="${GITHUB_WORKSPACE}/presto-native-execution/_build/release/presto_cpp/main/presto_server"
222+
export TESTFILES=`find ./presto-native-execution/src/test -type f -name 'TestPrestoSpark*.java'`
223+
# Convert file paths to comma separated class names
224+
export TESTCLASSES=
225+
for test_file in $TESTFILES
226+
do
227+
tmp=${test_file##*/}
228+
test_class=${tmp%%\.*}
229+
export TESTCLASSES="${TESTCLASSES},$test_class"
230+
done
231+
export TESTCLASSES=${TESTCLASSES#,}
232+
echo "TESTCLASSES = $TESTCLASSES"
233+
mvn test \
234+
${MAVEN_TEST} \
235+
-pl 'presto-native-execution' \
236+
-Dtest="${TESTCLASSES}" \
237+
-DPRESTO_SERVER=${PRESTO_SERVER_PATH} \
238+
-DDATA_DIR=${RUNNER_TEMP} \
239+
-Duser.timezone=America/Bahia_Banderas \
240+
-T1C

‎.github/workflows/macos-build.yml

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: macos-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
macos-build:
11+
runs-on: macos-13
12+
steps:
13+
- uses: actions/[email protected]
14+
- name: Fix git permissions
15+
# Usually actions/checkout does this but as we run in a container
16+
# it doesn't work
17+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
18+
- name: Update submodules
19+
run: |
20+
git config --global --add safe.directory $(pwd)
21+
cd presto-native-execution
22+
make submodules
23+
- uses: maxim-lobanov/[email protected]
24+
with:
25+
xcode-version: 15
26+
- name: "Setup MacOS"
27+
run: |
28+
set -xu
29+
mkdir ~/deps ~/deps-src
30+
git clone --depth 1 https://github.com/Homebrew/brew ~/deps
31+
PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./presto-native-execution/scripts/setup-macos.sh
32+
# Calculate the prefix path before we delete brew's repos and taps.
33+
rm -rf ~/deps/.git ~/deps/Library/Taps/ # Reduce cache size by 70%.
34+
rm -rf ~/deps-src
35+
- name: "Build presto_cpp on MacOS"
36+
run: |
37+
clang --version
38+
export INSTALL_PREFIX=~/deps
39+
export PATH=~/deps/bin:$(brew --prefix m4)/bin:$(brew --prefix bison)/bin:${PATH}
40+
cd presto-native-execution
41+
cmake -B _build/debug -GNinja -DTREAT_WARNINGS_AS_ERRORS=1 -DENABLE_ALL_WARNINGS=1 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
42+
ninja -C _build/debug

‎.github/workflows/prestocpp-linux-build.yml

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
name: prestocpp-linux-build
2+
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
paths:
7+
- 'presto-native-execution/**'
8+
9+
jobs:
10+
linux-build-all:
11+
runs-on: ubuntu-22.04
12+
container:
13+
image: prestodb/presto-native-dependency:0.290-20241014120930-e1fc090
14+
steps:
15+
- uses: actions/[email protected]
16+
- name: Fix git permissions
17+
# Usually actions/checkout does this but as we run in a container
18+
# it doesn't work
19+
run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
20+
- name: Update velox
21+
run: |
22+
cd presto-native-execution
23+
make velox-submodule
24+
- name: Install all adapter dependencies
25+
run: |
26+
mkdir -p ${GITHUB_WORKSPACE}/adapter-deps/install
27+
source /opt/rh/gcc-toolset-12/enable
28+
set -xu
29+
cd presto-native-execution
30+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./velox/scripts/setup-adapters.sh
31+
DEPENDENCY_DIR=${GITHUB_WORKSPACE}/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
32+
- name: Build All
33+
run: |
34+
source /opt/rh/gcc-toolset-12/enable
35+
cd presto-native-execution
36+
cmake \
37+
-B _build/debug \
38+
-GNinja \
39+
-DAWSSDK_ROOT_DIR=${GITHUB_WORKSPACE}/adapter-deps/install \
40+
-DTREAT_WARNINGS_AS_ERRORS=1 \
41+
-DENABLE_ALL_WARNINGS=1 \
42+
-DCMAKE_BUILD_TYPE=Debug \
43+
-DPRESTO_ENABLE_PARQUET=ON \
44+
-DPRESTO_ENABLE_S3=ON \
45+
-DPRESTO_ENABLE_REMOTE_FUNCTIONS=ON \
46+
-DPRESTO_ENABLE_JWT=ON \
47+
-DPRESTO_STATS_REPORTER_TYPE=PROMETHEUS \
48+
-DPRESTO_MEMORY_CHECKER_TYPE=LINUX_MEMORY_CHECKER \
49+
-DPRESTO_ENABLE_TESTING=OFF \
50+
-DCMAKE_PREFIX_PATH=/usr/local \
51+
-DThrift_ROOT=/usr/local \
52+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
53+
-DMAX_LINK_JOBS=2
54+
ninja -C _build/debug -j 4

0 commit comments

Comments
 (0)