Add CI job to build and run Python tests

.github/workflows/ci-pr-validation.yaml

@@ -0,0 +1,66 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: PR validation
+on:
+  pull_request:
+    branches: ['main']
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  unit-tests:
+    name: Run unit tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Install deps
+        run: sudo apt-get install -y libboost-python-dev
+
+      - name: Install Pulsar C++ client
+        run: build-support/install-cpp-client.sh
+
+      - name: CMake
+        run: cmake .
+
+      - name: Build
+        run: make -j8
+
+      - name: Python install
+        run: |
+            ./setup.py bdist_wheel
+            WHEEL=$(find dist -name '*.whl')
+            pip3 install ${WHEEL}[avro]
+
+      - name: Start Pulsar service
+        run: ./build-support/pulsar-test-service-start.sh
+
+      - name: Run unit tests
+        run: ./tests/run-unit-tests.sh
+
+      - name: Stop Pulsar service
+        run: ./build-support/pulsar-test-service-stop.sh
+

.gitignore

@@ -9,3 +9,6 @@ Makefile
 _pulsar.so
 cmake_install.cmake
 __pycache__
+.build
+.pulsar-mac-wheels-cache
+.DS_Store

CMakeLists.txt

@@ -21,20 +21,26 @@ project (pulsar-client-python)
 cmake_minimum_required(VERSION 3.12)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
+option(LINK_STATIC "Link against static libraries" OFF)
+MESSAGE(STATUS "LINK_STATIC:  " ${LINK_STATIC})
+
 MESSAGE(STATUS "CMAKE_BUILD_TYPE:  " ${CMAKE_BUILD_TYPE})
 set(THREADS_PREFER_PTHREAD_FLAG TRUE)
 find_package(Threads REQUIRED)
 MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT})
 
-
-find_library(PULSAR_LIBRARY NAMES libpulsar.a)
+if (LINK_STATIC)
+    find_library(PULSAR_LIBRARY NAMES libpulsar.a)
+else()
+    find_library(PULSAR_LIBRARY NAMES libpulsar.so)
+endif()
 message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}")
 
 find_path(PULSAR_INCLUDE pulsar/Client.h)
 message(STATUS "PULSAR_INCLUDE: ${PULSAR_INCLUDE}")
 
 SET(Boost_NO_BOOST_CMAKE ON)
-SET(Boost_USE_STATIC_LIBS   ON)
+SET(Boost_USE_STATIC_LIBS   ${LINK_STATIC})
 
 SET(CMAKE_CXX_STANDARD 11)
 
@@ -63,32 +69,6 @@ endif ()
 
 MESSAGE(STATUS "BOOST_PYTHON_NAME_FOUND: " ${BOOST_PYTHON_NAME_FOUND})
 
-set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/lib64/)
-
-### This part is to find and keep SSL dynamic libs in RECORD_OPENSSL_SSL_LIBRARY and RECORD_OPENSSL_CRYPTO_LIBRARY
-### After find the libs, will unset related cache, and will not affect another same call to find_package.
-if (APPLE)
-    set(OPENSSL_INCLUDE_DIR /usr/local/opt/openssl/include/ /opt/homebrew/opt/openssl/include)
-    set(OPENSSL_ROOT_DIR ${OPENSSL_ROOT_DIR} /usr/local/opt/openssl/ /opt/homebrew/opt/openssl)
-endif ()
-
-set(OPENSSL_USE_STATIC_LIBS TRUE)
-find_package(OpenSSL REQUIRED)
-
-find_library(ZLIB_LIBRARIES REQUIRED NAMES libz.a z zlib)
-message(STATUS "ZLIB_LIBRARIES: ${ZLIB_LIBRARIES}")
-
-find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a)
-message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}")
-find_library(Protobuf_LIBRARIES NAMES libprotobuf.a libprotobuf)
-message(STATUS "Protobuf: ${Protobuf_LIBRARIES}")
-find_library(CURL_LIBRARIES NAMES libcurl.a curl curl_a libcurl_a)
-message(STATUS "CURL_LIBRARIES: ${CURL_LIBRARIES}")
-find_library(LIB_ZSTD NAMES libzstd.a)
-message(STATUS "ZStd: ${LIB_ZSTD}")
-find_library(LIB_SNAPPY NAMES libsnappy.a)
-message(STATUS "LIB_SNAPPY: ${LIB_SNAPPY}")
-
 ########################################################################################################################
 
 INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} "${Boost_INCLUDE_DIRS}" "${Python3_INCLUDE_DIRS}")
@@ -138,12 +118,6 @@ endif()
 
 # Try all possible boost-python variable namings
 set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY}
-                        ${OPENSSL_LIBRARIES}
-                        ${ZLIB_LIBRARIES}
-                        ${CURL_LIBRARIES}
-                        ${Protobuf_LIBRARIES}
-                        ${LIB_ZSTD}
-                        ${LIB_SNAPPY}
                         ${Boost_PYTHON_LIBRARY}
                         ${Boost_PYTHON3_LIBRARY}
                         ${Boost_PYTHON37-MT_LIBRARY}
@@ -173,13 +147,17 @@ endif ()
 
 message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}")
 
-if (APPLE)
-    set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
-    target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS})
-else ()
-    if (NOT MSVC)
-      set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc  -static-libstdc++")
-    endif()
+if (LINK_STATIC)
+    if (APPLE)
+        set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup")
+        target_link_libraries(_pulsar -Wl,-all_load ${PYTHON_WRAPPER_LIBS})
+    else ()
+        if (NOT MSVC)
+          set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc  -static-libstdc++")
+        endif()
+        target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
+    endif ()
+else()
     target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS})
 endif ()
 

build-support/install-cpp-client.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e -x
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+
+cd $ROOT_DIR
+
+CPP_CLIENT_VERSION=$(cat pulsar-client-cpp-version.txt | xargs)
+
+# Fetch the client binaries
+## TODO: Fetch from official release once it's available
+pushd /tmp
+  curl -L -O https://github.com/merlimat/pulsar-client-cpp/releases/download/${CPP_CLIENT_VERSION}/apache-pulsar-client.deb
+  curl -L -O https://github.com/merlimat/pulsar-client-cpp/releases/download/${CPP_CLIENT_VERSION}/apache-pulsar-client-dev.deb
+popd
+
+sudo apt install /tmp/apache-pulsar-client.deb /tmp/apache-pulsar-client-dev.deb

build-support/pulsar-test-service-start.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+SRC_DIR=$(git rev-parse --show-toplevel)
+cd $SRC_DIR
+
+./build-support/pulsar-test-service-stop.sh
+
+CONTAINER_ID=$(docker run -i -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:latest sleep 3600)
+echo $CONTAINER_ID > .tests-container-id.txt
+
+docker cp tests/test-conf $CONTAINER_ID:/pulsar/test-conf
+docker cp build-support/start-test-service-inside-container.sh $CONTAINER_ID:start-test-service-inside-container.sh
+
+docker exec -i $CONTAINER_ID /start-test-service-inside-container.sh
+
+docker cp $CONTAINER_ID:/pulsar/data/tokens/token.txt tests/.test-token.txt
+
+echo "-- Ready to start tests"

build-support/pulsar-test-service-stop.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+SRC_DIR=$(git rev-parse --show-toplevel)
+cd $SRC_DIR
+
+CONTAINER_ID_PATH=".tests-container-id.txt"
+
+if [ -f ${CONTAINER_ID_PATH} ]; then
+  CONTAINER_ID=$(cat $CONTAINER_ID_PATH)
+  docker kill $CONTAINER_ID || true
+  rm .tests-container-id.txt
+fi

build-support/start-test-service-inside-container.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e -x
+
+export PULSAR_EXTRA_OPTS=-Dpulsar.auth.basic.conf=test-conf/.htpasswd
+
+# Generate secret key and token
+mkdir -p data/tokens
+bin/pulsar tokens create-secret-key --output data/tokens/secret.key
+
+bin/pulsar tokens create \
+            --subject token-principal \
+            --secret-key file:///pulsar/data/tokens/secret.key \
+            > /pulsar/data/tokens/token.txt
+
+export PULSAR_STANDALONE_CONF=test-conf/standalone-ssl.conf
+export PULSAR_PID_DIR=/tmp
+bin/pulsar-daemon start standalone \
+        --no-functions-worker --no-stream-storage \
+        --bookkeeper-dir data/bookkeeper
+
+echo "-- Wait for Pulsar service to be ready"
+until curl http://localhost:8080/metrics > /dev/null 2>&1 ; do sleep 1; done
+
+echo "-- Pulsar service is ready -- Configure permissions"
+
+export PULSAR_CLIENT_CONF=test-conf/client-ssl.conf
+
+# Create "standalone" cluster if it does not exist
+bin/pulsar-admin clusters list | grep -q '^standalone$' ||
+        bin/pulsar-admin clusters create \
+                standalone \
+                --url http://localhost:8080/ \
+                --url-secure https://localhost:8443/ \
+                --broker-url pulsar://localhost:6650/ \
+                --broker-url-secure pulsar+ssl://localhost:6651/
+
+# Update "public" tenant
+bin/pulsar-admin tenants create public -r "anonymous" -c "standalone"
+
+# Update "public/default" with no auth required
+bin/pulsar-admin namespaces create public/default -c standalone
+bin/pulsar-admin namespaces grant-permission public/default \
+                        --actions produce,consume \
+                        --role "anonymous"
+
+# Create "public/default-2" with no auth required
+bin/pulsar-admin namespaces create public/default-2 \
+                        --clusters standalone
+bin/pulsar-admin namespaces grant-permission public/default-2 \
+                        --actions produce,consume \
+                        --role "anonymous"
+
+# Create "public/default-3" with no auth required
+bin/pulsar-admin namespaces create public/default-3 \
+                        --clusters standalone
+bin/pulsar-admin namespaces grant-permission public/default-3 \
+                        --actions produce,consume \
+                        --role "anonymous"
+
+# Create "public/default-4" with encryption required
+bin/pulsar-admin namespaces create public/default-4 \
+                        --clusters standalone
+bin/pulsar-admin namespaces grant-permission public/default-4 \
+                        --actions produce,consume \
+                        --role "anonymous"
+bin/pulsar-admin namespaces set-encryption-required public/default-4 -e
+
+# Create "public/test-backlog-quotas" to test backlog quotas policy
+bin/pulsar-admin namespaces create public/test-backlog-quotas \
+                        --clusters standalone
+
+# Create "private" tenant
+bin/pulsar-admin tenants create private -r "" -c "standalone"
+
+# Create "private/auth" with required authentication
+bin/pulsar-admin namespaces create private/auth --clusters standalone
+
+bin/pulsar-admin namespaces grant-permission private/auth \
+                        --actions produce,consume \
+                        --role "token-principal"
+
+echo "-- Ready to start tests"

pulsar-client-cpp-version.txt

@@ -0,0 +1 @@
+3.0.0-pre-1