pax_global_header00006660000000000000000000000064152010110520014475gustar00rootroot0000000000000052 comment=2b7a9138d8126dc411c611522b2940b6076ec0c8 pulsar-client-python-3.12.0-candidate-0/000077500000000000000000000000001520101105200177305ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/.asf.yaml000066400000000000000000000032131520101105200214420ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # github: description: "Apache Pulsar Python client library" homepage: https://pulsar.apache.org/ labels: - pulsar - pubsub - messaging - streaming - queuing - event-streaming features: wiki: false issues: true projects: true enabled_merge_buttons: squash: true merge: false rebase: false protected_branches: main: required_status_checks: # Contexts are the names of checks that must pass. # See ./github/workflows/README.md for more documentation on this list. contexts: - Check Completion required_pull_request_reviews: required_approving_review_count: 1 notifications: commits: commits@pulsar.apache.org issues: commits@pulsar.apache.org pullrequests: commits@pulsar.apache.org discussions: dev@pulsar.apache.org jira_options: link label pulsar-client-python-3.12.0-candidate-0/.clang-format000066400000000000000000000016271520101105200223110ustar00rootroot00000000000000# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. BasedOnStyle: Google IndentWidth: 4 ColumnLimit: 110 SortIncludes: false BreakBeforeBraces: Custom BraceWrapping: AfterEnum: true pulsar-client-python-3.12.0-candidate-0/.github/000077500000000000000000000000001520101105200212705ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/.github/workflows/000077500000000000000000000000001520101105200233255ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/.github/workflows/ci-build-release-wheels.yaml000066400000000000000000000137331520101105200306130ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # name: Build release wheels on: push: tags: - '*' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: linux-wheel: name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}} runs-on: ubuntu-latest timeout-minutes: 360 strategy: fail-fast: false matrix: image: - {name: 'manylinux', py_suffix: ''} - {name: 'manylinux_musl', py_suffix: '-alpine'} python: - {version: '3.10', spec: 'cp310-cp310'} - {version: '3.11', spec: 'cp311-cp311'} - {version: '3.12', spec: 'cp312-cp312'} - {version: '3.13', spec: 'cp313-cp313'} - {version: '3.14', spec: 'cp314-cp314'} cpu: - {arch: 'x86_64', platform: 'x86_64'} - {arch: 'aarch64', platform: 'arm64'} steps: - name: checkout uses: actions/checkout@v3 - name: Set up QEMU uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - name: Build Manylinux Docker image uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 with: context: ./pkg/${{matrix.image.name}} load: true tags: build:latest platforms: linux/${{matrix.cpu.arch}} build-args: | PLATFORM=${{matrix.cpu.platform}} ARCH=${{matrix.cpu.arch}} PYTHON_VERSION=${{matrix.python.version}} PYTHON_SPEC=${{matrix.python.spec}} cache-from: type=gha cache-to: type=gha,mode=max - name: Build wheel file run: | docker run -i -v $PWD:/pulsar-client-python build:latest \ /pulsar-client-python/pkg/build-wheel-inside-docker.sh - name: Test wheel file run: | docker run -i -v $PWD:/pulsar-client-python \ --platform linux/${{matrix.cpu.arch}} \ python:${{matrix.python.version}}${{matrix.image.py_suffix}} \ /pulsar-client-python/pkg/test-wheel.sh - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: wheel-${{matrix.image.name}}-py${{matrix.python.version}}-${{matrix.cpu.platform}} path: wheelhouse/*.whl mac-wheels: name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} runs-on: macos-14 timeout-minutes: 360 strategy: fail-fast: false matrix: py: - {version: '3.10', version_long: '3.10.15'} - {version: '3.11', version_long: '3.11.11'} - {version: '3.12', version_long: '3.12.8'} - {version: '3.13', version_long: '3.13.1'} - {version: '3.14', version_long: '3.14.0'} steps: - name: checkout uses: actions/checkout@v3 - name: Build and test Mac wheels run: pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}} - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: wheel-mac-py${{matrix.py.version}} path: dist/*.whl windows-wheels: name: Wheel Windows - Py ${{matrix.python.version}} runs-on: windows-2022 env: PULSAR_CPP_DIR: 'C:\\pulsar-cpp' timeout-minutes: 360 strategy: fail-fast: false matrix: python: - {version: '3.10'} - {version: '3.11'} - {version: '3.12'} - {version: '3.13'} - {version: '3.14'} steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python.version }} - name: Download Pulsar C++ client on Windows shell: bash run: | source ./build-support/dep-url.sh BASE_URL=$(pulsar_cpp_base_url $(grep pulsar-cpp dependencies.yaml | awk '{print $2}')) mkdir -p ${{ env.PULSAR_CPP_DIR }} cd ${{ env.PULSAR_CPP_DIR }} curl -O -L ${BASE_URL}/x64-windows-static.tar.gz tar zxf x64-windows-static.tar.gz mv x64-windows-static/* . ls -l ${{ env.PULSAR_CPP_DIR }} - name: Configure CMake shell: bash run: | pip3 install pyyaml export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11) curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz tar zxf v${PYBIND11_VERSION}.tar.gz rm -rf pybind11 mv pybind11-${PYBIND11_VERSION} pybind11 cmake -B build -A x64 \ -DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \ -DLINK_STATIC=ON - name: Build Python wheel shell: bash run: | cmake --build build --config Release --target install python -m pip install wheel setuptools python setup.py bdist_wheel python -m pip install ./dist/*.whl python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()' - name: Upload artifacts uses: actions/upload-artifact@v4 with: name: wheel-windows-py${{matrix.python.version}} path: dist/*.whl pulsar-client-python-3.12.0-candidate-0/.github/workflows/ci-pr-validation.yaml000066400000000000000000000173361520101105200273650ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # name: PR validation on: pull_request: branches: ['main'] concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: check-and-lint: name: Lint and check code runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: 3.x - name: Install deps run: pip install -U pydoctor - name: Check docs syntax run: | pydoctor --make-html \ --html-viewsource-base=https://github.com/apache/pulsar-client-python/tree/main \ --docformat=numpy --theme=readthedocs \ --intersphinx=https://docs.python.org/3/objects.inv \ --html-output=apidocs \ pulsar unit-tests: name: Run unit tests for Python ${{matrix.version}} runs-on: ubuntu-latest timeout-minutes: 120 strategy: fail-fast: false matrix: version: ['3.10', '3.14'] steps: - name: checkout uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "${{matrix.version}}" - name: Install Pulsar C++ client run: build-support/install-dependencies.sh - name: CMake run: cmake . - name: Build run: make -j8 - name: Python install run: | python3 -m pip install -U pip setuptools wheel requests python3 setup.py bdist_wheel WHEEL=$(find dist -name '*.whl') pip3 install ${WHEEL}[avro,protobuf] - name: Run Oauth2 tests run: | docker compose -f ./build-support/docker-compose-pulsar-oauth2.yml up -d # Wait until the namespace is created, currently there is no good way to check it via CLI sleep 10 python3 tests/oauth2_test.py docker compose -f ./build-support/docker-compose-pulsar-oauth2.yml down - name: Start Pulsar service run: ./build-support/pulsar-test-service-start.sh - name: Run unit tests run: ./tests/run-unit-tests.sh - name: Stop Pulsar service run: ./build-support/pulsar-test-service-stop.sh - name: Test functions install run: | WHEEL=$(find dist -name '*.whl') pip3 install ${WHEEL}[all] --force-reinstall linux-wheel: name: Wheel ${{matrix.image.name}} - Py ${{matrix.python.version}} - ${{matrix.cpu.platform}} needs: unit-tests runs-on: ubuntu-latest timeout-minutes: 300 strategy: fail-fast: false matrix: image: - {name: 'manylinux', py_suffix: ''} - {name: 'manylinux_musl', py_suffix: '-alpine'} python: - {version: '3.14', spec: 'cp314-cp314'} cpu: - {arch: 'x86_64', platform: 'x86_64'} steps: - name: checkout uses: actions/checkout@v3 - name: Set up QEMU uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd - name: Build Manylinux Docker image uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 with: context: ./pkg/${{matrix.image.name}} load: true tags: build:latest platforms: linux/${{matrix.cpu.arch}} build-args: | PLATFORM=${{matrix.cpu.platform}} ARCH=${{matrix.cpu.arch}} PYTHON_VERSION=${{matrix.python.version}} PYTHON_SPEC=${{matrix.python.spec}} cache-from: type=gha cache-to: type=gha,mode=max - name: Build wheel file run: | docker run -i -v $PWD:/pulsar-client-python build:latest \ /pulsar-client-python/pkg/build-wheel-inside-docker.sh - name: Test wheel file run: | docker run -i -v $PWD:/pulsar-client-python python:${{matrix.python.version}}${{matrix.image.py_suffix}} \ /pulsar-client-python/pkg/test-wheel.sh mac-wheels: name: Wheel MacOS Universal2 - Py ${{matrix.py.version}} needs: unit-tests runs-on: macos-14 timeout-minutes: 300 strategy: matrix: py: - {version: '3.14', version_long: '3.14.0'} steps: - name: checkout uses: actions/checkout@v3 - name: Build and test Mac wheels run: | pkg/mac/build-mac-wheels.sh ${{matrix.py.version}} ${{matrix.py.version_long}} windows-wheels: name: "Python ${{ matrix.python.version }} Wheel on Windows x64" needs: unit-tests runs-on: windows-2022 timeout-minutes: 120 env: PULSAR_CPP_DIR: 'C:\\pulsar-cpp' strategy: fail-fast: false matrix: python: - version: '3.14' steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python.version }} - name: Download Pulsar C++ client on Windows shell: bash run: | source ./build-support/dep-url.sh BASE_URL=$(pulsar_cpp_base_url $(grep pulsar-cpp dependencies.yaml | awk '{print $2}')) mkdir -p ${{ env.PULSAR_CPP_DIR }} cd ${{ env.PULSAR_CPP_DIR }} curl -O -L ${BASE_URL}/x64-windows-static.tar.gz tar zxf x64-windows-static.tar.gz mv x64-windows-static/* . ls -l ${{ env.PULSAR_CPP_DIR }} - name: Configure CMake shell: bash run: | pip3 install pyyaml export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11) curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz tar zxf v${PYBIND11_VERSION}.tar.gz rm -rf pybind11 mv pybind11-${PYBIND11_VERSION} pybind11 cmake -B build -A x64 \ -DCMAKE_PREFIX_PATH=${{ env.PULSAR_CPP_DIR }} \ -DLINK_STATIC=ON - name: Build Python wheel shell: bash run: | cmake --build build --config Release --target install python -m pip install wheel setuptools python setup.py bdist_wheel python -m pip install ./dist/*.whl python -c 'import pulsar; c = pulsar.Client("pulsar://localhost:6650"); c.close()' check-completion: name: Check Completion runs-on: ubuntu-latest if: ${{ always() }} needs: [ check-and-lint, unit-tests, linux-wheel, mac-wheels, windows-wheels ] steps: - run: | if [[ ! ( \ "${{ needs.check-and-lint.result }}" == "success" \ && "${{ needs.unit-tests.result }}" == "success" \ && "${{ needs.linux-wheel.result }}" == "success" \ && "${{ needs.mac-wheels.result }}" == "success" \ && "${{ needs.windows-wheels.result }}" == "success" \ ) ]]; then echo "Required jobs haven't been completed successfully." exit 1 fi pulsar-client-python-3.12.0-candidate-0/.github/workflows/codeql.yml000066400000000000000000000046771520101105200253350ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # name: "CodeQL" on: push: branches: [ "main" ] pull_request: branches: [ "main" ] schedule: - cron: '27 21 * * 6' jobs: analyze: name: Analyze runs-on: 'ubuntu-latest' timeout-minutes: 360 permissions: # required for all workflows security-events: write # only required for workflows in private repositories actions: read contents: read strategy: fail-fast: false matrix: language: [ 'c-cpp', 'python' ] steps: - name: Checkout repository uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality - uses: actions/setup-python@v5 with: python-version: "3.12" - name: Install Pulsar C++ client run: build-support/install-dependencies.sh - name: CMake run: cmake . - name: Build run: make -j8 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" pulsar-client-python-3.12.0-candidate-0/.gitignore000066400000000000000000000004171520101105200217220ustar00rootroot00000000000000MANIFEST build dist *.egg-info .idea CMakeCache.txt CMakeFiles Makefile _pulsar.so cmake_install.cmake __pycache__ .build .pulsar-mac-wheels-cache .DS_Store wheelhouse .pulsar-mac-build vcpkg_installed/ *.pyd *.lib lib_pulsar.so tests/test.log .tests-container-id.txt pulsar-client-python-3.12.0-candidate-0/.gitmodules000066400000000000000000000001261520101105200221040ustar00rootroot00000000000000[submodule "pybind11"] path = pybind11 url = https://github.com/pybind/pybind11.git pulsar-client-python-3.12.0-candidate-0/CMakeLists.txt000066400000000000000000000101051520101105200224650ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # cmake_minimum_required(VERSION 3.18) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") project (pulsar-client-python) set(CMAKE_PREFIX_PATH ${PROJECT_SOURCE_DIR}/pybind11/include ${CMAKE_PREFIX_PATH}) option(LINK_STATIC "Link against static libraries" OFF) MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC}) if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif () MESSAGE(STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE}) set(THREADS_PREFER_PTHREAD_FLAG TRUE) find_package(Threads REQUIRED) MESSAGE(STATUS "Threads library: " ${CMAKE_THREAD_LIBS_INIT}) if (MSVC) add_compile_options(/wd4819) endif () if (LINK_STATIC) find_library(PULSAR_LIBRARY NAMES pulsarwithdeps pulsarWithDeps.lib) add_definitions("-DPULSAR_STATIC") else () find_library(PULSAR_LIBRARY NAMES pulsar libpulsar) endif() message(STATUS "PULSAR_LIBRARY: ${PULSAR_LIBRARY}") find_path(PULSAR_INCLUDE pulsar/Client.h) message(STATUS "PULSAR_INCLUDE: ${PULSAR_INCLUDE}") SET(CMAKE_CXX_STANDARD 17) find_package (Python3 REQUIRED COMPONENTS Development.Module) MESSAGE(STATUS "PYTHON: " ${Python3_VERSION} " - " ${Python3_INCLUDE_DIRS}) find_path(PYBIND11_INCLUDE_DIRS NAMES "pybind11/pybind11.h") message(STATUS "PYBIND11_INCLUDE_DIRS: " ${PYBIND11_INCLUDE_DIRS}) ######################################################################################################################## INCLUDE_DIRECTORIES(${PULSAR_INCLUDE} ${PYBIND11_INCLUDE_DIRS} ${Python3_INCLUDE_DIRS}) file(GLOB SOURCES src/*.cc) ADD_LIBRARY(_pulsar SHARED ${SOURCES}) if (MSVC) set(CMAKE_SHARED_LIBRARY_SUFFIX .pyd) else () set(CMAKE_SHARED_LIBRARY_SUFFIX .so) endif () if (NOT APPLE AND NOT MSVC) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_PYTHON}") endif() if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -Qunused-arguments -undefined dynamic_lookup") endif() set(PYTHON_WRAPPER_LIBS ${PULSAR_LIBRARY} ) set(PYTHON_WRAPPER_LIBS ${PYTHON_WRAPPER_LIBS} Python3::Module) message(STATUS "All libraries: ${PYTHON_WRAPPER_LIBS}") if (LINK_STATIC AND NOT MSVC) if (APPLE) set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS "${CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS} -undefined dynamic_lookup") target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) else () set (CMAKE_SHARED_LINKER_FLAGS " -static-libgcc -static-libstdc++") target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) endif () elseif (LINK_STATIC) # MSVC set_property(TARGET _pulsar PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) else() target_link_libraries(_pulsar ${PYTHON_WRAPPER_LIBS}) endif () install(TARGETS _pulsar DESTINATION ${CMAKE_SOURCE_DIR}) find_package(ClangTools) set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py ${CLANG_FORMAT_BIN} 0 ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt ${CMAKE_SOURCE_DIR}/src) # `make check-format` option (for CI test) add_custom_target(check-format ${BUILD_SUPPORT_DIR}/run_clang_format.py ${CLANG_FORMAT_BIN} 1 ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt ${CMAKE_SOURCE_DIR}/src) pulsar-client-python-3.12.0-candidate-0/CONTRIBUTING.md000066400000000000000000000020561520101105200221640ustar00rootroot00000000000000 ## Contributing to Apache Pulsar We would love for you to contribute to Apache Pulsar and make it even better! Please check the [Contributing to Apache Pulsar](https://pulsar.apache.org/contributing/) page before starting to work on the project. pulsar-client-python-3.12.0-candidate-0/LICENSE000066400000000000000000000261361520101105200207450ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. pulsar-client-python-3.12.0-candidate-0/NOTICE000066400000000000000000000002471520101105200206370ustar00rootroot00000000000000Apache Pulsar Copyright 2017-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). pulsar-client-python-3.12.0-candidate-0/README.md000066400000000000000000000116741520101105200212200ustar00rootroot00000000000000 # Pulsar Python client library Pulsar Python clients support a variety of Pulsar features to enable building applications connecting to your Pulsar cluster. For the supported Pulsar features, see [Client Feature Matrix](https://pulsar.apache.org/client-feature-matrix/). ## Requirements - Python 3.10, 3.11, 3.12, 3.13 or 3.14 - A C++ compiler that supports C++11 - CMake >= 3.18 - [Pulsar C++ client library](https://github.com/apache/pulsar-client-cpp) - [PyBind11](https://github.com/pybind/pybind11) PyBind11 is a header-only library and a submodule, so you can simply download the submodule so that CMake can find this dependency. ```bash git submodule update --init ``` You can also download the pybind11 directly like: ```bash pip3 install pyyaml export PYBIND11_VERSION=$(./build-support/dep-version.py pybind11) curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz tar zxf v${PYBIND11_VERSION}.tar.gz mv pybind11-${PYBIND11_VERSION} pybind11 ``` After that, you only need to install the Pulsar C++ client dependency into the system path. You can [install the pre-built binaries](https://pulsar.apache.org/docs/next/client-libraries-cpp/#installation) or [build from source](https://github.com/apache/pulsar-client-cpp#compilation). ## Install the Python wheel Make sure the PyBind11 submodule has been downloaded and the Pulsar C++ client has been installed. Then run the following commands: ```bash cmake -B build cmake --build build cmake --install build python3 ./setup.py bdist_wheel python3 -m pip install dist/pulsar_client-*.whl --force-reinstall ``` > **NOTE** > > 1. The separate `build` directory is created to store all CMake temporary files. However, the `setup.py` requires the `_pulsar.so` to be under the project directory. > 2. Add the `--force-reinstall` option to overwrite the existing Python wheel in case your system has already installed a wheel before. > 3. On Windows, the Python command is `py` instead of `python3`. ## Running examples You can run `python3 -c 'import pulsar'` to see whether the wheel has been installed successfully. If it fails, check whether dependencies (e.g., `libpulsar.so`) are in the system path. If not, make sure the dependencies are in `LD_LIBRARY_PATH` (on Linux) or `DYLD_LIBRARY_PATH` (on macOS). Then you can run examples as a simple end-to-end test. ```bash # In terminal 1 python3 ./examples/consumer.py ``` ```bash # In terminal 2 python3 ./examples/producer.py ``` Before executing the commands above, you must ensure the Pulsar service is running. See [here](https://pulsar.apache.org/docs/getting-started-standalone) for quick start. ## Unit tests Before running the unit tests, you must run a Pulsar service with all things set up: ```bash ./build-support/pulsar-test-service-start.sh ``` The command above runs a Pulsar standalone in a Docker container. You can run `./build-support/pulsar-test-service-stop.sh` to stop it. Run all unit tests: ```bash ./tests/run-unit-tests.sh ``` Run a single unit test (e.g., `PulsarTest.test_tls_auth`): ```bash python3 ./tests/pulsar_test.py 'PulsarTest.test_tls_auth' ``` ## Generate API docs Pulsar Python Client uses [pydoctor](https://github.com/twisted/pydoctor) to generate API docs. To generate by yourself, you need to install the Python library first. Then run the following command in the root path of this repository: ```bash sudo python3 -m pip install pydoctor cp $(python3 -c 'import _pulsar, os; print(_pulsar.__file__)') ./_pulsar.so pydoctor --make-html \ --docformat=numpy --theme=readthedocs \ --intersphinx=https://docs.python.org/3/objects.inv \ --html-output= \ --introspect-c-modules \ ./_pulsar.so \ pulsar ``` Then the index page will be generated in `/index.html`. ## Contribute We welcome contributions from the open source community! If your contribution adds Pulsar features for Python clients, you need to update both the [Pulsar docs](https://pulsar.apache.org/docs/client-libraries/) and the [Client Feature Matrix](https://pulsar.apache.org/client-feature-matrix/). See [Contribution Guide](https://pulsar.apache.org/contribute/site-intro/#pages) for more details. pulsar-client-python-3.12.0-candidate-0/RELEASE.md000066400000000000000000000270111520101105200213330ustar00rootroot00000000000000 # Release Process This page contains instructions for Pulsar committers on how to perform a release for the Pulsar Python client. ## Versioning Bump up the version number as follows. * Major version (e.g. 3.0.0 => 4.0.0) * Changes that break backward compatibility * Minor version (e.g. 3.0.0 => 3.1.0) * Backward compatible new features * Patch version (e.g. 3.0.0 => 3.0.1) * Backward compatible bug fixes * C++ Client upgrade (even though there are no new commits in the Python client) ## Requirements If you haven't already done it, [create and publish the GPG key](https://pulsar.apache.org/contribute/create-gpg-keys/) to sign the release artifacts. Before running the commands in the following sections, make sure the `GPG_TTY` environment variable has been set. ```bash export GPG_TTY=$(tty) ``` ## Upgrade the C++ client dependency During the development, the C++ client dependency might be downloaded from an unofficial release. But when releasing the Python client, the dependency must be downloaded from an official release. You should modify the base url in [dep-url.sh](./build-support/dep-url.sh). Example: https://github.com/apache/pulsar-client-python/pull/62 ## Cut the candidate release After all necessary PRs are cherry-picked to `branch-X.Y`, you should cut the release by pushing a tag. For major and minor releases (`X.Y.0`), you need to create a new branch: ```bash git checkout -b branch-X.Y perl -pi -e "s/__version__.*/__version__='X.Y.Z'/" pulsar/__about__.py git add pulsar/__about__.py git commit -m "Bump version to X.Y.0" git push origin branch-X.Y # N starts with 1 git tag -u $USER@apache.org vX.Y.0-candidate-N -m "Release vX.Y.0 candidate N" git push origin vX.Y.0-candidate-N ``` For patch releases (`X.Y.Z`), you need to reuse the existing branch: ```bash git checkout branch-X.Y perl -pi -e "s/__version__.*/__version__='X.Y.Z'/" pulsar/__about__.py git add pulsar/__about__.py git commit -m "Bump version to X.Y.Z" git push origin branch-X.Y # N starts with 1 git tag -u $USER@apache.org vX.Y.Z-candidate-N -m "Release vX.Y.Z candidate N" git push origin vX.Y.Z-candidate-N ``` Then, [create a new milestone](https://github.com/apache/pulsar-client-python/milestones/new) for the next major release. ## Sign and stage the candidate release After a tag is pushed, a workflow will be triggered to build the Python wheels in GitHub Actions. You can find it in https://github.com/apache/pulsar-client-python/actions/workflows/ci-build-release-wheels.yaml For example, https://github.com/apache/pulsar-client-python/actions/runs/3709463737 is the workflow of `v3.0.0-candidate-3`, the workflow id is 3709463737. Remember the workflow id, which will be passed as an argument of `stage-release.sh` in the following step. Once the workflow is completed, the wheels will be available for downloading. Generate a GitHub token by following the guide [here](https://github.com/settings/tokens). The `repo` and `workflow` checkboxes must be selected. Then export the token as the environment variable: ```bash export GITHUB_TOKEN= ``` Edit `~/.gnupg/gpg.conf` to ensure the default GPG key is from your Apache mail (`@apache.org`): ``` default-key ``` Make sure `curl`, `jq`, `unzip`, `gpg`, `shasum` commands are available. Then you should run the following commands in an empty directory: ```bash svn co https://dist.apache.org/repos/dist/dev/pulsar pulsar-dist-dev-keys --depth empty cd pulsar-dist-dev-keys svn mkdir pulsar-client-python-X.Y.Z-candidate-N && cd pulsar-client-python-X.Y.Z-candidate-N # PROJECT_DIR is the directory of the pulsar-client-python repository $PROJECT_DIR/build-support/stage-release.sh X.Y.Z-candidate-N $WORKFLOW_ID svn add * svn ci -m "Staging artifacts and signature for Python client X.Y.Z-candidate-N" ``` ## Start the vote for the candidate Send an email to dev@pulsar.apache.org to start the vote for the candidate: ``` To: dev@pulsar.apache.org Subject: [VOTE] Pulsar Client Python Release X.Y.Z Candidate N This is the Nth release candidate for Apache Pulsar Client Python, version X.Y.Z. It fixes the following issues: https://github.com/apache/pulsar-client-python/milestone/?closed=1 *** Please download, test and vote on this release. This vote will stay open for at least 72 hours *** Python wheels: https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-python-X.Y.Z-candidate-N/ The supported python versions are 3.10, 3.11, 3.12, 3.13 and 3.14. The supported platforms and architectures are: - Windows x86_64 (windows/) - glibc-based Linux x86_64 (linux-glibc-x86_64/) - glibc-based Linux arm64 (linux-glibc-arm64/) - musl-based Linux x86_64 (linux-musl-x86_64/) - musl-based Linux arm64 (linux-musl-arm64/) - macOS universal 2 (macos/) You can download the wheel (the `.whl` file) according to your own OS and Python version and install the wheel: - Windows: `py -m pip install *.whl --force-reinstall` - Linux or macOS: `python3 -m pip install *.whl --force-reinstall` The tag to be voted upon: vX.Y.Z-candidate-N () https://github.com/apache/pulsar-client-python/releases/tag/vX.Y.Z-candidate-N Pulsar's KEYS file containing PGP keys you use to sign the release: https://downloads.apache.org/pulsar/KEYS Please download the Python wheels and follow the README to test. ``` Once there are at least 3 binding +1s, the vote will be ready to close and you can continue the following steps. If there is something wrong with the candidate, you need to fix it and repeat the steps from the **Cut the next candidate** section again. ## Move main branch to next version You need to move the main version to the next iteration. ```bash git checkout main perl -pi -e "s/X.Y.0a1/X.Y+1.0a1/" pulsar/__about__.py git add pulsar/__about__.py git commit -m "Bumped version to X.Y.0a1" ``` Since this needs to be merged into main, you need to follow the regular process and create a Pull Request on GitHub. ## Promote the release Ask a PMC member to promote the release: ```bash svn move -m "Release Apache Pulsar Client Python X.Y.Z" \ https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-python-X.Y.Z-candidate-N \ https://dist.apache.org/repos/dist/release/pulsar/pulsar-client-python-X.Y.Z ``` ## Upload the wheels to PyPI 1. You need to create an account on PyPI: https://pypi.org/account/register/ 2. Ask anyone that has been a release manager before to add you as a maintainer for [pulsar-client](https://pypi.org/manage/project/pulsar-client/releases/) on PyPI 3. PyPI has discontinued the use of plain usernames and passwords for login. It is now necessary to generate and obtain an API token for PyPI. Please visit pypi.org, navigate to "Account Settings", then "API tokens" to generate your API token. Remember to copy it. 4. Once you have completed the following steps in this section, you can check if the wheels are uploaded successfully in Download files. Remember to switch to the correct version in Release history. Then, upload the wheels to PyPI: ```bash sudo python3 -m pip install twine ./build-support/upload-pypi.sh https://dist.apache.org/repos/dist/release/pulsar/pulsar-client-python-X.Y.Z ``` Please ensure an API token is available to upload the wheels because `twine upload` requires an API token. If you don't have an available API token, navigate to https://pypi.org/manage/account/token/ to generate a new token. To verify the wheels have been uploaded successfully, you can try installing the wheel: ```bash python3 -m pip install pulsar-client==X.Y.Z ``` ## Write release notes Push the official tag: ```bash git checkout vX.Y.Z-candidate-N git tag -u $USER@apache.org vX.Y.Z -m "Release vX.Y.Z" git push origin vX.Y.Z ``` Then, [create a release](https://github.com/apache/pulsar-client-python/releases/new). Choose the `vX.Y.Z` tag and click the `Generate release notes` button to generate the release note automatically. Here is an example release note: https://github.com/apache/pulsar-client-python/releases/tag/v3.0.0 Then, create a PR in [`pulsar-site`](https://github.com/apache/pulsar-site) repo to update the website. Here is an example: https://github.com/apache/pulsar-site/pull/761 ## Generate the API documents For minor releases, skip this section. For major releases, you should generate the HTML files into the [`pulsar-site`](https://github.com/apache/pulsar-site) repo: ```bash # Use the first two version numbers, e.g. export VERSION=3.2 VERSION=X.Y # You need to install the wheel to have the _pulsar.so installed # It's better to run the following commands in an empty directory python3 -m pip install pulsar-client==$VERSION.0 --force-reinstall C_MODULE_PATH=$(python3 -c 'import _pulsar, os; print(_pulsar.__file__)') git clone git@github.com:apache/pulsar-client-python.git cd pulsar-client-python git checkout v$VERSION.0 # You can skip this step if you already have the `pulsar-site` repository in your local env. # In this case, you only need to modify the `--html-output` parameter in the following command. git clone git@github.com:apache/pulsar-site.git sudo python3 -m pip install pydoctor pydoctor --make-html \ --html-viewsource-base=https://github.com/apache/pulsar-client-python/tree/v$VERSION.0 \ --docformat=numpy --theme=readthedocs \ --intersphinx=https://docs.python.org/3/objects.inv \ --html-output=./pulsar-site/static/api/python/$VERSION.x \ --introspect-c-modules \ $C_MODULE_PATH \ pulsar cd pulsar-site git checkout -b py-docs-$VERSION git add . git commit -m "Generate Python client $VERSION.0 doc" git push origin py-docs-$VERSION ``` Then open a PR like: https://github.com/apache/pulsar-site/pull/600 ## Announce the release Use your Apache email account (`user@apache.org`) to send an email like: ``` To: dev@pulsar.apache.org, users@pulsar.apache.org, announce@apache.org Subject: [ANNOUNCE] Apache Pulsar Client Python X.Y.Z released The Apache Pulsar team is proud to announce Apache Pulsar Client Python version X.Y.Z. Pulsar is a highly scalable, low latency messaging platform running on commodity hardware. It provides simple pub-sub semantics over topics, guaranteed at-least-once delivery of messages, automatic cursor management for subscribers, and cross-datacenter replication. You can download the source code and the Python wheels in: https://archive.apache.org/dist/pulsar/pulsar-client-python-X.Y.Z/ The Python wheels were uploaded to PyPI as well so that they can be installed by `pip install pulsar-client==X.Y.Z`. Release Notes are at: https://pulsar.apache.org/release-notes/versioned/client-python-X.Y.Z/ We would like to thank the contributors that made the release possible. Regards, The Pulsar Team ``` Send the email in plain text mode since the announce@apache.org mailing list will reject messages with text/html content. In Gmail, there's an option to set Plain text mode in the ⋮/ More options menu. pulsar-client-python-3.12.0-candidate-0/SECURITY.md000066400000000000000000000027461520101105200215320ustar00rootroot00000000000000# Security Policy ## Security Vulnerability Process The Pulsar community follows the ASF [security vulnerability handling process](https://apache.org/security/#vulnerability-handling). To report a new vulnerability you have discovered, please follow the [ASF security vulnerability reporting process](https://apache.org/security/#reporting-a-vulnerability). To report a vulnerability for Pulsar, contact the [Apache Security Team](https://www.apache.org/security/). When reporting a vulnerability to [security@apache.org](mailto:security@apache.org), you can copy your email to [private@pulsar.apache.org](mailto:private@pulsar.apache.org) to send your report to the Apache Pulsar Project Management Committee. This is a private mailing list. It is the responsibility of the security vulnerability handling project team (Apache Pulsar PMC in most cases) to make public security vulnerability announcements. You can follow announcements on the [users@pulsar.apache.org](mailto:users@pulsar.apache.org) mailing list. For instructions on how to subscribe, please see https://pulsar.apache.org/contact/. ## Security Policy details and supported versions of Apache Pulsar The security policy and supported versions are outlined on the Pulsar website under [Security > Security Policy and Supported Versions](https://pulsar.apache.org/docs/security-policy-and-supported-versions/). ## Security Advisories Please visit the [Security Advisories](https://github.com/apache/pulsar/wiki/Security-advisories) page.pulsar-client-python-3.12.0-candidate-0/build-support/000077500000000000000000000000001520101105200225415ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/build-support/clang_format_exclusions.txt000066400000000000000000000014251520101105200302140ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # pulsar-client-python-3.12.0-candidate-0/build-support/dep-url.sh000066400000000000000000000033751520101105200244550ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # pulsar_cpp_base_url() { if [[ $# -ne 1 ]]; then echo "Usage: pulsar_cpp_base_url " exit 1 fi VERSION=$1 echo "https://archive.apache.org/dist/pulsar/pulsar-client-cpp-${VERSION}" } download_dependency() { if [[ $# -ne 2 ]]; then echo "Usage: download_dependency " exit 1 fi DEP_FILE=$1 DEP=$2 # Here we don't use read command to make it available in Alpine VERSION=$(grep $DEP $DEP_FILE | sed 's/://' | awk '{print $2}') case $DEP in "pulsar-cpp") URL=$(pulsar_cpp_base_url $VERSION)/apache-pulsar-client-cpp-${VERSION}.tar.gz ;; "pybind11") URL=https://github.com/pybind/pybind11/archive/refs/tags/v${VERSION}.tar.gz ;; "openssl") URL=https://github.com/openssl/openssl/archive/OpenSSL_$(echo $VERSION | sed 's/\./_/g').tar.gz ;; *) echo "Unknown dependency $DEP for version $VERSION" exit 1 esac curl -O -L $URL tar zxf $(basename $URL) } pulsar-client-python-3.12.0-candidate-0/build-support/dep-version.py000077500000000000000000000022271520101105200253540ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import yaml, sys if len(sys.argv) < 2: print(f'''Usage: {sys.argv[0]} dependency-name [dependency-file] The dependency file is "dependencies.yaml" by default.''') sys.exit(1) if len(sys.argv) > 2: dependency_file = sys.argv[2] else: dependency_file = 'dependencies.yaml' deps = yaml.safe_load(open(dependency_file)) print(deps[sys.argv[1]]) pulsar-client-python-3.12.0-candidate-0/build-support/docker-compose-pulsar-oauth2.yml000066400000000000000000000050301520101105200307000ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # version: '3' networks: pulsar: driver: bridge services: standalone: image: apachepulsar/pulsar:latest container_name: standalone hostname: local restart: "no" networks: - pulsar environment: - metadataStoreUrl=zk:localhost:2181 - clusterName=standalone-oauth2 - advertisedAddress=localhost - advertisedListeners=external:pulsar://localhost:6650 - PULSAR_MEM=-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m - PULSAR_PREFIX_authenticationEnabled=true - PULSAR_PREFIX_authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken - PULSAR_PREFIX_tokenPublicKey=data:;base64,MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2tZd/4gJda3U2Pc3tpgRAN7JPGWx/Gn17v/0IiZlNNRbP/Mmf0Vc6G1qsnaRaWNWOR+t6/a6ekFHJMikQ1N2X6yfz4UjMc8/G2FDPRmWjA+GURzARjVhxc/BBEYGoD0Kwvbq/u9CZm2QjlKrYaLfg3AeB09j0btNrDJ8rBsNzU6AuzChRvXj9IdcE/A/4N/UQ+S9cJ4UXP6NJbToLwajQ5km+CnxdGE6nfB7LWHvOFHjn9C2Rb9e37CFlmeKmIVFkagFM0gbmGOb6bnGI8Bp/VNGV0APef4YaBvBTqwoZ1Z4aDHy5eRxXfAMdtBkBupmBXqL6bpd15XRYUbu/7ck9QIDAQAB - PULSAR_PREFIX_brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 - PULSAR_PREFIX_brokerClientAuthenticationParameters={"issuerUrl":"https://dev-kt-aa9ne.us.auth0.com","audience":"https://dev-kt-aa9ne.us.auth0.com/api/v2/","privateKey":"data:application/json;base64,ewogICAgICAgICAgICAiY2xpZW50X2lkIjoiWGQyM1JIc1VudlVsUDd3Y2hqTllPYUlmYXpnZUhkOXgiLAogICAgICAgICAgICAiY2xpZW50X3NlY3JldCI6InJUN3BzN1dZOHVoZFZ1QlRLV1prdHR3TGRRb3RtZEVsaWFNNXJMZm1nTmlidnF6aVotZzA3Wkg1Mk5fcG9HQWIiCiAgICAgICAgfQ=="} ports: - "6650:6650" - "8080:8080" command: bash -c "bin/apply-config-from-env.py conf/standalone.conf && exec bin/pulsar standalone -nss -nfw" pulsar-client-python-3.12.0-candidate-0/build-support/install-dependencies.sh000077500000000000000000000046341520101105200272010ustar00rootroot00000000000000#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e -x cd `dirname $0` python3 -m pip install pyyaml CPP_CLIENT_VERSION=$(./dep-version.py pulsar-cpp ../dependencies.yaml) PYBIND11_VERSION=$(./dep-version.py pybind11 ../dependencies.yaml) source ./dep-url.sh if [ $USER != "root" ]; then SUDO="sudo" fi # Get the flavor of Linux export $(cat /etc/*-release | grep "^ID=") cd /tmp # Fetch the client binaries BASE_URL=$(pulsar_cpp_base_url ${CPP_CLIENT_VERSION}) UNAME_ARCH=$(uname -m) if [ $UNAME_ARCH == 'aarch64' ]; then PLATFORM=arm64 else PLATFORM=x86_64 fi if [ $ID == 'ubuntu' ]; then curl -L -O ${BASE_URL}/deb-${PLATFORM}/apache-pulsar-client.deb curl -L -O ${BASE_URL}/deb-${PLATFORM}/apache-pulsar-client-dev.deb $SUDO apt install -y /tmp/*.deb elif [ $ID == 'alpine' ]; then curl -L -O ${BASE_URL}/apk-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-${CPP_CLIENT_VERSION}-r0.apk curl -L -O ${BASE_URL}/apk-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-dev-${CPP_CLIENT_VERSION}-r0.apk $SUDO apk add --allow-untrusted /tmp/*.apk elif [ $ID == '"centos"' ]; then curl -L -O ${BASE_URL}/rpm-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-${CPP_CLIENT_VERSION}-1.${UNAME_ARCH}.rpm curl -L -O ${BASE_URL}/rpm-${PLATFORM}/${UNAME_ARCH}/apache-pulsar-client-devel-${CPP_CLIENT_VERSION}-1.${UNAME_ARCH}.rpm $SUDO rpm -i /tmp/*.rpm else echo "Unknown Linux distribution: '$ID'" exit 1 fi curl -L -O https://github.com/pybind/pybind11/archive/refs/tags/v${PYBIND11_VERSION}.tar.gz tar zxf v${PYBIND11_VERSION}.tar.gz $SUDO cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 /usr/include/ rm -rf pybind11-${PYBIND11_VERSION} v${PYBIND11_VERSION}.tar.gz pulsar-client-python-3.12.0-candidate-0/build-support/pulsar-test-service-start.sh000077500000000000000000000026711520101105200301620ustar00rootroot00000000000000#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e SRC_DIR=$(git rev-parse --show-toplevel) cd $SRC_DIR ./build-support/pulsar-test-service-stop.sh CONTAINER_ID=$(docker run -i --user $(id -u) -p 8080:8080 -p 6650:6650 -p 8443:8443 -p 6651:6651 --rm --detach apachepulsar/pulsar:4.0.0 sleep 3600) echo $CONTAINER_ID > .tests-container-id.txt docker cp tests/test-conf $CONTAINER_ID:/pulsar/test-conf docker cp build-support/start-test-service-inside-container.sh $CONTAINER_ID:start-test-service-inside-container.sh docker exec -i $CONTAINER_ID /start-test-service-inside-container.sh docker cp $CONTAINER_ID:/pulsar/data/tokens/token.txt tests/.test-token.txt echo "-- Ready to start tests" pulsar-client-python-3.12.0-candidate-0/build-support/pulsar-test-service-stop.sh000077500000000000000000000020471520101105200300070ustar00rootroot00000000000000#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e SRC_DIR=$(git rev-parse --show-toplevel) cd $SRC_DIR CONTAINER_ID_PATH=".tests-container-id.txt" if [ -f ${CONTAINER_ID_PATH} ]; then CONTAINER_ID=$(cat $CONTAINER_ID_PATH) docker kill $CONTAINER_ID || true rm .tests-container-id.txt fi pulsar-client-python-3.12.0-candidate-0/build-support/run_clang_format.py000077500000000000000000000052171520101105200264430ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # Original: https://github.com/apache/arrow/blob/4dbce607d50031a405af39d36e08cd03c5ffc764/cpp/build-support/run_clang_format.py # ChangeLog: # 2018-01-08: Accept multiple source directories (@Licht-T) import fnmatch import os import subprocess import sys if len(sys.argv) < 5: sys.stderr.write("Usage: %s $CLANG_FORMAT $CHECK_FORMAT exclude_globs.txt " "$source_dir1 $source_dir2\n" % sys.argv[0]) sys.exit(1) CLANG_FORMAT = sys.argv[1] CHECK_FORMAT = int(sys.argv[2]) == 1 EXCLUDE_GLOBS_FILENAME = sys.argv[3] SOURCE_DIRS = sys.argv[4:] exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")] files_to_format = [] matches = [] for source_dir in SOURCE_DIRS: for directory, subdirs, files in os.walk(source_dir): for name in files: name = os.path.join(directory, name) if not (name.endswith('.h') or name.endswith('.cc')): continue excluded = False for g in exclude_globs: if fnmatch.fnmatch(name, g): excluded = True break if not excluded: files_to_format.append(name) if CHECK_FORMAT: output = subprocess.check_output([CLANG_FORMAT, '-output-replacements-xml'] + files_to_format, stderr=subprocess.STDOUT).decode('utf8') to_fix = [] for line in output.split('\n'): if 'offset' in line: to_fix.append(line) if len(to_fix) > 0: print("clang-format checks failed, run 'make format' to fix") sys.exit(-1) else: try: cmd = [CLANG_FORMAT, '-i'] + files_to_format subprocess.check_output(cmd, stderr=subprocess.STDOUT) except Exception as e: print(e) print(' '.join(cmd)) raisepulsar-client-python-3.12.0-candidate-0/build-support/stage-release.sh000077500000000000000000000047221520101105200256260ustar00rootroot00000000000000#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e if [[ $# -lt 2 ]]; then echo "Usage: $0 \$VERSION \$WORKFLOW_ID" exit 1 fi if [[ ! $GITHUB_TOKEN ]]; then echo "GITHUB_TOKEN (must have the actions scope) is not set" exit 2 fi VERSION=$1 TAG=v$VERSION WORKFLOW_ID=$2 # Download the source tar curl -O -L https://github.com/apache/pulsar-client-python/archive/refs/tags/$TAG.tar.gz # Remove the "-candidate-N" suffix VERSION=$(echo $VERSION | sed 's/-candidate-.*//') mv $TAG.tar.gz pulsar-client-python-$VERSION.tar.gz # Download the Python wheels URLS=$(curl -L https://api.github.com/repos/apache/pulsar-client-python/actions/runs/$WORKFLOW_ID/artifacts \ | jq -r '.artifacts[] | select(((.name // "") | contains("dockerbuild")) | not) | .archive_download_url') for URL in $URLS; do curl -O -L "$URL" -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" unzip -q zip rm -f zip done sign() { FILE=$1 gpg --armor --output $FILE.asc --detach-sig $FILE shasum -a 512 $FILE > $FILE.sha512 } export GPG_TTY=$(tty) set -x for WHEEL in $(ls *.whl); do sign $WHEEL done sign pulsar-client-python-$VERSION.tar.gz mkdir windows && cd windows mv ../pulsar_client*win*.whl* . && cd .. mkdir macos && cd macos mv ../pulsar_client*macos*.whl* . && cd .. mkdir linux-glibc-x86_64 && cd linux-glibc-x86_64 mv ../pulsar_client*manylinux*x86_64.whl* . && cd .. mkdir linux-glibc-arm64 && cd linux-glibc-arm64 mv ../pulsar_client*manylinux*aarch64.whl* . && cd .. mkdir linux-musl-x86_64 && cd linux-musl-x86_64 mv ../pulsar_client*musllinux*x86_64.whl* . && cd .. mkdir linux-musl-arm64 && cd linux-musl-arm64 mv ../pulsar_client*musllinux*aarch64.whl* . && cd .. pulsar-client-python-3.12.0-candidate-0/build-support/start-test-service-inside-container.sh000077500000000000000000000066211520101105200321060ustar00rootroot00000000000000#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e -x export PULSAR_EXTRA_OPTS=-Dpulsar.auth.basic.conf=test-conf/.htpasswd # Show logs immediately for debugging sed -i.bak 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml # Generate secret key and token mkdir -p data/tokens bin/pulsar tokens create-secret-key --output data/tokens/secret.key bin/pulsar tokens create \ --subject token-principal \ --secret-key file:///pulsar/data/tokens/secret.key \ > /pulsar/data/tokens/token.txt # Unset the HTTP proxy to avoid the REST requests being affected export http_proxy= TOKEN=$(bin/pulsar tokens create \ --subject superUser \ --secret-key file:///pulsar/data/tokens/secret.key) put() { curl -H "Authorization: Bearer $TOKEN" \ -L http://localhost:8080/admin/v2/$1 \ -H 'Content-Type: application/json' \ -X PUT \ -d $(echo $2 | sed 's/ //g') } export PULSAR_STANDALONE_CONF=test-conf/standalone-ssl.conf export PULSAR_PID_DIR=/tmp sed -i 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml bin/pulsar-daemon start standalone \ --no-functions-worker --no-stream-storage \ --bookkeeper-dir data/bookkeeper echo "-- Wait for Pulsar service to be ready" until curl http://localhost:8080/metrics > /dev/null 2>&1 ; do sleep 1; done echo "-- Pulsar service is ready -- Configure permissions" export PULSAR_CLIENT_CONF=test-conf/client-ssl.conf # Create "standalone" cluster if it does not exist put clusters/standalone '{ "serviceUrl": "http://localhost:8080/", "serviceUrlTls": "https://localhost:8443/", "brokerServiceUrl": "pulsar://localhost:6650/", "brokerServiceUrlTls": "pulsar+ssl://localhost:6651/" }' # Update "public" tenant put tenants/public '{ "adminRoles": ["anonymous"], "allowedClusters": ["standalone"] }' # Update "public/default" with no auth required put namespaces/public/default '{ "auth_policies": { "namespace_auth": { "anonymous": ["produce", "consume"] } }, "replication_clusters": ["standalone"] }' # Create "public/default-2" with no auth required put namespaces/public/default-2 '{ "auth_policies": { "namespace_auth": { "anonymous": ["produce", "consume"] } }, "replication_clusters": ["standalone"] }' # Create "private" tenant put tenants/private '{ "adminRoles": [], "allowedClusters": ["standalone"] }' # Create "private/auth" with required authentication put namespaces/private/auth '{ "auth_policies": { "namespace_auth": { "token-principal": ["produce", "consume"] } }, "replication_clusters": ["standalone"] }' echo "-- Ready to start tests" pulsar-client-python-3.12.0-candidate-0/build-support/upload-pypi.sh000077500000000000000000000026271520101105200253520ustar00rootroot00000000000000#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e if [[ $# -lt 1 ]]; then echo "Usage: $0 BASE_URL" exit 1 fi BASE_URL=$1 download_wheels() { URL=$BASE_URL/$1 LIST=$(curl -L $URL | grep ".whl<" | sed 's/^.*>pulsar\(.*\)\.whl.*/pulsar\1.whl/') for WHEEL in $LIST; do if [[ ! -f $WHEEL ]]; then echo "Download $URL/$WHEEL" curl -O -L $URL/$WHEEL fi done } download_wheels linux-glibc-arm64 download_wheels linux-glibc-x86_64 download_wheels linux-musl-arm64 download_wheels linux-musl-x86_64 download_wheels macos download_wheels windows twine upload pulsar_client-*.whl rm -f pulsar_client-*.whl pulsar-client-python-3.12.0-candidate-0/cmake_modules/000077500000000000000000000000001520101105200225405ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/cmake_modules/FindClangTools.cmake000066400000000000000000000066441520101105200264220ustar00rootroot00000000000000# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Tries to find the clang-tidy and clang-format modules # # Usage of this module as follows: # # find_package(ClangTools) # # Variables used by this module, they can change the default behaviour and need # to be set before calling find_package: # # ClangToolsBin_HOME - # When set, this path is inspected instead of standard library binary locations # to find clang-tidy and clang-format # # This module defines # CLANG_TIDY_BIN, The path to the clang tidy binary # CLANG_TIDY_FOUND, Whether clang tidy was found # CLANG_FORMAT_BIN, The path to the clang format binary # CLANG_TIDY_FOUND, Whether clang format was found list(APPEND CLANG_SEARCH_PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin /opt/homebrew/bin) if (WIN32) list(APPEND CLANG_SEARCH_PATHS "C:/Program Files/LLVM/bin" "C:/Program Files (x86)/LLVM/bin") endif() find_program(CLANG_TIDY_BIN NAMES clang-tidy-4.0 clang-tidy-3.9 clang-tidy-3.8 clang-tidy-3.7 clang-tidy-3.6 clang-tidy PATHS ${CLANG_SEARCH_PATHS} NO_DEFAULT_PATH ) if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" ) set(CLANG_TIDY_FOUND 0) message("clang-tidy not found") else() set(CLANG_TIDY_FOUND 1) message("clang-tidy found at ${CLANG_TIDY_BIN}") endif() if (CLANG_FORMAT_VERSION) find_program(CLANG_FORMAT_BIN NAMES clang-format-${CLANG_FORMAT_VERSION} PATHS ${CLANG_SEARCH_PATHS} NO_DEFAULT_PATH ) # If not found yet, search alternative locations if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE) # Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/ STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}") STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}") if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0") find_program(CLANG_FORMAT_BIN NAMES clang-format PATHS /usr/local/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin NO_DEFAULT_PATH ) else() find_program(CLANG_FORMAT_BIN NAMES clang-format PATHS /usr/local/opt/llvm@${CLANG_FORMAT_VERSION}/bin NO_DEFAULT_PATH ) endif() endif() else() find_program(CLANG_FORMAT_BIN NAMES clang-format-5 clang-format-5.0 clang-format PATHS ${CLANG_SEARCH_PATHS} NO_DEFAULT_PATH ) endif() if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" ) set(CLANG_FORMAT_FOUND 0) message("clang-format not found") else() set(CLANG_FORMAT_FOUND 1) message("clang-format found at ${CLANG_FORMAT_BIN}") endif() pulsar-client-python-3.12.0-candidate-0/dependencies.yaml000066400000000000000000000016171520101105200232470ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # pulsar-cpp: 4.1.0 pybind11: 3.0.1 # The OpenSSL dependency is only used when building Python from source openssl: 1.1.1q pulsar-client-python-3.12.0-candidate-0/examples/000077500000000000000000000000001520101105200215465ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/examples/async.py000066400000000000000000000045751520101105200232500ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # NOTE: requires version 3.9.0+ of pulsar-client library import asyncio from pulsar.asyncio import Client, Producer, Consumer from pulsar import BatchingType NUM_MESSAGES = 100 TOPIC_NAME = 'my-async-topic' SUBSCRIPTION_NAME = 'my-async-subscription' SERVICE_URL = 'pulsar://localhost:6650' async def produce(producer: Producer, id: int) -> None: await producer.send((f'hello-{id}').encode('utf-8'), None) await producer.flush() async def consume(consumer: Consumer) -> None: msg = await consumer.receive() print("Received message '{0}' id='{1}'".format(msg.data().decode('utf-8'), msg.message_id())) await consumer.acknowledge(msg) async def main() -> None: client: Client = Client(SERVICE_URL) consumer = await client.subscribe(TOPIC_NAME, SUBSCRIPTION_NAME, properties={ "consumer-name": "test-consumer-name", "consumer-id": "test-consumer-id" }) producer = await client.create_producer( TOPIC_NAME, block_if_queue_full=True, batching_enabled=True, batching_max_publish_delay_ms=10, properties={ "producer-name": "test-producer-name", "producer-id": "test-producer-id" }, batching_type=BatchingType.KeyBased ) tasks = [] for id in range(NUM_MESSAGES): tasks.append(asyncio.create_task(produce(producer, id))) tasks.append(asyncio.create_task(consume(consumer))) await asyncio.gather(*tasks) await producer.close() await consumer.close() await client.close() if __name__ == '__main__': asyncio.run(main()) pulsar-client-python-3.12.0-candidate-0/examples/company.avsc000066400000000000000000000014441520101105200240750ustar00rootroot00000000000000{ "doc": "this is doc", "namespace": "example.avro", "type": "record", "name": "Company", "fields": [ {"name": "name", "type": ["null", "string"]}, {"name": "address", "type": ["null", "string"]}, {"name": "employees", "type": ["null", {"type": "array", "items": { "type": "record", "name": "Employee", "fields": [ {"name": "name", "type": ["null", "string"]}, {"name": "age", "type": ["null", "int"]} ] }}]}, {"name": "labels", "type": ["null", {"type": "map", "values": "string"}]}, {"name": "companyType", "type": ["null", {"type": "enum", "name": "CompanyType", "symbols": ["companyType1", "companyType2", "companyType3"]}]} ] }pulsar-client-python-3.12.0-candidate-0/examples/consumer.py000077500000000000000000000026401520101105200237600ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import pulsar client = pulsar.Client('pulsar://localhost:6650') consumer = client.subscribe('my-topic', "my-subscription", properties={ "consumer-name": "test-consumer-name", "consumer-id": "test-consumer-id" }) while True: try: msg = consumer.receive() print("Received message '{0}' id='{1}'".format(msg.data().decode('utf-8'), msg.message_id())) consumer.acknowledge(msg) except pulsar.Interrupted: print("Stop receiving messages") break client.close() pulsar-client-python-3.12.0-candidate-0/examples/producer.py000077500000000000000000000030351520101105200237470ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from pulsar import BatchingType import pulsar client = pulsar.Client('pulsar://localhost:6650') producer = client.create_producer( 'my-topic', block_if_queue_full=True, batching_enabled=True, batching_max_publish_delay_ms=10, properties={ "producer-name": "test-producer-name", "producer-id": "test-producer-id" }, batching_type=BatchingType.KeyBased ) for i in range(10): try: producer.send('hello'.encode('utf-8'), None) except Exception as e: print("Failed to send message: %s", e) producer.flush() producer.close() pulsar-client-python-3.12.0-candidate-0/examples/rpc_client.py000077500000000000000000000047661520101105200242620ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import pulsar import threading import uuid DEFAULT_CLIENT_TOPIC = 'rpc-client-topic' DEFAULT_SERVER_TOPIC = 'rpc-server-topic' UUID = str(uuid.uuid4()) NUM_CLIENT = 0 LOCK = threading.Lock() class RPCClient(object): def __init__(self, client_topic=DEFAULT_CLIENT_TOPIC, server_topic=DEFAULT_SERVER_TOPIC): self.client_topic = client_topic self.server_topic = server_topic global NUM_CLIENT with LOCK: self.client_no = NUM_CLIENT NUM_CLIENT += 1 self.response = None self.partition_key = '{0}_{1}'.format(UUID, self.client_no) self.client = pulsar.Client('pulsar://localhost:6650') self.producer = self.client.create_producer(server_topic) self.consumer = \ self.client.subscribe(client_topic, 'rpc-client-{}'.format(self.partition_key), message_listener=self.on_response) self.consumer.resume_message_listener() def on_response(self, consumer, message): if message.partition_key() == self.partition_key \ and consumer.topic() == self.client_topic: msg = message.data().decode('utf-8') print('Received: {0}'.format(msg)) self.response = msg consumer.acknowledge(message) def call(self, message): self.response = None self.producer.send(message.encode('utf-8'), partition_key=self.partition_key) while self.response is None: pass return self.response msg = 'foo' rpc_client = RPCClient() ret = rpc_client.call(msg) print('RPCClient message sent: {0}, result: {1}'.format(msg, ret)) pulsar-client-python-3.12.0-candidate-0/examples/rpc_server.py000077500000000000000000000041031520101105200242730ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import pulsar DEFAULT_CLIENT_TOPIC = 'rpc-client-topic' DEFAULT_SERVER_TOPIC = 'rpc-server-topic' class RPCServer(object): def __init__(self, client_topic=DEFAULT_CLIENT_TOPIC, server_topic=DEFAULT_SERVER_TOPIC): self.client_topic = client_topic self.server_topic = server_topic self.client = pulsar.Client('pulsar://localhost:6650') self.producer = self.client.create_producer(client_topic) self.consumer = \ self.client.subscribe(server_topic, 'rpc-server', pulsar.ConsumerType.Shared, message_listener=self.on_response) def on_response(self, consumer, message): print('Received from {0}: {1}'.format(message.partition_key(), message.data().decode('utf-8'))) self.producer.send('{} bar'.format(message.data().decode('utf-8')), partition_key=message.partition_key()) consumer.acknowledge(message) def start(self): self.consumer.resume_message_listener() rpc_server = RPCServer() rpc_server.start() try: while True: pass except KeyboardInterrupt: print('Interrupted.') pulsar-client-python-3.12.0-candidate-0/pkg/000077500000000000000000000000001520101105200205115ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/pkg/build-wheel-inside-docker.sh000077500000000000000000000060021520101105200257650ustar00rootroot00000000000000#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e -x cd /pulsar-client-python ROOT_DIR=$PWD source build-support/dep-url.sh # Build cpp wheels if [[ $ARCH == "aarch64" ]]; then export VCPKG_FORCE_SYSTEM_BINARIES=1 fi PULSAR_CPP_VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}') if [ $CPP_BINARY_TYPE == "rpm" ]; then # The pre-built RPM packages have incompatible ABI with manylinux, so we have to build from source download_dependency ./dependencies.yaml pulsar-cpp cd apache-pulsar-client-cpp-${PULSAR_CPP_VERSION} git clone https://github.com/microsoft/vcpkg.git cd vcpkg # manylinux does not have ninja in the system package manager git clone https://github.com/ninja-build/ninja.git cd ninja git checkout release ./configure.py --bootstrap mv ninja /usr/bin/ cd .. ./bootstrap-vcpkg.sh cd .. if [ $PULSAR_CPP_VERSION == "3.7.0" ]; then patch lib/CMakeLists.txt $ROOT_DIR/pkg/manylinux/pulsar-client-cpp-3.7.0.patch fi cmake -B build-cpp -DINTEGRATE_VCPKG=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTS=OFF -DBUILD_DYNAMIC_LIB=ON -DBUILD_STATIC_LIB=ON cmake --build build-cpp -j8 --target install cd .. rm -rf apache-pulsar-client-cpp-$(PULSAR_CPP_VERSION) else # apk if [ $ARCH == "aarch64" ]; then APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-arm64/aarch64 else APK_ROOT_DIR=$(pulsar_cpp_base_url $PULSAR_CPP_VERSION)/apk-x86_64/x86_64 fi curl -O -L $APK_ROOT_DIR/apache-pulsar-client-$PULSAR_CPP_VERSION-r0.apk curl -O -L $APK_ROOT_DIR/apache-pulsar-client-dev-$PULSAR_CPP_VERSION-r0.apk apk add --allow-untrusted *.apk fi download_dependency $PWD/dependencies.yaml pybind11 rm -rf pybind11 mv pybind11-* pybind11 cmake -B build -DCMAKE_BUILD_TYPE=Release cmake --build build -j8 mv build/lib_pulsar.so . ./setup.py bdist_wheel # Audit wheel is required to convert a wheel that is tagged as generic # 'linux' into a 'multilinux' wheel. # Only wheel files tagged as multilinux can be uploaded to PyPI # Audit wheel will make sure no external dependencies are needed for # the shared library and that only symbols supported by most linux # distributions are used. auditwheel repair dist/pulsar_client*-$PYTHON_SPEC-linux_*.whl pulsar-client-python-3.12.0-candidate-0/pkg/mac/000077500000000000000000000000001520101105200212515ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/pkg/mac/build-mac-wheels.sh000077500000000000000000000133601520101105200247350ustar00rootroot00000000000000#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e -x if [[ $# -ne 2 ]]; then echo "Usage: $0 " exit 1 fi ROOT_DIR=$(git rev-parse --show-toplevel) cd "${ROOT_DIR}" source build-support/dep-url.sh CACHE_DIR=$ROOT_DIR/.pulsar-mac-build PREFIX=${CACHE_DIR}/install mkdir -p $PREFIX mkdir -p $PREFIX/lib/ if [ ! -f $PREFIX/lib/libpulsarwithdeps.a ]; then VERSION=$(cat ./dependencies.yaml | grep pulsar-cpp | awk '{print $2}') curl -O -L $(pulsar_cpp_base_url $VERSION)/macos-arm64.zip curl -O -L $(pulsar_cpp_base_url $VERSION)/macos-x86_64.zip unzip -q macos-arm64.zip -d arm64 unzip -q macos-x86_64.zip -d x86_64 libtool -static -o libpulsarwithdeps.a arm64/lib/libpulsarwithdeps.a x86_64/lib/libpulsarwithdeps.a mv arm64/include/ $PREFIX/ mv libpulsarwithdeps.a $PREFIX/lib/ rm -rf arm64/ x86_64/ macos-arm64.zip macos-x86_64.zip fi PYTHON_VERSION=$1 PYTHON_VERSION_LONG=$2 # When building Python from source, it will read this environment variable to determine the minimum supported macOS version export MACOSX_DEPLOYMENT_TARGET=13 pushd $CACHE_DIR # We need to build OpenSSL from source to have universal2 binaries OPENSSL_VERSION=$(cat $ROOT_DIR/dependencies.yaml | grep openssl | awk '{print $2}') OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') if [ ! -f openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done ]; then echo "Building OpenSSL" download_dependency $ROOT_DIR/dependencies.yaml openssl tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64 echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-arm64-cc make -j8 make install_sw popd tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz mv openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 pushd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64 echo -e "#include \n$(cat test/v3ext.c)" > test/v3ext.c CFLAGS="-fPIC -mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}" \ ./Configure --prefix=$PREFIX no-shared no-unit-test darwin64-x86_64-cc make -j8 make install_sw popd # Create universal binaries lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libssl.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libssl.a \ -output $PREFIX/lib/libssl.a lipo -create openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-arm64/libcrypto.a openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}-x86_64/libcrypto.a \ -output $PREFIX/lib/libcrypto.a touch openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.done else echo "Using cached OpenSSL" fi if [ ! -f Python-${PYTHON_VERSION_LONG}/.done ]; then echo "Building Python $PYTHON_VERSION_LONG" curl -O -L https://www.python.org/ftp/python/${PYTHON_VERSION_LONG}/Python-${PYTHON_VERSION_LONG}.tgz tar xfz Python-${PYTHON_VERSION_LONG}.tgz pushd Python-${PYTHON_VERSION_LONG} export CFLAGS="-fPIC -O3" ./configure --prefix=$PREFIX --enable-shared --enable-universalsdk --with-universal-archs=universal2 --with-openssl=$PREFIX make -j16 make install curl -O -L https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl export SSL_CERT_FILE=/etc/ssl/cert.pem $PREFIX/bin/pip3 install wheel setuptools $PREFIX/bin/pip3 install wheel-*.whl touch .done popd else echo "Using cached Python $PYTHON_VERSION_LONG" fi PYBIND11_VERSION=$(cat $ROOT_DIR/dependencies.yaml | grep pybind11 | awk '{print $2}') if [ ! -f pybind11/.done ]; then download_dependency $ROOT_DIR/dependencies.yaml pybind11 mkdir -p $PREFIX/include/ cp -rf pybind11-${PYBIND11_VERSION}/include/pybind11 $PREFIX/include/ mkdir -p pybind11 touch pybind11/.done fi popd # $CACHE_DIR PYTHON_CLIENT_VERSION=$(grep -v '^#' pulsar/__about__.py | cut -d "=" -f2 | sed "s/'//g") echo "Build wheel for Python $PYTHON_VERSION" PY_EXE=$PREFIX/bin/python3 PIP_EXE=$PREFIX/bin/pip3 ARCHS='arm64;x86_64' PIP_TAG='universal2' cmake -B build \ -DCMAKE_OSX_ARCHITECTURES=${ARCHS} \ -DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET} \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_PREFIX_PATH=$PREFIX \ -DLINK_STATIC=ON \ -DPython3_ROOT_DIR=$PREFIX cmake --build build --config Release -j8 cp -f build/lib_pulsar.so . $PY_EXE setup.py bdist_wheel PY_SPEC=$(echo $PYTHON_VERSION | sed 's/\.//g') cd /tmp $PIP_EXE install --no-dependencies --force-reinstall \ $ROOT_DIR/dist/pulsar_client-${PYTHON_CLIENT_VERSION}-cp$PY_SPEC-*-macosx*_${PIP_TAG}.whl $PY_EXE -c 'import pulsar' pulsar-client-python-3.12.0-candidate-0/pkg/manylinux/000077500000000000000000000000001520101105200225355ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/pkg/manylinux/Dockerfile000066400000000000000000000022451520101105200245320ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # ARG ARCH FROM quay.io/pypa/manylinux_2_28_${ARCH} ARG PYTHON_VERSION ARG PYTHON_SPEC ARG ARCH ENV CPP_BINARY_TYPE=rpm ENV PYTHON_VERSION=${PYTHON_VERSION} ENV PYTHON_SPEC=${PYTHON_SPEC} ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" ENV ARCH=${ARCH} RUN pip3 install setuptools # Dependencies for vcpkg on arm64 architecture RUN yum install -y curl zip unzip tar perl-IPC-Cmd perl-Time-Piece pulsar-client-python-3.12.0-candidate-0/pkg/manylinux/pulsar-client-cpp-3.7.0.patch000066400000000000000000000006751520101105200275730ustar00rootroot00000000000000--- lib/CMakeLists.txt +++ lib/CMakeLists.txt @@ -93,10 +93,6 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") target_link_options(pulsarShared PRIVATE -Wl,-Bsymbolic) endif () - check_cxx_symbol_exists(__GLIBCXX__ iostream GLIBCXX) - if (GLIBCXX) - target_link_libraries(pulsarShared PUBLIC -static-libgcc -static-libstdc++) - endif () endif() check_cxx_symbol_exists(getauxval sys/auxv.h HAVE_AUXV_GETAUXVAL) pulsar-client-python-3.12.0-candidate-0/pkg/manylinux_musl/000077500000000000000000000000001520101105200235755ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/pkg/manylinux_musl/Dockerfile000066400000000000000000000020641520101105200255710ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # ARG ARCH FROM quay.io/pypa/musllinux_1_2_${ARCH} ARG PYTHON_VERSION ARG PYTHON_SPEC ENV PYTHON_VERSION=${PYTHON_VERSION} ENV PYTHON_SPEC=${PYTHON_SPEC} ARG ARCH ENV ARCH=${ARCH} ENV PATH="/opt/python/${PYTHON_SPEC}/bin:${PATH}" RUN pip install setuptools ENV CPP_BINARY_TYPE="apk" pulsar-client-python-3.12.0-candidate-0/pkg/test-wheel.sh000077500000000000000000000017201520101105200231310ustar00rootroot00000000000000#!/usr/bin/env sh # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e -x cd / pip3 install /pulsar-client-python/wheelhouse/pulsar_client-*.whl # Load the wheel to ensure there are no linking problems python3 -c 'import pulsar' pulsar-client-python-3.12.0-candidate-0/pulsar/000077500000000000000000000000001520101105200212365ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/pulsar/__about__.py000066400000000000000000000014521520101105200235200ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # __version__='3.12.0' pulsar-client-python-3.12.0-candidate-0/pulsar/__init__.py000066400000000000000000002715551520101105200233660ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # """ The Pulsar Python client library is based on the existing C++ client library. All the same features are exposed through the Python interface. Currently, the supported Python versions are 3.10, 3.11, 3.12, 3.13 and 3.14. ================= Install from PyPI ================= Download Python wheel binary files for macOS and Linux directly from the PyPI archive: .. code-block:: shell sudo pip install pulsar-client ======================== Install from source code ======================== Read the instructions on `source code repository `_. """ import logging from typing import Callable, List, Tuple, Optional, Union import _pulsar from _pulsar import Result, CompressionType, ConsumerType, InitialPosition, PartitionsRoutingMode, BatchingType, \ LoggerLevel, BatchReceivePolicy, KeySharedPolicy, KeySharedMode, ProducerAccessMode, RegexSubscriptionMode, \ DeadLetterPolicyBuilder, ConsumerCryptoFailureAction # noqa: F401 from pulsar.__about__ import __version__ from pulsar.exceptions import * from pulsar.schema.schema import BytesSchema from pulsar.tableview import TableView from pulsar.functions.function import Function from pulsar.functions.context import Context from pulsar.functions.serde import SerDe, IdentitySerDe, PickleSerDe from pulsar import schema _schema = schema import re _retype = type(re.compile('x')) from datetime import timedelta class MessageId: """ Represents a message id. Attributes ---------- earliest: Represents the earliest message stored in a topic latest: Represents the latest message published on a topic """ def __init__(self, partition=-1, ledger_id=-1, entry_id=-1, batch_index=-1): self._msg_id: _pulsar.MessageId = _pulsar.MessageId(partition, ledger_id, entry_id, batch_index) earliest = _pulsar.MessageId.earliest latest = _pulsar.MessageId.latest def ledger_id(self): return self._msg_id.ledger_id() def entry_id(self): return self._msg_id.entry_id() def batch_index(self): return self._msg_id.batch_index() def partition(self): return self._msg_id.partition() def serialize(self): """ Returns a bytes representation of the message id. This byte sequence can be stored and later deserialized. """ return self._msg_id.serialize() def __str__(self) -> str: """ Returns the string representation of the message id. """ return str(self._msg_id) def __eq__(self, other) -> bool: return self._msg_id == other._msg_id def __ne__(self, other) -> bool: return self._msg_id != other._msg_id def __le__(self, other) -> bool: return self._msg_id <= other._msg_id def __lt__(self, other) -> bool: return self._msg_id < other._msg_id def __ge__(self, other) -> bool: return self._msg_id >= other._msg_id def __gt__(self, other) -> bool: return self._msg_id > other._msg_id @staticmethod def deserialize(message_id_bytes, topic: Optional[str] = None) -> _pulsar.MessageId: """ Deserialize a message id object from a previously serialized bytes sequence. Parameters ---------- topic: str, optional For multi-topics consumers, the topic name is required to deserialize the message id. .. code-block:: python msg = consumer.receive() topic = msg.topic_name() msg_id_bytes = msg.message_id().serialize() # Store topic and msg_id_bytes somewhere # Later, deserialize the message id msg_id = MessageId.deserialize(msg_id_bytes, topic=topic) """ msg_id = _pulsar.MessageId.deserialize(message_id_bytes) if topic is not None: msg_id.topic_name(topic) return msg_id @classmethod def wrap(cls, msg_id: _pulsar.MessageId): """ Wrap the underlying MessageId type from the C extension to the Python type. """ self = cls() self._msg_id = msg_id return self class EncryptionKey: """ The key used for encryption. """ def __init__(self, key: _pulsar.EncryptionKey): """ Create EncryptionKey instance. Parameters ---------- key: _pulsar.EncryptionKey The underlying EncryptionKey instance from the C extension. """ self._key = key @property def key(self) -> str: """ Returns the key, which is usually the key file's name. """ return self._key.key @property def value(self) -> bytes: """ Returns the value, which is usually the key bytes used for encryption. """ return self._key.value() @property def metadata(self) -> dict: """ Returns the metadata associated with the key. """ return self._key.metadata def __str__(self) -> str: return f"EncryptionKey(key={self.key}, value_len={len(self.value)}, metadata={self.metadata})" def __repr__(self) -> str: return self.__str__() class EncryptionContext: """ It contains encryption and compression information in it using which application can decrypt consumed message with encrypted-payload. """ def __init__(self, context: _pulsar.EncryptionContext): """ Create EncryptionContext instance. Parameters ---------- context: _pulsar.EncryptionContext The underlying EncryptionContext instance from the C extension. """ self._context = context def keys(self) -> List[EncryptionKey]: """ Returns all EncryptionKey instances when performing encryption. """ keys = self._context.keys() return [EncryptionKey(key) for key in keys] def param(self) -> bytes: """ Returns the encryption param bytes. """ return self._context.param() def algorithm(self) -> str: """ Returns the encryption algorithm. """ return self._context.algorithm() def compression_type(self) -> CompressionType: """ Returns the compression type of the message. """ return self._context.compression_type() def uncompressed_message_size(self) -> int: """ Returns the uncompressed message size or 0 if the compression type is NONE. """ return self._context.uncompressed_message_size() def batch_size(self) -> int: """ Returns the number of messages in the batch or -1 if the message is not batched. """ return self._context.batch_size() def is_decryption_failed(self) -> bool: """ Returns whether decryption has failed for this message. """ return self._context.is_decryption_failed() def __str__(self) -> str: return f"EncryptionContext(algorithm={self.algorithm()}, " \ f"compression_type={self.compression_type().name}, " \ f"uncompressed_message_size={self.uncompressed_message_size()}, " \ f"is_decryption_failed={self.is_decryption_failed()}, " \ f"keys=[{', '.join(str(key) for key in self.keys())}])" def __repr__(self) -> str: return self.__str__() class Message: """ Message objects are returned by a consumer, either by calling `receive` or through a listener. """ def data(self): """ Returns object typed bytes with the payload of the message. """ return self._message.data() def value(self): """ Returns object with the de-serialized version of the message content """ return self._schema.decode_message(self._message) def properties(self): """ Return the properties attached to the message. Properties are application-defined key/value pairs that will be attached to the message. """ return self._message.properties() def partition_key(self): """ Get the partitioning key for the message. """ return self._message.partition_key() def ordering_key(self): """ Get the ordering key for the message. """ return self._message.ordering_key() def publish_timestamp(self): """ Get the timestamp in milliseconds with the message publish time. """ return self._message.publish_timestamp() def event_timestamp(self): """ Get the timestamp in milliseconds with the message event time. """ return self._message.event_timestamp() def message_id(self) -> _pulsar.MessageId: """ The message ID that can be used to refer to this particular message. Returns ---------- A `_pulsar.MessageId` object that represents where the message is persisted. """ return self._message.message_id() def topic_name(self): """ Get the topic Name from which this message originated from """ return self._message.topic_name() def redelivery_count(self): """ Get the redelivery count for this message """ return self._message.redelivery_count() def schema_version(self): """ Get the schema version for this message """ return self._message.schema_version() def producer_name(self) -> str: """ Get the producer name which produced this message """ return self._message.producer_name() def encryption_context(self) -> EncryptionContext | None: """ Get the encryption context for this message or None if it's not encrypted. It should be noted that the result should not be accessed after the current Message instance is deleted. """ context = self._message.encryption_context() return None if context is None else EncryptionContext(context) @staticmethod def _wrap(_message): self = Message() self._schema = BytesSchema() self._message = _message return self class MessageBatch: def __init__(self): self._msg_batch = _pulsar.MessageBatch() def with_message_id(self, msg_id): if not isinstance(msg_id, _pulsar.MessageId): if isinstance(msg_id, MessageId): msg_id = msg_id._msg_id else: raise TypeError("unknown message id type") self._msg_batch.with_message_id(msg_id) return self def parse_from(self, data, size): self._msg_batch.parse_from(data, size) _msgs = self._msg_batch.messages() return list(map(Message._wrap, _msgs)) class Authentication: """ Authentication provider object. Used to load authentication from an external shared library. """ def __init__(self, dynamicLibPath, authParamsString): """ Create the authentication provider instance. Parameters ---------- dynamicLibPath: str Path to the authentication provider shared library (such as ``tls.so``) authParamsString: str Comma-separated list of provider-specific configuration params """ _check_type(str, dynamicLibPath, 'dynamicLibPath') _check_type(str, authParamsString, 'authParamsString') self.auth = _pulsar.Authentication.create(dynamicLibPath, authParamsString) class AuthenticationTLS(Authentication): """ TLS Authentication implementation """ def __init__(self, certificate_path, private_key_path): """ Create the TLS authentication provider instance. Parameters ---------- certificate_path: str Path to the public certificate private_key_path: str Path to private TLS key """ _check_type(str, certificate_path, 'certificate_path') _check_type(str, private_key_path, 'private_key_path') self.auth = _pulsar.AuthenticationTLS.create(certificate_path, private_key_path) class AuthenticationToken(Authentication): """ Token based authentication implementation """ def __init__(self, token): """ Create the token authentication provider instance. Parameters ---------- token A string containing the token or a functions that provides a string with the token """ if not (isinstance(token, str) or callable(token)): raise ValueError("Argument token is expected to be of type 'str' or a function returning 'str'") self.auth = _pulsar.AuthenticationToken.create(token) class AuthenticationAthenz(Authentication): """ Athenz Authentication implementation """ def __init__(self, auth_params_string): """ Create the Athenz authentication provider instance. Parameters ---------- auth_params_string: str JSON encoded configuration for Athenz client """ _check_type(str, auth_params_string, 'auth_params_string') self.auth = _pulsar.AuthenticationAthenz.create(auth_params_string) class AuthenticationOauth2(Authentication): """ Oauth2 Authentication implementation """ def __init__(self, auth_params_string: str): """ Create the Oauth2 authentication provider instance. You can create the instance by setting the necessary fields in the JSON string. .. code-block:: python auth = AuthenticationOauth2('{"issuer_url": "xxx", "private_key": "yyy"}') The valid JSON fields are: * issuer_url (required) The URL of the authentication provider which allows the Pulsar client to obtain an access token. * private_key (required) The URL to the JSON credentials file. It supports the following pattern formats: * ``/path/to/file`` * ``file:///path/to/file`` * ``file:/path/to/file`` * ``data:application/json;base64,`` The file content or the based64 encoded value is the encoded JSON string that contains the following fields: * ``client_id`` * ``client_secret`` * audience The OAuth 2.0 "resource server" identifier for a Pulsar cluster. * scope The scope of an access request. Parameters ---------- auth_params_string : str JSON encoded configuration for Oauth2 client """ _check_type(str, auth_params_string, 'auth_params_string') self.auth = _pulsar.AuthenticationOauth2.create(auth_params_string) class AuthenticationBasic(Authentication): """ Basic Authentication implementation """ def __init__(self, username=None, password=None, method='basic', auth_params_string=None): """ Create the Basic authentication provider instance. For example, if you want to create a basic authentication instance whose username is "my-user" and password is "my-pass", there are two ways: .. code-block:: python auth = AuthenticationBasic('my-user', 'my-pass') auth = AuthenticationBasic(auth_params_string='{"username": "my-user", "password": "my-pass"}') Parameters ---------- username : str, optional password : str, optional method : str, default='basic' The authentication method name auth_params_string : str, optional The JSON presentation of all fields above. If it's not None, the other parameters will be ignored. Here is an example JSON presentation: {"username": "my-user", "password": "my-pass", "method": "oms3.0"} The ``username`` and ``password`` fields are required. If the "method" field is not set, it will be "basic" by default. """ if auth_params_string is not None: _check_type(str, auth_params_string, 'auth_params_string') self.auth = _pulsar.AuthenticationBasic.create(auth_params_string) else: _check_type(str, username, 'username') _check_type(str, password, 'password') _check_type(str, method, 'method') self.auth = _pulsar.AuthenticationBasic.create(username, password, method) class ServiceInfoProvider: """ Base class for Python-defined service discovery and failover providers. Subclasses must return the initial :class:`ServiceInfo` and may keep the provided update callback to push later service changes into the client. """ def initial_service_info(self) -> "ServiceInfo": raise NotImplementedError def initialize(self, on_service_info_update: Callable[["ServiceInfo"], None]) -> None: raise NotImplementedError def close(self) -> None: """ Stop background work and release resources. This is invoked when the underlying C++ client destroys the provider, typically during :meth:`Client.close`. """ return None class ServiceInfo: """ Connection information for one Pulsar cluster endpoint. This is primarily used with :class:`AutoClusterFailover`. """ def __init__(self, service_url: str, authentication: Optional[Authentication] = None, tls_trust_certs_file_path: Optional[str] = None): """ Create a service info entry. Parameters ---------- service_url: str The Pulsar service URL for this cluster. authentication: Authentication, optional Authentication to use when connecting to this cluster. tls_trust_certs_file_path: str, optional Trust store path for TLS connections to this cluster. """ _check_type(str, service_url, 'service_url') _check_type_or_none(Authentication, authentication, 'authentication') _check_type_or_none(str, tls_trust_certs_file_path, 'tls_trust_certs_file_path') self._authentication = authentication self._service_info = _pulsar.ServiceInfo( service_url, authentication.auth if authentication else None, tls_trust_certs_file_path, ) @property def service_url(self) -> str: return self._service_info.service_url @property def use_tls(self) -> bool: return self._service_info.use_tls @property def tls_trust_certs_file_path(self) -> Optional[str]: return self._service_info.tls_trust_certs_file_path def __repr__(self) -> str: return ( "ServiceInfo(" f"service_url={self.service_url!r}, " f"use_tls={self.use_tls!r}, " f"tls_trust_certs_file_path={self.tls_trust_certs_file_path!r})" ) @classmethod def wrap(cls, service_info: _pulsar.ServiceInfo): self = cls.__new__(cls) self._authentication = None self._service_info = service_info return self class AutoClusterFailover: """ Cluster-level automatic failover configuration for :class:`Client`. """ def __init__(self, primary: ServiceInfo, secondary: List[ServiceInfo], check_interval_ms: int = 5000, failover_threshold: int = 1, switch_back_threshold: int = 1): """ Create an automatic failover configuration. Parameters ---------- primary: ServiceInfo The preferred cluster to use. secondary: list[ServiceInfo] Ordered fallback clusters to probe when the primary becomes unavailable. check_interval_ms: int, default=5000 Probe interval in milliseconds. failover_threshold: int, default=1 Number of consecutive probe failures required before failover. switch_back_threshold: int, default=1 Number of consecutive successful primary probes required before switching back. """ _check_type(ServiceInfo, primary, 'primary') _check_type(list, secondary, 'secondary') _check_type(int, check_interval_ms, 'check_interval_ms') _check_type(int, failover_threshold, 'failover_threshold') _check_type(int, switch_back_threshold, 'switch_back_threshold') if not secondary: raise ValueError("Argument secondary is expected to contain at least one ServiceInfo") for index, service_info in enumerate(secondary): if not isinstance(service_info, ServiceInfo): raise ValueError( "Argument secondary[%d] is expected to be of type 'ServiceInfo' and not '%s'" % (index, type(service_info).__name__) ) if check_interval_ms <= 0: raise ValueError("Argument check_interval_ms is expected to be greater than 0") if failover_threshold <= 0: raise ValueError("Argument failover_threshold is expected to be greater than 0") if switch_back_threshold <= 0: raise ValueError("Argument switch_back_threshold is expected to be greater than 0") self.primary = primary self.secondary = list(secondary) self.check_interval_ms = check_interval_ms self.failover_threshold = failover_threshold self.switch_back_threshold = switch_back_threshold def __repr__(self) -> str: return ( "AutoClusterFailover(" f"primary={self.primary!r}, " f"secondary={self.secondary!r}, " f"check_interval_ms={self.check_interval_ms!r}, " f"failover_threshold={self.failover_threshold!r}, " f"switch_back_threshold={self.switch_back_threshold!r})" ) class ConsumerDeadLetterPolicy: """ Configuration for the "dead letter queue" feature in consumer. """ def __init__(self, max_redeliver_count: int, dead_letter_topic: str = None, initial_subscription_name: str = None): """ Wrapper DeadLetterPolicy. Parameters ---------- max_redeliver_count: Maximum number of times that a message is redelivered before being sent to the dead letter queue. - The maxRedeliverCount must be greater than 0. dead_letter_topic: Name of the dead topic where the failing messages are sent. The default value is: sourceTopicName + "-" + subscriptionName + "-DLQ" initial_subscription_name: Name of the initial subscription name of the dead letter topic. If this field is not set, the initial subscription for the dead letter topic is not created. If this field is set but the broker's `allowAutoSubscriptionCreation` is disabled, the DLQ producer fails to be created. """ builder = DeadLetterPolicyBuilder() if max_redeliver_count is None or max_redeliver_count < 1: raise ValueError("max_redeliver_count must be greater than 0") builder.maxRedeliverCount(max_redeliver_count) if dead_letter_topic is not None: builder.deadLetterTopic(dead_letter_topic) if initial_subscription_name is not None: builder.initialSubscriptionName(initial_subscription_name) self._policy = builder.build() @property def dead_letter_topic(self) -> str: """ Return the dead letter topic for dead letter policy. """ return self._policy.getDeadLetterTopic() @property def max_redeliver_count(self) -> int: """ Return the max redeliver count for dead letter policy. """ return self._policy.getMaxRedeliverCount() @property def initial_subscription_name(self) -> str: """ Return the initial subscription name for dead letter policy. """ return self._policy.getInitialSubscriptionName() def policy(self): """ Returns the actual one DeadLetterPolicy. """ return self._policy class CryptoKeyReader: """ Default crypto key reader implementation """ def __init__(self, public_key_path, private_key_path): """ Create crypto key reader. Parameters ---------- public_key_path: str Path to the public key private_key_path: str Path to private key """ _check_type(str, public_key_path, 'public_key_path') _check_type(str, private_key_path, 'private_key_path') self.cryptoKeyReader = _pulsar.CryptoKeyReader(public_key_path, private_key_path) class Client: """ The Pulsar client. A single client instance can be used to create producers and consumers on multiple topics. The client will share the same connection pool and threads across all producers and consumers. """ def __init__(self, service_url, authentication=None, operation_timeout_seconds=30, io_threads=1, message_listener_threads=1, concurrent_lookup_requests=50000, log_conf_file_path=None, stats_interval_in_seconds=600, use_tls=False, tls_trust_certs_file_path=None, tls_allow_insecure_connection=False, tls_validate_hostname=False, logger=None, connection_timeout_ms=10000, listener_name=None, tls_private_key_file_path: Optional[str] = None, tls_certificate_file_path: Optional[str] = None, ): """ Create a new Pulsar client instance. Parameters ---------- service_url: str or AutoClusterFailover or ServiceInfoProvider The Pulsar service URL, for example ``pulsar://my-broker.com:6650/``, or an :class:`AutoClusterFailover` or :class:`ServiceInfoProvider` configuration. authentication: Authentication, optional Set the authentication provider to be used with the broker. Supported methods: * `AuthenticationTLS` * `AuthenticationToken` * `AuthenticationAthenz` * `AuthenticationOauth2` operation_timeout_seconds: int, default=30 Set timeout on client operations (subscribe, create producer, close, unsubscribe). io_threads: int, default=1 Set the number of IO threads to be used by the Pulsar client. message_listener_threads: int, default=1 Set the number of threads to be used by the Pulsar client when delivering messages through message listener. The default is 1 thread per Pulsar client. If using more than 1 thread, messages for distinct ``message_listener``s will be delivered in different threads, however a single ``MessageListener`` will always be assigned to the same thread. concurrent_lookup_requests: int, default=50000 Number of concurrent lookup-requests allowed on each broker connection to prevent overload on the broker. log_conf_file_path: str, optional This parameter is deprecated and makes no effect. It's retained only for compatibility. Use `logger` to customize a logger. stats_interval_in_seconds: int, default=600 Set the interval between each stats information update. Stats are printed and/or passed to the statistics listener at this interval. Set to 0 to disable stats collection. use_tls: bool, default=False it's a deprecated config that never works, whether TLS is enabled is determined by ``service_url``. tls_trust_certs_file_path: str, optional Set the path to the trusted TLS certificate file. If empty defaults to certifi. tls_allow_insecure_connection: bool, default=False Configure whether the Pulsar client accepts untrusted TLS certificates from the broker. tls_validate_hostname: bool, default=False Configure whether the Pulsar client validates that the hostname of the endpoint, matches the common name on the TLS certificate presented by the endpoint. logger: optional Set a Python logger for this Pulsar client. Should be an instance of `logging.Logger`. It should be noted that if the Python logger is configured, during the termination of the Python interpreter, the Python logger will be unavailable and the default logger will be used for logging. To avoid strange behavior, you'd better delete all instances explicitly before exiting. .. code-block:: python import logging client = Client(service_url, logger=logging.getLogger('pulsar')) producer = client.create_producer(topic) # ... del producer del client connection_timeout_ms: int, default=10000 Set timeout in milliseconds on TCP connections. listener_name: str, optional Listener name for lookup. Clients can use listenerName to choose one of the listeners as the service URL to create a connection to the broker as long as the network is accessible. ``advertisedListeners`` must be enabled in broker side. tls_private_key_file_path: str, optional The path to the TLS private key file tls_certificate_file_path: str, optional The path to the TLS certificate file. """ if not isinstance(service_url, (str, AutoClusterFailover, ServiceInfoProvider)): raise ValueError( "Argument service_url is expected to be of type 'str', 'AutoClusterFailover' or " "'ServiceInfoProvider'" ) if isinstance(service_url, (AutoClusterFailover, ServiceInfoProvider)) and authentication is not None: raise ValueError( "Argument authentication is not supported when service_url is an AutoClusterFailover or " "ServiceInfoProvider; set authentication on each ServiceInfo instead" ) if isinstance(service_url, (AutoClusterFailover, ServiceInfoProvider)) and \ tls_trust_certs_file_path is not None: raise ValueError( "Argument tls_trust_certs_file_path is not supported when service_url is an " "AutoClusterFailover or ServiceInfoProvider; set tls_trust_certs_file_path on each " "ServiceInfo instead" ) _check_type_or_none(Authentication, authentication, 'authentication') _check_type(int, operation_timeout_seconds, 'operation_timeout_seconds') _check_type(int, connection_timeout_ms, 'connection_timeout_ms') _check_type(int, io_threads, 'io_threads') _check_type(int, message_listener_threads, 'message_listener_threads') _check_type(int, concurrent_lookup_requests, 'concurrent_lookup_requests') _check_type_or_none(str, log_conf_file_path, 'log_conf_file_path') _check_type(int, stats_interval_in_seconds, 'stats_interval_in_seconds') _check_type(bool, use_tls, 'use_tls') _check_type_or_none(str, tls_trust_certs_file_path, 'tls_trust_certs_file_path') _check_type(bool, tls_allow_insecure_connection, 'tls_allow_insecure_connection') _check_type(bool, tls_validate_hostname, 'tls_validate_hostname') _check_type_or_none(str, listener_name, 'listener_name') _check_type_or_none(str, tls_private_key_file_path, 'tls_private_key_file_path') _check_type_or_none(str, tls_certificate_file_path, 'tls_certificate_file_path') conf = _pulsar.ClientConfiguration() if authentication: conf.authentication(authentication.auth) conf.operation_timeout_seconds(operation_timeout_seconds) conf.connection_timeout(connection_timeout_ms) conf.io_threads(io_threads) conf.message_listener_threads(message_listener_threads) conf.concurrent_lookup_requests(concurrent_lookup_requests) conf.stats_interval_in_seconds(stats_interval_in_seconds) if isinstance(logger, logging.Logger): conf.set_logger(self._prepare_logger(logger)) elif isinstance(logger, ConsoleLogger): conf.set_console_logger(logger.log_level) elif isinstance(logger, FileLogger): conf.set_file_logger(logger.log_level, logger.log_file) elif logger is not None: raise ValueError("Logger is expected to be either None, logger.Logger, pulsar.ConsoleLogger or pulsar.FileLogger") if listener_name: conf.listener_name(listener_name) if tls_trust_certs_file_path: conf.tls_trust_certs_file_path(tls_trust_certs_file_path) else: import certifi conf.tls_trust_certs_file_path(certifi.where()) conf.tls_allow_insecure_connection(tls_allow_insecure_connection) conf.tls_validate_hostname(tls_validate_hostname) if tls_private_key_file_path is not None: conf.tls_private_key_file_path(tls_private_key_file_path) if tls_certificate_file_path is not None: conf.tls_certificate_file_path(tls_certificate_file_path) if isinstance(service_url, AutoClusterFailover): self._client = _pulsar.Client.create_auto_cluster_failover( service_url.primary._service_info, [service_info._service_info for service_info in service_url.secondary], service_url.check_interval_ms, service_url.failover_threshold, service_url.switch_back_threshold, conf, ) elif isinstance(service_url, ServiceInfoProvider): try: self._client = _pulsar.Client.create_service_info_provider(service_url, conf) except RuntimeError as e: if str(e) == "Expected a pulsar.ServiceInfo or _pulsar.ServiceInfo instance": raise ValueError(str(e)) raise else: self._client = _pulsar.Client(service_url, conf) self._consumers = [] @staticmethod def _prepare_logger(logger): import logging def log(level, message): old_threads = logging.logThreads logging.logThreads = False logger.log(logging.getLevelName(level), message) logging.logThreads = old_threads return log def create_producer(self, topic, producer_name=None, schema=schema.BytesSchema(), initial_sequence_id=None, send_timeout_millis=30000, compression_type: CompressionType = CompressionType.NONE, max_pending_messages=1000, max_pending_messages_across_partitions=50000, block_if_queue_full=False, batching_enabled=False, batching_max_messages=1000, batching_max_allowed_size_in_bytes=128*1024, batching_max_publish_delay_ms=10, chunking_enabled=False, message_routing_mode: PartitionsRoutingMode = PartitionsRoutingMode.RoundRobinDistribution, lazy_start_partitioned_producers=False, properties=None, batching_type: BatchingType = BatchingType.Default, encryption_key=None, crypto_key_reader: Union[None, CryptoKeyReader] = None, access_mode: ProducerAccessMode = ProducerAccessMode.Shared, message_router: Callable[[Message, int], int]=None, ): """ Create a new producer on a given topic. Parameters ---------- topic: str The topic name producer_name: str, optional Specify a name for the producer. If not assigned, the system will generate a globally unique name which can be accessed with `Producer.producer_name()`. When specifying a name, it is app to the user to ensure that, for a given topic, the producer name is unique across all Pulsar's clusters. schema: pulsar.schema.Schema, default=pulsar.schema.BytesSchema Define the schema of the data that will be published by this producer, e.g, ``schema=JsonSchema(MyRecordClass)``. The schema will be used for two purposes: * Validate the data format against the topic defined schema * Perform serialization/deserialization between data and objects initial_sequence_id: int, optional Set the baseline for the sequence ids for messages published by the producer. First message will be using ``(initialSequenceId + 1)`` as its sequence id and subsequent messages will be assigned incremental sequence ids, if not otherwise specified. send_timeout_millis: int, default=30000 If a message is not acknowledged by the server before the ``send_timeout`` expires, an error will be reported. compression_type: CompressionType, default=CompressionType.NONE Set the compression type for the producer. By default, message payloads are not compressed. Supported compression types: * CompressionType.LZ4 * CompressionType.ZLib * CompressionType.ZSTD * CompressionType.SNAPPY ZSTD is supported since Pulsar 2.3. Consumers will need to be at least at that release in order to be able to receive messages compressed with ZSTD. SNAPPY is supported since Pulsar 2.4. Consumers will need to be at least at that release in order to be able to receive messages compressed with SNAPPY. batching_enabled: bool, default=False When automatic batching is enabled, multiple calls to `send` can result in a single batch to be sent to the broker, leading to better throughput, especially when publishing small messages. All messages in a batch will be published as a single batched message. The consumer will be delivered individual messages in the batch in the same order they were enqueued. batching_max_messages: int, default=1000 When you set this option to a value greater than 1, messages are queued until this threshold or `batching_max_allowed_size_in_bytes` is reached or batch interval has elapsed. batching_max_allowed_size_in_bytes: int, default=128*1024 When you set this option to a value greater than 1, messages are queued until this threshold or `batching_max_messages` is reached or batch interval has elapsed. batching_max_publish_delay_ms: int, default=10 The batch interval in milliseconds. Queued messages will be sent in batch after this interval even if both the threshold of `batching_max_messages` and `batching_max_allowed_size_in_bytes` are not reached. max_pending_messages: int, default=1000 Set the max size of the queue holding the messages pending to receive an acknowledgment from the broker. max_pending_messages_across_partitions: int, default=50000 Set the max size of the queue holding the messages pending to receive an acknowledgment across partitions from the broker. block_if_queue_full: bool, default=False Set whether `send_async` operations should block when the outgoing message queue is full. message_routing_mode: PartitionsRoutingMode, default=PartitionsRoutingMode.RoundRobinDistribution Set the message routing mode for the partitioned producer. Supported modes: * ``PartitionsRoutingMode.RoundRobinDistribution`` * ``PartitionsRoutingMode.UseSinglePartition`` lazy_start_partitioned_producers: bool, default=False This config affects producers of partitioned topics only. It controls whether producers register and connect immediately to the owner broker of each partition or start lazily on demand. The internal producer of one partition is always started eagerly, chosen by the routing policy, but the internal producers of any additional partitions are started on demand, upon receiving their first message. Using this mode can reduce the strain on brokers for topics with large numbers of partitions and when the SinglePartition routing policy is used without keyed messages. Because producer connection can be on demand, this can produce extra send latency for the first messages of a given partition. properties: dict, optional Sets the properties for the producer. The properties associated with a producer can be used for identify a producer at broker side. batching_type: BatchingType, default=BatchingType.Default Sets the batching type for the producer. There are two batching type: DefaultBatching and KeyBasedBatching. DefaultBatching will batch single messages: (k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3) ... into single batch message: [(k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3)] KeyBasedBatching will batch incoming single messages: (k1, v1), (k2, v1), (k3, v1), (k1, v2), (k2, v2), (k3, v2), (k1, v3), (k2, v3), (k3, v3) ... into single batch message: [(k1, v1), (k1, v2), (k1, v3)], [(k2, v1), (k2, v2), (k2, v3)], [(k3, v1), (k3, v2), (k3, v3)] chunking_enabled: bool, default=False If message size is higher than allowed max publish-payload size by broker then chunking_enabled helps producer to split message into multiple chunks and publish them to broker separately and in order. So, it allows client to successfully publish large size of messages in pulsar. encryption_key: str, optional The key used for symmetric encryption, configured on the producer side crypto_key_reader: CryptoKeyReader, optional Symmetric encryption class implementation, configuring public key encryption messages for the producer and private key decryption messages for the consumer access_mode: ProducerAccessMode, optional Set the type of access mode that the producer requires on the topic. Supported modes: * Shared: By default multiple producers can publish on a topic. * Exclusive: Require exclusive access for producer. Fail immediately if there's already a producer connected. * WaitForExclusive: Producer creation is pending until it can acquire exclusive access. * ExclusiveWithFencing: Acquire exclusive access for the producer. Any existing producer will be removed and invalidated immediately. message_router: optional A custom message router function that takes a `Message` and the number of partitions and returns the partition index to which the message should be routed. If not provided, the default routing policy defined by `message_routing_mode` will be used. """ _check_type(str, topic, 'topic') _check_type_or_none(str, producer_name, 'producer_name') _check_type(_schema.Schema, schema, 'schema') _check_type_or_none(int, initial_sequence_id, 'initial_sequence_id') _check_type(int, send_timeout_millis, 'send_timeout_millis') _check_type(CompressionType, compression_type, 'compression_type') _check_type(int, max_pending_messages, 'max_pending_messages') _check_type(int, max_pending_messages_across_partitions, 'max_pending_messages_across_partitions') _check_type(bool, block_if_queue_full, 'block_if_queue_full') _check_type(bool, batching_enabled, 'batching_enabled') _check_type(int, batching_max_messages, 'batching_max_messages') _check_type(int, batching_max_allowed_size_in_bytes, 'batching_max_allowed_size_in_bytes') _check_type(int, batching_max_publish_delay_ms, 'batching_max_publish_delay_ms') _check_type(bool, chunking_enabled, 'chunking_enabled') _check_type_or_none(dict, properties, 'properties') _check_type(BatchingType, batching_type, 'batching_type') _check_type_or_none(str, encryption_key, 'encryption_key') _check_type_or_none(CryptoKeyReader, crypto_key_reader, 'crypto_key_reader') _check_type(bool, lazy_start_partitioned_producers, 'lazy_start_partitioned_producers') _check_type(ProducerAccessMode, access_mode, 'access_mode') conf = _pulsar.ProducerConfiguration() conf.send_timeout_millis(send_timeout_millis) conf.compression_type(compression_type) conf.max_pending_messages(max_pending_messages) conf.max_pending_messages_across_partitions(max_pending_messages_across_partitions) conf.block_if_queue_full(block_if_queue_full) conf.batching_enabled(batching_enabled) conf.batching_max_messages(batching_max_messages) conf.batching_max_allowed_size_in_bytes(batching_max_allowed_size_in_bytes) conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms) conf.partitions_routing_mode(message_routing_mode) conf.batching_type(batching_type) conf.chunking_enabled(chunking_enabled) conf.lazy_start_partitioned_producers(lazy_start_partitioned_producers) conf.access_mode(access_mode) if message_router is not None: underlying_router = lambda msg, num_partitions: int(message_router(Message._wrap(msg), num_partitions)) conf.message_router(underlying_router) if producer_name: conf.producer_name(producer_name) if initial_sequence_id: conf.initial_sequence_id(initial_sequence_id) if properties: for k, v in properties.items(): conf.property(k, v) conf.schema(schema.schema_info()) if encryption_key: conf.encryption_key(encryption_key) if crypto_key_reader: conf.crypto_key_reader(crypto_key_reader.cryptoKeyReader) if batching_enabled and chunking_enabled: raise ValueError("Batching and chunking of messages can't be enabled together.") p = Producer() p._producer = self._client.create_producer(topic, conf) p._schema = schema p._client = self._client return p def subscribe(self, topic, subscription_name, consumer_type: ConsumerType = ConsumerType.Exclusive, schema=schema.BytesSchema(), message_listener=None, receiver_queue_size=1000, max_total_receiver_queue_size_across_partitions=50000, consumer_name=None, unacked_messages_timeout_ms=None, broker_consumer_stats_cache_time_ms=30000, negative_ack_redelivery_delay_ms=60000, is_read_compacted=False, properties=None, pattern_auto_discovery_period=60, initial_position: InitialPosition = InitialPosition.Latest, crypto_key_reader: Union[None, CryptoKeyReader] = None, replicate_subscription_state_enabled=False, max_pending_chunked_message=10, auto_ack_oldest_chunked_message_on_queue_full=False, start_message_id_inclusive=False, batch_receive_policy=None, key_shared_policy=None, batch_index_ack_enabled=False, regex_subscription_mode: RegexSubscriptionMode = RegexSubscriptionMode.PersistentOnly, dead_letter_policy: Union[None, ConsumerDeadLetterPolicy] = None, crypto_failure_action: ConsumerCryptoFailureAction = ConsumerCryptoFailureAction.FAIL, ): """ Subscribe to the given topic and subscription combination. Parameters ---------- topic: The name of the topic, list of topics or regex pattern. This method will accept these forms: * ``topic='my-topic'`` * ``topic=['topic-1', 'topic-2', 'topic-3']`` * ``topic=re.compile('persistent://public/default/topic-*')`` subscription_name: str The name of the subscription. consumer_type: ConsumerType, default=ConsumerType.Exclusive Select the subscription type to be used when subscribing to the topic. schema: pulsar.schema.Schema, default=pulsar.schema.BytesSchema Define the schema of the data that will be received by this consumer. message_listener: optional Sets a message listener for the consumer. When the listener is set, the application will receive messages through it. Calls to ``consumer.receive()`` will not be allowed. The listener function needs to accept (consumer, message), for example: .. code-block:: python def my_listener(consumer, message): # process message consumer.acknowledge(message) receiver_queue_size: int, default=1000 Sets the size of the consumer receive queue. The consumer receive queue controls how many messages can be accumulated by the consumer before the application calls `receive()`. Using a higher value could potentially increase the consumer throughput at the expense of higher memory utilization. Setting the consumer queue size to zero decreases the throughput of the consumer by disabling pre-fetching of messages. This approach improves the message distribution on shared subscription by pushing messages only to those consumers that are ready to process them. Neither receive with timeout nor partitioned topics can be used if the consumer queue size is zero. The `receive()` function call should not be interrupted when the consumer queue size is zero. The default value is 1000 messages and should work well for most use cases. max_total_receiver_queue_size_across_partitions: int, default=50000 Set the max total receiver queue size across partitions. This setting will be used to reduce the receiver queue size for individual partitions consumer_name: str, optional Sets the consumer name. unacked_messages_timeout_ms: int, optional Sets the timeout in milliseconds for unacknowledged messages. The timeout needs to be greater than 10 seconds. An exception is thrown if the given value is less than 10 seconds. If a successful acknowledgement is not sent within the timeout, all the unacknowledged messages are redelivered. negative_ack_redelivery_delay_ms: int, default=60000 The delay after which to redeliver the messages that failed to be processed (with the ``consumer.negative_acknowledge()``) broker_consumer_stats_cache_time_ms: int, default=30000 Sets the time duration for which the broker-side consumer stats will be cached in the client. is_read_compacted: bool, default=False Selects whether to read the compacted version of the topic properties: dict, optional Sets the properties for the consumer. The properties associated with a consumer can be used for identify a consumer at broker side. pattern_auto_discovery_period: int, default=60 Periods of seconds for consumer to auto discover match topics. initial_position: InitialPosition, default=InitialPosition.Latest Set the initial position of a consumer when subscribing to the topic. It could be either: ``InitialPosition.Earliest`` or ``InitialPosition.Latest``. crypto_key_reader: CryptoKeyReader, optional Symmetric encryption class implementation, configuring public key encryption messages for the producer and private key decryption messages for the consumer replicate_subscription_state_enabled: bool, default=False Set whether the subscription status should be replicated. max_pending_chunked_message: int, default=10 Consumer buffers chunk messages into memory until it receives all the chunks of the original message. While consuming chunk-messages, chunks from same message might not be contiguous in the stream, and they might be mixed with other messages' chunks. so, consumer has to maintain multiple buffers to manage chunks coming from different messages. This mainly happens when multiple publishers are publishing messages on the topic concurrently or publisher failed to publish all chunks of the messages. If it's zero, the pending chunked messages will not be limited. auto_ack_oldest_chunked_message_on_queue_full: bool, default=False Buffering large number of outstanding uncompleted chunked messages can create memory pressure, and it can be guarded by providing the maxPendingChunkedMessage threshold. See setMaxPendingChunkedMessage. Once, consumer reaches this threshold, it drops the outstanding unchunked-messages by silently acking if autoAckOldestChunkedMessageOnQueueFull is true else it marks them for redelivery. start_message_id_inclusive: bool, default=False Set the consumer to include the given position of any reset operation like Consumer::seek. batch_receive_policy: class ConsumerBatchReceivePolicy Set the batch collection policy for batch receiving. key_shared_policy: class ConsumerKeySharedPolicy Set the key shared policy for use when the ConsumerType is KeyShared. batch_index_ack_enabled: Enable the batch index acknowledgement. It should be noted that this option can only work when the broker side also enables the batch index acknowledgement. See the `acknowledgmentAtBatchIndexLevelEnabled` config in `broker.conf`. regex_subscription_mode: RegexSubscriptionMode, optional Set the regex subscription mode for use when the topic is a regex pattern. Supported modes: * PersistentOnly: By default only subscribe to persistent topics. * NonPersistentOnly: Only subscribe to non-persistent topics. * AllTopics: Subscribe to both persistent and non-persistent topics. dead_letter_policy: class ConsumerDeadLetterPolicy Set dead letter policy for consumer. By default, some messages are redelivered many times, even to the extent that they can never be stopped. By using the dead letter mechanism, messages have the max redelivery count, when they're exceeding the maximum number of redeliveries. Messages are sent to dead letter topics and acknowledged automatically. crypto_failure_action: ConsumerCryptoFailureAction, default=ConsumerCryptoFailureAction.FAIL Set the behavior when the decryption fails. The default is to fail the message. Supported actions: * ConsumerCryptoFailureAction.FAIL: Fail consume until crypto succeeds * ConsumerCryptoFailureAction.DISCARD: Message is silently acknowledged and not delivered to the application. * ConsumerCryptoFailureAction.CONSUME: Deliver the encrypted message to the application. It's the application's responsibility to decrypt the message. If message is also compressed, decompression will fail. If the message contains batch messages, client will not be able to retrieve individual messages in the batch. """ _check_type(str, subscription_name, 'subscription_name') _check_type(ConsumerType, consumer_type, 'consumer_type') _check_type(_schema.Schema, schema, 'schema') _check_type(int, receiver_queue_size, 'receiver_queue_size') _check_type(int, max_total_receiver_queue_size_across_partitions, 'max_total_receiver_queue_size_across_partitions') _check_type_or_none(str, consumer_name, 'consumer_name') _check_type_or_none(int, unacked_messages_timeout_ms, 'unacked_messages_timeout_ms') _check_type(int, broker_consumer_stats_cache_time_ms, 'broker_consumer_stats_cache_time_ms') _check_type(int, negative_ack_redelivery_delay_ms, 'negative_ack_redelivery_delay_ms') _check_type(int, pattern_auto_discovery_period, 'pattern_auto_discovery_period') _check_type(bool, is_read_compacted, 'is_read_compacted') _check_type_or_none(dict, properties, 'properties') _check_type(InitialPosition, initial_position, 'initial_position') _check_type_or_none(CryptoKeyReader, crypto_key_reader, 'crypto_key_reader') _check_type(int, max_pending_chunked_message, 'max_pending_chunked_message') _check_type(bool, auto_ack_oldest_chunked_message_on_queue_full, 'auto_ack_oldest_chunked_message_on_queue_full') _check_type(bool, start_message_id_inclusive, 'start_message_id_inclusive') _check_type_or_none(ConsumerBatchReceivePolicy, batch_receive_policy, 'batch_receive_policy') _check_type_or_none(ConsumerKeySharedPolicy, key_shared_policy, 'key_shared_policy') _check_type(bool, batch_index_ack_enabled, 'batch_index_ack_enabled') _check_type(RegexSubscriptionMode, regex_subscription_mode, 'regex_subscription_mode') _check_type(ConsumerCryptoFailureAction, crypto_failure_action, 'crypto_failure_action') conf = _pulsar.ConsumerConfiguration() conf.consumer_type(consumer_type) conf.regex_subscription_mode(regex_subscription_mode) conf.read_compacted(is_read_compacted) if message_listener: conf.message_listener(_listener_wrapper(message_listener, schema)) conf.receiver_queue_size(receiver_queue_size) conf.max_total_receiver_queue_size_across_partitions(max_total_receiver_queue_size_across_partitions) if consumer_name: conf.consumer_name(consumer_name) if unacked_messages_timeout_ms: conf.unacked_messages_timeout_ms(unacked_messages_timeout_ms) conf.negative_ack_redelivery_delay_ms(negative_ack_redelivery_delay_ms) conf.broker_consumer_stats_cache_time_ms(broker_consumer_stats_cache_time_ms) if properties: for k, v in properties.items(): conf.property(k, v) conf.subscription_initial_position(initial_position) conf.schema(schema.schema_info()) if crypto_key_reader: conf.crypto_key_reader(crypto_key_reader.cryptoKeyReader) conf.replicate_subscription_state_enabled(replicate_subscription_state_enabled) conf.max_pending_chunked_message(max_pending_chunked_message) conf.auto_ack_oldest_chunked_message_on_queue_full(auto_ack_oldest_chunked_message_on_queue_full) conf.start_message_id_inclusive(start_message_id_inclusive) if batch_receive_policy: conf.batch_receive_policy(batch_receive_policy.policy()) if key_shared_policy: conf.key_shared_policy(key_shared_policy.policy()) conf.batch_index_ack_enabled(batch_index_ack_enabled) if dead_letter_policy: conf.dead_letter_policy(dead_letter_policy.policy()) conf.crypto_failure_action(crypto_failure_action) c = Consumer() if isinstance(topic, str): # Single topic c._consumer = self._client.subscribe(topic, subscription_name, conf) elif isinstance(topic, list): # List of topics c._consumer = self._client.subscribe_topics(topic, subscription_name, conf) elif isinstance(topic, _retype): # Regex pattern c._consumer = self._client.subscribe_pattern(topic.pattern, subscription_name, conf) else: raise ValueError("Argument 'topic' is expected to be of a type between (str, list, re.pattern)") c._client = self c._schema = schema c._schema.attach_client(self._client) self._consumers.append(c) return c def create_reader(self, topic, start_message_id, schema=schema.BytesSchema(), reader_listener=None, receiver_queue_size=1000, reader_name=None, subscription_role_prefix=None, is_read_compacted=False, crypto_key_reader: Union[None, CryptoKeyReader] = None, start_message_id_inclusive=False, crypto_failure_action: ConsumerCryptoFailureAction = ConsumerCryptoFailureAction.FAIL, ): """ Create a reader on a particular topic Parameters ---------- topic: The name of the topic. start_message_id: The initial reader positioning is done by specifying a message id. The options are: * ``MessageId.earliest``: Start reading from the earliest message available in the topic * ``MessageId.latest``: Start reading from the end topic, only getting messages published after the reader was created * ``MessageId``: When passing a particular message id, the reader will position itself on that specific position. The first message to be read will be the message next to the specified messageId. Message id can be serialized into a string and deserialized back into a `MessageId` object: .. code-block:: python # Serialize to string s = msg.message_id().serialize() # Deserialize from string msg_id = MessageId.deserialize(s) schema: pulsar.schema.Schema, default=pulsar.schema.BytesSchema Define the schema of the data that will be received by this reader. reader_listener: optional Sets a message listener for the reader. When the listener is set, the application will receive messages through it. Calls to ``reader.read_next()`` will not be allowed. The listener function needs to accept (reader, message), for example: .. code-block:: python def my_listener(reader, message): # process message pass receiver_queue_size: int, default=1000 Sets the size of the reader receive queue. The reader receive queue controls how many messages can be accumulated by the reader before the application calls `read_next()`. Using a higher value could potentially increase the reader throughput at the expense of higher memory utilization. reader_name: str, optional Sets the reader name. subscription_role_prefix: str, optional Sets the subscription role prefix. is_read_compacted: bool, default=False Selects whether to read the compacted version of the topic crypto_key_reader: CryptoKeyReader, optional Symmetric encryption class implementation, configuring public key encryption messages for the producer and private key decryption messages for the consumer start_message_id_inclusive: bool, default=False Set the reader to include the startMessageId or given position of any reset operation like Reader.seek crypto_failure_action: ConsumerCryptoFailureAction, default=ConsumerCryptoFailureAction.FAIL Set the behavior when the decryption fails. The default is to fail the message. Supported actions: * ConsumerCryptoFailureAction.FAIL: Fail consume until crypto succeeds * ConsumerCryptoFailureAction.DISCARD: Message is silently acknowledged and not delivered to the application. * ConsumerCryptoFailureAction.CONSUME: Deliver the encrypted message to the application. It's the application's responsibility to decrypt the message. If message is also compressed, decompression will fail. If the message contains batch messages, client will not be able to retrieve individual messages in the batch. """ # If a pulsar.MessageId object is passed, access the _pulsar.MessageId object if isinstance(start_message_id, MessageId): start_message_id = start_message_id._msg_id _check_type(str, topic, 'topic') _check_type(_pulsar.MessageId, start_message_id, 'start_message_id') _check_type(_schema.Schema, schema, 'schema') _check_type(int, receiver_queue_size, 'receiver_queue_size') _check_type_or_none(str, reader_name, 'reader_name') _check_type_or_none(str, subscription_role_prefix, 'subscription_role_prefix') _check_type(bool, is_read_compacted, 'is_read_compacted') _check_type_or_none(CryptoKeyReader, crypto_key_reader, 'crypto_key_reader') _check_type(bool, start_message_id_inclusive, 'start_message_id_inclusive') _check_type(ConsumerCryptoFailureAction, crypto_failure_action, 'crypto_failure_action') conf = _pulsar.ReaderConfiguration() if reader_listener: conf.reader_listener(_listener_wrapper(reader_listener, schema)) conf.receiver_queue_size(receiver_queue_size) if reader_name: conf.reader_name(reader_name) if subscription_role_prefix: conf.subscription_role_prefix(subscription_role_prefix) conf.schema(schema.schema_info()) conf.read_compacted(is_read_compacted) if crypto_key_reader: conf.crypto_key_reader(crypto_key_reader.cryptoKeyReader) conf.start_message_id_inclusive(start_message_id_inclusive) conf.crypto_failure_action(crypto_failure_action) c = Reader() c._reader = self._client.create_reader(topic, start_message_id, conf) c._client = self c._schema = schema c._schema.attach_client(self._client) self._consumers.append(c) return c def create_table_view(self, topic: str, subscription_name: Optional[str] = None, schema: schema.Schema = schema.BytesSchema()) -> TableView: """ Create a table view on a particular topic Parameters ---------- topic: str The name of the topic. subscription_name: str, optional The name of the subscription. If it's not specified, a random subscription name will be used. schema: pulsar.schema.Schema, default=pulsar.schema.BytesSchema Define the schema of this table view. If the schema is incompatible with the topic's schema, this method will throw an exception. This schema is also used to deserialize the value of messages in the table view. Returns ------- TableView A table view instance. """ _check_type(str, topic, 'topic') _check_type_or_none(str, subscription_name, 'subscription_name') _check_type(_schema.Schema, schema, 'schema') tv_conf = _pulsar.TableViewConfiguration() if subscription_name is not None: tv_conf.subscription_name(subscription_name) tv_conf.schema(schema.schema_info()) tv = self._client.create_table_view(topic, tv_conf) self._table_view = TableView(tv, topic, subscription_name, schema) return self._table_view def get_topic_partitions(self, topic): """ Get the list of partitions for a given topic. If the topic is partitioned, this will return a list of partition names. If the topic is not partitioned, the returned list will contain the topic name itself. This can be used to discover the partitions and create Reader, Consumer or Producer instances directly on a particular partition. Parameters ---------- topic: str the topic name to lookup Returns ------- list a list of partition name """ _check_type(str, topic, 'topic') return self._client.get_topic_partitions(topic) def get_service_info(self) -> ServiceInfo: """ Get the current service info used by this client. """ return ServiceInfo.wrap(self._client.get_service_info()) def shutdown(self): """ Perform immediate shutdown of Pulsar client. Release all resources and close all producer, consumer, and readers without waiting for ongoing operations to complete. """ self._client.shutdown() def close(self): """ Close the client and all the associated producers and consumers """ self._client.close() class Producer: """ The Pulsar message producer, used to publish messages on a topic. Examples -------- .. code-block:: python import pulsar client = pulsar.Client('pulsar://localhost:6650') producer = client.create_producer('my-topic') for i in range(10): producer.send(('Hello-%d' % i).encode('utf-8')) client.close() """ def topic(self): """ Return the topic which producer is publishing to """ return self._producer.topic() def producer_name(self): """ Return the producer name which could have been assigned by the system or specified by the client """ return self._producer.producer_name() def last_sequence_id(self): """ Get the last sequence id that was published by this producer. This represents either the automatically assigned or custom sequence id (set on the ``MessageBuilder``) that was published and acknowledged by the broker. After recreating a producer with the same producer name, this will return the last message that was published in the previous producer session, or -1 if there was no message ever published. """ return self._producer.last_sequence_id() def send(self, content, properties=None, partition_key=None, ordering_key=None, sequence_id=None, replication_clusters=None, disable_replication=False, event_timestamp=None, deliver_at=None, deliver_after=None, ) -> _pulsar.MessageId: """ Publish a message on the topic. Blocks until the message is acknowledged Returns a `MessageId` object that represents where the message is persisted. Parameters ---------- content: A ``bytes`` object with the message payload. properties: optional A dict of application-defined string properties. partition_key: optional Sets the partition key for message routing. A hash of this key is used to determine the message's topic partition. ordering_key: optional Sets the ordering key for message routing. sequence_id: optional Specify a custom sequence id for the message being published. replication_clusters: optional Override namespace replication clusters. Note that it is the caller's responsibility to provide valid cluster names and that all clusters have been previously configured as topics. Given an empty list, the message will replicate according to the namespace configuration. disable_replication: bool, default=False Do not replicate this message. event_timestamp: optional Timestamp in millis of the timestamp of event creation deliver_at: optional Specify the message should not be delivered earlier than the specified timestamp. The timestamp is milliseconds and based on UTC deliver_after: optional Specify a delay in timedelta for the delivery of the messages. Returns ---------- A `_pulsar.MessageId` object that represents where the message is persisted. """ msg = self._build_msg(content, properties, partition_key, ordering_key, sequence_id, replication_clusters, disable_replication, event_timestamp, deliver_at, deliver_after) return self._producer.send(msg) def send_async(self, content, callback, properties=None, partition_key=None, ordering_key=None, sequence_id=None, replication_clusters=None, disable_replication=False, event_timestamp=None, deliver_at=None, deliver_after=None, ): """ Send a message asynchronously. Examples -------- The ``callback`` will be invoked once the message has been acknowledged by the broker. Users are responsible to handle the exception inside the callback. If any exception was thrown from the callback, the process would terminate immediately. .. code-block:: python import pulsar client = pulsar.Client('pulsar://localhost:6650') producer = client.create_producer( 'my-topic', block_if_queue_full=True, batching_enabled=True, batching_max_publish_delay_ms=10) def callback(res, msg_id): print('Message published res=%s', res) while True: producer.send_async(('Hello-%d' % i).encode('utf-8'), callback) client.close() When the producer queue is full, by default the message will be rejected and the callback invoked with an error code. Parameters ---------- content A `bytes` object with the message payload. callback A callback that is invoked once the message has been acknowledged by the broker. properties: optional A dict of application0-defined string properties. partition_key: optional Sets the partition key for the message routing. A hash of this key is used to determine the message's topic partition. ordering_key: optional Sets the ordering key for the message routing. sequence_id: optional Specify a custom sequence id for the message being published. replication_clusters: optional Override namespace replication clusters. Note that it is the caller's responsibility to provide valid cluster names and that all clusters have been previously configured as topics. Given an empty list, the message will replicate per the namespace configuration. disable_replication: optional Do not replicate this message. event_timestamp: optional Timestamp in millis of the timestamp of event creation deliver_at: optional Specify the message should not be delivered earlier than the specified timestamp. deliver_after: optional Specify a delay in timedelta for the delivery of the messages. """ msg = self._build_msg(content, properties, partition_key, ordering_key, sequence_id, replication_clusters, disable_replication, event_timestamp, deliver_at, deliver_after) self._producer.send_async(msg, callback) def flush(self): """ Flush all the messages buffered in the client and wait until all messages have been successfully persisted """ self._producer.flush() def close(self): """ Close the producer. """ self._producer.close() def _build_msg(self, content, properties, partition_key, ordering_key, sequence_id, replication_clusters, disable_replication, event_timestamp, deliver_at, deliver_after): data = self._schema.encode(content) _check_type(bytes, data, 'data') _check_type_or_none(dict, properties, 'properties') _check_type_or_none(str, partition_key, 'partition_key') _check_type_or_none(str, ordering_key, 'ordering_key') _check_type_or_none(int, sequence_id, 'sequence_id') _check_type_or_none(list, replication_clusters, 'replication_clusters') _check_type(bool, disable_replication, 'disable_replication') _check_type_or_none(int, event_timestamp, 'event_timestamp') _check_type_or_none(int, deliver_at, 'deliver_at') _check_type_or_none(timedelta, deliver_after, 'deliver_after') mb = _pulsar.MessageBuilder() mb.content(data) if properties: for k, v in properties.items(): mb.property(k, v) if partition_key: mb.partition_key(partition_key) if ordering_key: mb.ordering_key(ordering_key) if sequence_id: mb.sequence_id(sequence_id) if replication_clusters: mb.replication_clusters(replication_clusters) if disable_replication: mb.disable_replication(disable_replication) if event_timestamp: mb.event_timestamp(event_timestamp) if deliver_at: mb.deliver_at(deliver_at) if deliver_after: mb.deliver_after(deliver_after) return mb.build() def is_connected(self): """ Check if the producer is connected or not. """ return self._producer.is_connected() class Consumer: """ Pulsar consumer. Examples -------- .. code-block:: python import pulsar client = pulsar.Client('pulsar://localhost:6650') consumer = client.subscribe('my-topic', 'my-subscription') while True: msg = consumer.receive() try: print("Received message '{}' id='{}'".format(msg.data(), msg.message_id())) consumer.acknowledge(msg) except Exception: consumer.negative_acknowledge(msg) client.close() """ def topic(self): """ Return the topic this consumer is subscribed to. """ return self._consumer.topic() def subscription_name(self): """ Return the subscription name. """ return self._consumer.subscription_name() def consumer_name(self): """ Return the consumer name. """ return self._consumer.consumer_name() def unsubscribe(self): """ Unsubscribe the current consumer from the topic. This method will block until the operation is completed. Once the consumer is unsubscribed, no more messages will be received and subsequent new messages will not be retained for this consumer. This consumer object cannot be reused. """ return self._consumer.unsubscribe() def receive(self, timeout_millis=None): """ Receive a single message. If a message is not immediately available, this method will block until a new message is available. Parameters ---------- timeout_millis: int, optional If specified, the receiver will raise an exception if a message is not available within the timeout. """ if timeout_millis is None: msg = self._consumer.receive() else: _check_type(int, timeout_millis, 'timeout_millis') msg = self._consumer.receive(timeout_millis) m = Message() m._message = msg m._schema = self._schema return m def batch_receive(self): """ Batch receiving messages. This calls blocks until has enough messages or wait timeout, more details to see {@link BatchReceivePolicy}. """ messages = [] msgs = self._consumer.batch_receive() for msg in msgs: m = Message() m._message = msg m._schema = self._schema messages.append(m) return messages def acknowledge(self, message: Union[Message, MessageId, _pulsar.Message, _pulsar.MessageId]): """ Acknowledge the reception of a single message. This method will block until an acknowledgement is sent to the broker. After that, the message will not be re-delivered to this consumer. Parameters ---------- message : Message, MessageId, _pulsar.Message, _pulsar.MessageId The received message or message id. Raises ------ OperationNotSupported if ``message`` is not allowed to acknowledge """ if isinstance(message, Message): self._consumer.acknowledge(message._message) elif isinstance(message, MessageId): self._consumer.acknowledge(message._msg_id) else: self._consumer.acknowledge(message) def acknowledge_cumulative(self, message: Union[Message, MessageId, _pulsar.Message, _pulsar.MessageId]): """ Acknowledge the reception of all the messages in the stream up to (and including) the provided message. This method will block until an acknowledgement is sent to the broker. After that, the messages will not be re-delivered to this consumer. Parameters ---------- message: The received message or message id. Raises ------ CumulativeAcknowledgementNotAllowedError if the consumer type is ConsumerType.KeyShared or ConsumerType.Shared """ if isinstance(message, Message): self._consumer.acknowledge_cumulative(message._message) elif isinstance(message, MessageId): self._consumer.acknowledge_cumulative(message._msg_id) else: self._consumer.acknowledge_cumulative(message) def negative_acknowledge(self, message): """ Acknowledge the failure to process a single message. When a message is "negatively acked" it will be marked for redelivery after some fixed delay. The delay is configurable when constructing the consumer with {@link ConsumerConfiguration#setNegativeAckRedeliveryDelayMs}. This call is not blocking. Parameters ---------- message: The received message or message id. """ if isinstance(message, Message): self._consumer.negative_acknowledge(message._message) else: self._consumer.negative_acknowledge(message) def pause_message_listener(self): """ Pause receiving messages via the ``message_listener`` until `resume_message_listener()` is called. """ self._consumer.pause_message_listener() def resume_message_listener(self): """ Resume receiving the messages via the message listener. Asynchronously receive all the messages enqueued from the time `pause_message_listener()` was called. """ self._consumer.resume_message_listener() def redeliver_unacknowledged_messages(self): """ Redelivers all the unacknowledged messages. In failover mode, the request is ignored if the consumer is not active for the given topic. In shared mode, the consumer's messages to be redelivered are distributed across all the connected consumers. This is a non-blocking call and doesn't throw an exception. In case the connection breaks, the messages are redelivered after reconnect. """ self._consumer.redeliver_unacknowledged_messages() def seek(self, messageid: Union[MessageId, _pulsar.MessageId, int]): """ Reset the subscription associated with this consumer to a specific message id or publish timestamp. The message id can either be a specific message or represent the first or last messages in the topic. Note: this operation can only be done on non-partitioned topics. For these, one can rather perform the seek() on the individual partitions. Parameters ---------- messageid: MessageId, _pulsar.MessageId or int The message id for seek, OR an integer event time to seek to """ self._consumer.seek(_seek_arg_convert(messageid)) def close(self): """ Close the consumer. """ self._consumer.close() self._client._consumers.remove(self) def is_connected(self): """ Check if the consumer is connected or not. """ return self._consumer.is_connected() def get_last_message_id(self): """ Get the last message id. """ return self._consumer.get_last_message_id() class ConsumerBatchReceivePolicy: """ Batch receive policy can limit the number and bytes of messages in a single batch, and can specify a timeout for waiting for enough messages for this batch. A batch receive action is completed as long as any one of the conditions (the batch has enough number or size of messages, or the waiting timeout is passed) are met. """ def __init__(self, max_num_message, max_num_bytes, timeout_ms): """ Wrapper BatchReceivePolicy. Parameters ---------- max_num_message: Max num message, if less than 0, it means no limit. default: -1 max_num_bytes: Max num bytes, if less than 0, it means no limit. default: 10 * 1024 * 1024 timeout_ms: If less than 0, it means no limit. default: 100 """ self._policy = BatchReceivePolicy(max_num_message, max_num_bytes, timeout_ms) def policy(self): """ Returns the actual one BatchReceivePolicy. """ return self._policy class ConsumerKeySharedPolicy: """ Consumer key shared policy is used to configure the consumer behaviour when the ConsumerType is KeyShared. """ def __init__( self, key_shared_mode: KeySharedMode = KeySharedMode.AutoSplit, allow_out_of_order_delivery: bool = False, sticky_ranges: Optional[List[Tuple[int, int]]] = None, ): """ Wrapper KeySharedPolicy. Parameters ---------- key_shared_mode: KeySharedMode, optional Set the key shared mode. eg: KeySharedMode.Sticky or KeysharedMode.AutoSplit allow_out_of_order_delivery: bool, optional Set whether to allow for out of order delivery If it is enabled, it relaxes the ordering requirement and allows the broker to send out-of-order messages in case of failures. This makes it faster for new consumers to join without being stalled by an existing slow consumer. If this is True, a single consumer still receives all keys, but they may come in different orders. sticky_ranges: List[Tuple[int, int]], optional Set the ranges used with sticky mode. The integers can be from 0 to 2^16 (0 <= val < 65,536) """ if key_shared_mode == KeySharedMode.Sticky and sticky_ranges is None: raise ValueError("When using key_shared_mode = KeySharedMode.Sticky you must also provide sticky_ranges") self._policy = KeySharedPolicy() self._policy.set_key_shared_mode(key_shared_mode) self._policy.set_allow_out_of_order_delivery(allow_out_of_order_delivery) if sticky_ranges is not None: self._policy.set_sticky_ranges(sticky_ranges) @property def key_shared_mode(self) -> KeySharedMode: """ Returns the key shared mode """ return self._policy.get_key_shared_mode() @property def allow_out_of_order_delivery(self) -> bool: """ Returns whether out of order delivery is enabled """ return self._policy.is_allow_out_of_order_delivery() @property def sticky_ranges(self) -> List[Tuple[int, int]]: """ Returns the actual sticky ranges """ return self._policy.get_sticky_ranges() def policy(self): """ Returns the actual KeySharedPolicy. """ return self._policy class Reader: """ Pulsar topic reader. """ def topic(self): """ Return the topic this reader is reading from. """ return self._reader.topic() def read_next(self, timeout_millis=None): """ Read a single message. If a message is not immediately available, this method will block until a new message is available. Parameters ---------- timeout_millis: int, optional If specified, the receiver will raise an exception if a message is not available within the timeout. """ if timeout_millis is None: msg = self._reader.read_next() else: _check_type(int, timeout_millis, 'timeout_millis') msg = self._reader.read_next(timeout_millis) m = Message() m._message = msg m._schema = self._schema return m def has_message_available(self): """ Check if there is any message available to read from the current position. """ return self._reader.has_message_available(); def seek(self, messageid: Union[MessageId, _pulsar.MessageId, int]): """ Reset this reader to a specific message id or publish timestamp. The message id can either be a specific message or represent the first or last messages in the topic. Note: this operation can only be done on non-partitioned topics. For these, one can rather perform the seek() on the individual partitions. Parameters ---------- messageid: MessageId, _pulsar.MessageId or int The message id for seek, OR an integer event time to seek to """ self._reader.seek(_seek_arg_convert(messageid)) def close(self): """ Close the reader. """ self._reader.close() self._client._consumers.remove(self) def is_connected(self): """ Check if the reader is connected or not. """ return self._reader.is_connected() class ConsoleLogger: """ Logger that writes on standard output Attributes ---------- log_level: The logging level, eg: ``pulsar.LoggerLevel.Info`` """ def __init__(self, log_level=_pulsar.LoggerLevel.Info): _check_type(_pulsar.LoggerLevel, log_level, 'log_level') self.log_level = log_level class FileLogger: """ Logger that writes into a file Attributes ---------- log_level: The logging level, eg: ``pulsar.LoggerLevel.Info`` log_file: The file where to write the logs """ def __init__(self, log_level, log_file): _check_type(_pulsar.LoggerLevel, log_level, 'log_level') _check_type(str, log_file, 'log_file') self.log_level = log_level self.log_file = log_file def _check_type(var_type, var, name): if not isinstance(var, var_type): raise ValueError("Argument %s is expected to be of type '%s' and not '%s'" % (name, var_type.__name__, type(var).__name__)) def _check_type_or_none(var_type, var, name): if var is not None and not isinstance(var, var_type): raise ValueError("Argument %s is expected to be either None or of type '%s'" % (name, var_type.__name__)) def _listener_wrapper(listener, schema): def wrapper(consumer, msg): c = Consumer() c._consumer = consumer m = Message() m._message = msg m._schema = schema listener(c, m) return wrapper def _seek_arg_convert(seek_arg): if isinstance(seek_arg, MessageId): return seek_arg._msg_id elif isinstance(seek_arg, (_pulsar.MessageId, int)): return seek_arg else: raise ValueError(f"invalid seek_arg type {type(seek_arg)}") pulsar-client-python-3.12.0-candidate-0/pulsar/asyncio.py000066400000000000000000001017561520101105200232670ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # pylint: disable=no-name-in-module,c-extension-no-member,protected-access """ The Pulsar Python client APIs that work with the asyncio module. """ import asyncio import functools from datetime import timedelta from typing import Any, Callable, List, Union import _pulsar from _pulsar import ( InitialPosition, CompressionType, PartitionsRoutingMode, BatchingType, ProducerAccessMode, RegexSubscriptionMode, ConsumerCryptoFailureAction, ) import pulsar from pulsar import _check_type class PulsarException(BaseException): """ The exception that wraps the Pulsar error code """ def __init__(self, result: pulsar.Result) -> None: """ Create the Pulsar exception. Parameters ---------- result: pulsar.Result The error code of the underlying Pulsar APIs. """ self._result = result def error(self) -> pulsar.Result: """ Returns the Pulsar error code. """ return self._result def __str__(self): """ Convert the exception to string. """ return f'{self._result.value} {self._result.name}' class Producer: """ The Pulsar message producer, used to publish messages on a topic. """ def __init__(self, producer: _pulsar.Producer, schema: pulsar.schema.Schema) -> None: """ Create the producer. Users should not call this constructor directly. Instead, create the producer via `Client.create_producer`. Parameters ---------- producer: _pulsar.Producer The underlying Producer object from the C extension. schema: pulsar.schema.Schema The schema of the data that will be sent by this producer. """ self._producer = producer self._schema = schema # pylint: disable=too-many-arguments,too-many-locals,too-many-positional-arguments async def send(self, content: Any, properties: dict | None = None, partition_key: str | None = None, ordering_key: str | None = None, sequence_id: int | None = None, replication_clusters: List[str] | None = None, disable_replication: bool | None = None, event_timestamp: int | None = None, deliver_at: int | None = None, deliver_after: timedelta | None = None) -> pulsar.MessageId: """ Send a message asynchronously. parameters ---------- content: Any The message payload, whose type should respect the schema defined in `Client.create_producer`. properties: dict | None A dict of application-defined string properties. partition_key: str | None Sets the partition key for the message routing. A hash of this key is used to determine the message's topic partition. ordering_key: str | None Sets the ordering key for the message routing. sequence_id: int | None Specify a custom sequence id for the message being published. replication_clusters: List[str] | None Override namespace replication clusters. Note that it is the caller's responsibility to provide valid cluster names and that all clusters have been previously configured as topics. Given an empty list, the message will replicate per the namespace configuration. disable_replication: bool | None Do not replicate this message. event_timestamp: int | None Timestamp in millis of the timestamp of event creation deliver_at: int | None Specify the message should not be delivered earlier than the specified timestamp. deliver_after: timedelta | None Specify a delay in timedelta for the delivery of the messages. Returns ------- pulsar.MessageId The message id that represents the persisted position of the message. Raises ------ PulsarException """ builder = _pulsar.MessageBuilder() builder.content(self._schema.encode(content)) if properties is not None: for k, v in properties.items(): builder.property(k, v) if partition_key is not None: builder.partition_key(partition_key) if ordering_key is not None: builder.ordering_key(ordering_key) if sequence_id is not None: builder.sequence_id(sequence_id) if replication_clusters is not None: builder.replication_clusters(replication_clusters) if disable_replication is not None: builder.disable_replication(disable_replication) if event_timestamp is not None: builder.event_timestamp(event_timestamp) if deliver_at is not None: builder.deliver_at(deliver_at) if deliver_after is not None: builder.deliver_after(deliver_after) future = asyncio.get_running_loop().create_future() self._producer.send_async(builder.build(), functools.partial(_set_future, future)) msg_id = await future return pulsar.MessageId( msg_id.partition(), msg_id.ledger_id(), msg_id.entry_id(), msg_id.batch_index(), ) async def flush(self) -> None: """ Flush all the messages buffered in the producer asynchronously. Raises ------ PulsarException """ future = asyncio.get_running_loop().create_future() self._producer.flush_async(functools.partial(_set_future, future, value=None)) await future async def close(self) -> None: """ Close the producer. Raises ------ PulsarException """ future = asyncio.get_running_loop().create_future() self._producer.close_async(functools.partial(_set_future, future, value=None)) await future def topic(self): """ Return the topic which producer is publishing to """ return self._producer.topic() def producer_name(self): """ Return the producer name which could have been assigned by the system or specified by the client """ return self._producer.producer_name() def last_sequence_id(self): """ Return the last sequence id that was published and acknowledged by this producer. The sequence id can be either automatically assigned or custom set on the message. After recreating a producer with the same name, this will return the sequence id of the last message that was published in the previous session, or -1 if no message was ever published. """ return self._producer.last_sequence_id() def is_connected(self) -> bool: """ Check if the producer is connected or not. """ return self._producer.is_connected() class Consumer: """ The Pulsar message consumer, used to subscribe to messages from a topic. """ def __init__(self, consumer: _pulsar.Consumer, schema: pulsar.schema.Schema) -> None: """ Create the consumer. Users should not call this constructor directly. Instead, create the consumer via `Client.subscribe`. Parameters ---------- consumer: _pulsar.Consumer The underlying Consumer object from the C extension. schema: pulsar.schema.Schema The schema of the data that will be received by this consumer. """ self._consumer = consumer self._schema = schema async def receive(self) -> pulsar.Message: """ Receive a single message asynchronously. Returns ------- pulsar.Message The message received. Raises ------ PulsarException """ future = asyncio.get_running_loop().create_future() self._consumer.receive_async(functools.partial(_set_future, future)) msg = await future m = pulsar.Message() m._message = msg m._schema = self._schema return m async def acknowledge( self, message: Union[pulsar.Message, pulsar.MessageId, _pulsar.Message, _pulsar.MessageId] ) -> None: """ Acknowledge the reception of a single message asynchronously. Parameters ---------- message : Message, MessageId, _pulsar.Message, _pulsar.MessageId The received message or message id. Raises ------ PulsarException """ future = asyncio.get_running_loop().create_future() if isinstance(message, pulsar.Message): msg = message._message elif isinstance(message, pulsar.MessageId): msg = message._msg_id else: msg = message self._consumer.acknowledge_async(msg, functools.partial(_set_future, future, value=None)) await future async def acknowledge_cumulative( self, message: Union[pulsar.Message, pulsar.MessageId, _pulsar.Message, _pulsar.MessageId] ) -> None: """ Acknowledge the reception of all the messages in the stream up to (and including) the provided message asynchronously. Parameters ---------- message : Message, MessageId, _pulsar.Message, _pulsar.MessageId The received message or message id. Raises ------ PulsarException """ future = asyncio.get_running_loop().create_future() if isinstance(message, pulsar.Message): msg = message._message elif isinstance(message, pulsar.MessageId): msg = message._msg_id else: msg = message self._consumer.acknowledge_cumulative_async( msg, functools.partial(_set_future, future, value=None) ) await future async def negative_acknowledge( self, message: Union[pulsar.Message, pulsar.MessageId, _pulsar.Message, _pulsar.MessageId] ) -> None: """ Acknowledge the failure to process a single message asynchronously. When a message is "negatively acked" it will be marked for redelivery after some fixed delay. The delay is configurable when constructing the consumer with {@link ConsumerConfiguration#setNegativeAckRedeliveryDelayMs}. This call is not blocking. Parameters ---------- message: The received message or message id. """ if isinstance(message, pulsar.Message): msg = message._message elif isinstance(message, pulsar.MessageId): msg = message._msg_id else: msg = message await asyncio.to_thread(self._consumer.negative_acknowledge, msg) async def unsubscribe(self) -> None: """ Unsubscribe the current consumer from the topic asynchronously. Raises ------ PulsarException """ future = asyncio.get_running_loop().create_future() self._consumer.unsubscribe_async(functools.partial(_set_future, future, value=None)) await future async def seek(self, messageid: Union[pulsar.MessageId, int]) -> None: """ Reset the subscription associated with this consumer to a specific message id or publish timestamp asynchronously. The message id can either be a specific message or represent the first or last messages in the topic. Parameters ---------- messageid : MessageId or int The message id for seek, OR an integer event time (timestamp) to seek to Raises ------ PulsarException """ future = asyncio.get_running_loop().create_future() if isinstance(messageid, pulsar.MessageId): msg_id = messageid._msg_id elif isinstance(messageid, int): msg_id = messageid else: raise ValueError(f"invalid messageid type {type(messageid)}") self._consumer.seek_async( msg_id, functools.partial(_set_future, future, value=None) ) await future async def close(self) -> None: """ Close the consumer asynchronously. Raises ------ PulsarException """ future = asyncio.get_running_loop().create_future() self._consumer.close_async(functools.partial(_set_future, future, value=None)) await future async def get_last_message_id(self) -> _pulsar.MessageId: """ Asynchronously get the last message id. """ future = asyncio.get_running_loop().create_future() self._consumer.get_last_message_id_async(functools.partial(_set_future, future)) id = await future return id def redeliver_unacknowledged_messages(self): """ Redelivers all the unacknowledged messages. In failover mode, the request is ignored if the consumer is not active for the given topic. In shared mode, the consumer's messages to be redelivered are distributed across all the connected consumers. This is a non-blocking call and doesn't throw an exception. In case the connection breaks, the messages are redelivered after reconnect. """ self._consumer.redeliver_unacknowledged_messages() def topic(self) -> str: """ Return the topic this consumer is subscribed to. """ return self._consumer.topic() def subscription_name(self) -> str: """ Return the subscription name. """ return self._consumer.subscription_name() def consumer_name(self) -> str: """ Return the consumer name. """ return self._consumer.consumer_name() class Client: """ The asynchronous version of `pulsar.Client`. """ def __init__(self, service_url, **kwargs) -> None: """ See `pulsar.Client.__init__` """ self._client: _pulsar.Client = pulsar.Client(service_url, **kwargs)._client # pylint: disable=too-many-arguments,too-many-locals,too-many-positional-arguments async def create_producer(self, topic: str, producer_name: str | None = None, schema: pulsar.schema.Schema | None = None, initial_sequence_id: int | None = None, send_timeout_millis: int = 30000, compression_type: CompressionType = CompressionType.NONE, max_pending_messages: int = 1000, max_pending_messages_across_partitions: int = 50000, block_if_queue_full: bool = False, batching_enabled: bool = True, batching_max_messages: int = 1000, batching_max_allowed_size_in_bytes: int = 128*1024, batching_max_publish_delay_ms: int = 10, chunking_enabled: bool = False, message_routing_mode: PartitionsRoutingMode = PartitionsRoutingMode.RoundRobinDistribution, lazy_start_partitioned_producers: bool = False, properties: dict | None = None, batching_type: BatchingType = BatchingType.Default, encryption_key: str | None = None, crypto_key_reader: pulsar.CryptoKeyReader | None = None, access_mode: ProducerAccessMode = ProducerAccessMode.Shared, message_router: Callable[[pulsar.Message, int], int] | None = None, ) -> Producer: """ Create a new producer on a given topic Parameters ---------- topic: str The topic name producer_name: str | None Specify a name for the producer. If not assigned, the system will generate a globally unique name which can be accessed with `Producer.producer_name()`. When specifying a name, it is up to the user to ensure that, for a given topic, the producer name is unique across all Pulsar's clusters. schema: pulsar.schema.Schema | None, default=None Define the schema of the data that will be published by this producer. initial_sequence_id: int | None, default=None Set the baseline for the sequence ids for messages published by the producer. send_timeout_millis: int, default=30000 If a message is not acknowledged by the server before the send_timeout expires, an error will be reported. compression_type: CompressionType, default=CompressionType.NONE Set the compression type for the producer. max_pending_messages: int, default=1000 Set the max size of the queue holding the messages pending to receive an acknowledgment from the broker. max_pending_messages_across_partitions: int, default=50000 Set the max size of the queue holding the messages pending to receive an acknowledgment across partitions. block_if_queue_full: bool, default=False Set whether send operations should block when the outgoing message queue is full. batching_enabled: bool, default=True Enable automatic message batching. Note that, unlike the synchronous producer API in ``pulsar.Client.create_producer``, batching is enabled by default for the asyncio producer. batching_max_messages: int, default=1000 Maximum number of messages in a batch. batching_max_allowed_size_in_bytes: int, default=128*1024 Maximum size in bytes of a batch. batching_max_publish_delay_ms: int, default=10 The batch interval in milliseconds. chunking_enabled: bool, default=False Enable chunking of large messages. message_routing_mode: PartitionsRoutingMode, default=PartitionsRoutingMode.RoundRobinDistribution Set the message routing mode for the partitioned producer. lazy_start_partitioned_producers: bool, default=False Start partitioned producers lazily on demand. properties: dict | None, default=None Sets the properties for the producer. batching_type: BatchingType, default=BatchingType.Default Sets the batching type for the producer. encryption_key: str | None, default=None The key used for symmetric encryption. crypto_key_reader: pulsar.CryptoKeyReader | None, default=None Symmetric encryption class implementation. access_mode: ProducerAccessMode, default=ProducerAccessMode.Shared Set the type of access mode that the producer requires on the topic. message_router: Callable[[pulsar.Message, int], int] | None, default=None A custom message router function that takes a Message and the number of partitions and returns the partition index. Returns ------- Producer The producer created Raises ------ PulsarException """ if batching_enabled and chunking_enabled: raise ValueError("Batching and chunking of messages can't be enabled together.") if schema is None: schema = pulsar.schema.BytesSchema() schema.attach_client(self._client) future = asyncio.get_running_loop().create_future() conf = _pulsar.ProducerConfiguration() if producer_name is not None: conf.producer_name(producer_name) conf.schema(schema.schema_info()) if initial_sequence_id is not None: conf.initial_sequence_id(initial_sequence_id) conf.send_timeout_millis(send_timeout_millis) conf.compression_type(compression_type) conf.max_pending_messages(max_pending_messages) conf.max_pending_messages_across_partitions(max_pending_messages_across_partitions) conf.block_if_queue_full(block_if_queue_full) conf.batching_enabled(batching_enabled) conf.batching_max_messages(batching_max_messages) conf.batching_max_allowed_size_in_bytes(batching_max_allowed_size_in_bytes) conf.batching_max_publish_delay_ms(batching_max_publish_delay_ms) conf.chunking_enabled(chunking_enabled) conf.partitions_routing_mode(message_routing_mode) conf.lazy_start_partitioned_producers(lazy_start_partitioned_producers) if properties is not None: for k, v in properties.items(): conf.property(k, v) conf.batching_type(batching_type) if encryption_key is not None: conf.encryption_key(encryption_key) if crypto_key_reader is not None: conf.crypto_key_reader(crypto_key_reader.cryptoKeyReader) conf.access_mode(access_mode) if message_router is not None: def underlying_router(msg: _pulsar.Message, num_partitions: int) -> int: return message_router(pulsar.Message._wrap(msg), num_partitions) conf.message_router(underlying_router) self._client.create_producer_async( topic, conf, functools.partial(_set_future, future) ) return Producer(await future, schema) # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-positional-arguments async def subscribe(self, topic: Union[str, List[str]], subscription_name: str, consumer_type: pulsar.ConsumerType = pulsar.ConsumerType.Exclusive, schema: pulsar.schema.Schema | None = None, receiver_queue_size: int = 1000, max_total_receiver_queue_size_across_partitions: int = 50000, consumer_name: str | None = None, unacked_messages_timeout_ms: int | None = None, broker_consumer_stats_cache_time_ms: int = 30000, negative_ack_redelivery_delay_ms: int = 60000, is_read_compacted: bool = False, properties: dict | None = None, initial_position: InitialPosition = InitialPosition.Latest, crypto_key_reader: pulsar.CryptoKeyReader | None = None, replicate_subscription_state_enabled: bool = False, max_pending_chunked_message: int = 10, auto_ack_oldest_chunked_message_on_queue_full: bool = False, start_message_id_inclusive: bool = False, batch_receive_policy: pulsar.ConsumerBatchReceivePolicy | None = None, key_shared_policy: pulsar.ConsumerKeySharedPolicy | None = None, batch_index_ack_enabled: bool = False, regex_subscription_mode: RegexSubscriptionMode = RegexSubscriptionMode.PersistentOnly, dead_letter_policy: pulsar.ConsumerDeadLetterPolicy | None = None, crypto_failure_action: ConsumerCryptoFailureAction = ConsumerCryptoFailureAction.FAIL, is_pattern_topic: bool = False) -> Consumer: """ Subscribe to the given topic and subscription combination. Parameters ---------- topic: str, List[str], or regex pattern The name of the topic, list of topics or regex pattern. When `is_pattern_topic` is True, `topic` is treated as a regex. subscription_name: str The name of the subscription. consumer_type: pulsar.ConsumerType, default=pulsar.ConsumerType.Exclusive Select the subscription type to be used when subscribing to the topic. schema: pulsar.schema.Schema | None, default=None Define the schema of the data that will be received by this consumer. receiver_queue_size: int, default=1000 Sets the size of the consumer receive queue. max_total_receiver_queue_size_across_partitions: int, default=50000 Set the max total receiver queue size across partitions. consumer_name: str | None, default=None Sets the consumer name. unacked_messages_timeout_ms: int | None, default=None Sets the timeout in milliseconds for unacknowledged messages. broker_consumer_stats_cache_time_ms: int, default=30000 Sets the time duration for which the broker-side consumer stats will be cached in the client. negative_ack_redelivery_delay_ms: int, default=60000 The delay after which to redeliver the messages that failed to be processed. is_read_compacted: bool, default=False Selects whether to read the compacted version of the topic. properties: dict | None, default=None Sets the properties for the consumer. initial_position: InitialPosition, default=InitialPosition.Latest Set the initial position of a consumer when subscribing to the topic. crypto_key_reader: pulsar.CryptoKeyReader | None, default=None Symmetric encryption class implementation. replicate_subscription_state_enabled: bool, default=False Set whether the subscription status should be replicated. max_pending_chunked_message: int, default=10 Consumer buffers chunk messages into memory until it receives all the chunks. auto_ack_oldest_chunked_message_on_queue_full: bool, default=False Automatically acknowledge oldest chunked messages on queue full. start_message_id_inclusive: bool, default=False Set the consumer to include the given position of any reset operation. batch_receive_policy: pulsar.ConsumerBatchReceivePolicy | None, default=None Set the batch collection policy for batch receiving. key_shared_policy: pulsar.ConsumerKeySharedPolicy | None, default=None Set the key shared policy for use when the ConsumerType is KeyShared. batch_index_ack_enabled: bool, default=False Enable the batch index acknowledgement. regex_subscription_mode: RegexSubscriptionMode, default=RegexSubscriptionMode.PersistentOnly Set the regex subscription mode for use when the topic is a regex pattern. dead_letter_policy: pulsar.ConsumerDeadLetterPolicy | None, default=None Set dead letter policy for consumer. crypto_failure_action: ConsumerCryptoFailureAction, default=ConsumerCryptoFailureAction.FAIL Set the behavior when the decryption fails. is_pattern_topic: bool, default=False Whether `topic` is a regex pattern. If it's True when `topic` is a list, a ValueError will be raised. Returns ------- Consumer The consumer created Raises ------ PulsarException """ if schema is None: schema = pulsar.schema.BytesSchema() future = asyncio.get_running_loop().create_future() conf = _pulsar.ConsumerConfiguration() conf.consumer_type(consumer_type) conf.regex_subscription_mode(regex_subscription_mode) conf.read_compacted(is_read_compacted) conf.receiver_queue_size(receiver_queue_size) conf.max_total_receiver_queue_size_across_partitions( max_total_receiver_queue_size_across_partitions ) if consumer_name: conf.consumer_name(consumer_name) if unacked_messages_timeout_ms: conf.unacked_messages_timeout_ms(unacked_messages_timeout_ms) conf.negative_ack_redelivery_delay_ms(negative_ack_redelivery_delay_ms) conf.broker_consumer_stats_cache_time_ms(broker_consumer_stats_cache_time_ms) if properties: for k, v in properties.items(): conf.property(k, v) conf.subscription_initial_position(initial_position) conf.schema(schema.schema_info()) if crypto_key_reader: conf.crypto_key_reader(crypto_key_reader.cryptoKeyReader) conf.replicate_subscription_state_enabled(replicate_subscription_state_enabled) conf.max_pending_chunked_message(max_pending_chunked_message) conf.auto_ack_oldest_chunked_message_on_queue_full( auto_ack_oldest_chunked_message_on_queue_full ) conf.start_message_id_inclusive(start_message_id_inclusive) if batch_receive_policy: conf.batch_receive_policy(batch_receive_policy.policy()) if key_shared_policy: conf.key_shared_policy(key_shared_policy.policy()) conf.batch_index_ack_enabled(batch_index_ack_enabled) if dead_letter_policy: conf.dead_letter_policy(dead_letter_policy.policy()) conf.crypto_failure_action(crypto_failure_action) if isinstance(topic, str): if is_pattern_topic: self._client.subscribe_async_pattern( topic, subscription_name, conf, functools.partial(_set_future, future) ) else: self._client.subscribe_async( topic, subscription_name, conf, functools.partial(_set_future, future) ) elif isinstance(topic, list): if is_pattern_topic: raise ValueError( "Argument 'topic' must be a string when " "'is_pattern_topic' is True; lists of topics do not " "support pattern subscriptions" ) self._client.subscribe_async_topics( topic, subscription_name, conf, functools.partial(_set_future, future) ) else: raise ValueError( "Argument 'topic' is expected to be of type 'str' or 'list'") schema.attach_client(self._client) return Consumer(await future, schema) def shutdown(self) -> None: """ Shutdown the client and all the associated producers and consumers Raises ------ PulsarException """ self._client.shutdown() async def get_topic_partitions(self, topic: str) -> List[str]: """ Get the list of partitions for a given topic in asynchronous mode. If the topic is partitioned, this will return a list of partition names. If the topic is not partitioned, the returned list will contain the topic name itself. This can be used to discover the partitions and create Reader, Consumer or Producer instances directly on a particular partition. Parameters ---------- topic: str the topic name to lookup Returns ------- list a list of partition names """ _check_type(str, topic, 'topic') future = asyncio.get_running_loop().create_future() self._client.get_topic_partitions_async(topic, functools.partial(_set_future, future)) id = await future return id async def close(self) -> None: """ Close the client and all the associated producers and consumers Raises ------ PulsarException """ future = asyncio.get_running_loop().create_future() self._client.close_async( functools.partial(_set_future, future, value=None) ) await future def _set_future(future: asyncio.Future, result: _pulsar.Result, value: Any): def complete(): if future.done(): return if result == _pulsar.Result.Ok: future.set_result(value) else: future.set_exception(PulsarException(result)) future.get_loop().call_soon_threadsafe(complete) pulsar-client-python-3.12.0-candidate-0/pulsar/exceptions.py000066400000000000000000000033731520101105200237770ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from _pulsar import PulsarException, UnknownError, InvalidConfiguration, Timeout, LookupError, ConnectError, \ ReadError, AuthenticationError, AuthorizationError, ErrorGettingAuthenticationData, BrokerMetadataError, \ BrokerPersistenceError, ChecksumError, ConsumerBusy, NotConnected, AlreadyClosed, InvalidMessage, \ ConsumerNotInitialized, ProducerNotInitialized, ProducerBusy, TooManyLookupRequestException, InvalidTopicName, \ InvalidUrl, ServiceUnitNotReady, OperationNotSupported, ProducerBlockedQuotaExceededError, \ ProducerBlockedQuotaExceededException, ProducerQueueIsFull, MessageTooBig, TopicNotFound, SubscriptionNotFound, \ ConsumerNotFound, UnsupportedVersionError, TopicTerminated, CryptoError, IncompatibleSchema, ConsumerAssignError, \ CumulativeAcknowledgementNotAllowedError, TransactionCoordinatorNotFoundError, InvalidTxnStatusError, \ NotAllowedError, TransactionConflict, TransactionNotFound, ProducerFenced, MemoryBufferIsFull, Interrupted pulsar-client-python-3.12.0-candidate-0/pulsar/functions/000077500000000000000000000000001520101105200232465ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/pulsar/functions/__init__.py000066400000000000000000000014601520101105200253600ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # -*- encoding: utf-8 -*- pulsar-client-python-3.12.0-candidate-0/pulsar/functions/context.py000066400000000000000000000141751520101105200253140ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # """ Context defines context information available during processing of a request. """ from abc import abstractmethod class Context(object): """Interface defining information available at process time""" @abstractmethod def get_message_id(self): """Return the messageid of the current message that we are processing""" pass @abstractmethod def get_message_key(self): """Return the key of the current message that we are processing""" pass @abstractmethod def get_message_eventtime(self): """Return the event time of the current message that we are processing""" pass @abstractmethod def get_message_properties(self): """Return the message properties kv map of the current message that we are processing""" pass @abstractmethod def get_current_message_topic_name(self): """Returns the topic name of the message that we are processing""" pass @abstractmethod def get_function_tenant(self): """Returns the tenant of the message that's being processed""" pass @abstractmethod def get_function_namespace(self): """Returns the namespace of the message that's being processed""" @abstractmethod def get_function_name(self): """Returns the function name that we are a part of""" pass @abstractmethod def get_function_id(self): """Returns the function id that we are a part of""" pass @abstractmethod def get_instance_id(self): """Returns the instance id that is executing the function""" pass @abstractmethod def get_function_version(self): """Returns the version of function that we are executing""" pass @abstractmethod def get_logger(self): """Returns the logger object that can be used to do logging""" pass @abstractmethod def get_user_config_value(self, key): """Returns the value of the user-defined config. If the key doesn't exist, None is returned""" pass @abstractmethod def get_user_config_map(self): """Returns the entire user-defined config as a dict (the dict will be empty if no user-defined config is supplied)""" pass @abstractmethod def get_secret(self, secret_name): """Returns the secret value associated with the name. None if nothing was found""" pass @abstractmethod def get_partition_key(self): """Returns partition key of the input message is one exists""" pass @abstractmethod def get_ordering_key(self): """Returns ordering key of the input message, if one exists""" pass @abstractmethod def record_metric(self, metric_name, metric_value): """Records the metric_value. metric_value has to satisfy isinstance(metric_value, numbers.Number)""" pass @abstractmethod def publish(self, topic_name, message, serde_class_name="serde.IdentitySerDe", properties=None, compression_type=None, callback=None, message_conf=None): """Publishes message to topic_name by first serializing the message using serde_class_name serde The message will have properties specified if any The available options for message_conf: properties, partition_key, ordering_key, sequence_id, replication_clusters, disable_replication, event_timestamp """ pass @abstractmethod def get_input_topics(self): """Returns the input topics of function""" pass @abstractmethod def get_output_topic(self): """Returns the output topic of function""" pass @abstractmethod def get_output_serde_class_name(self): """return output Serde class""" pass @abstractmethod def ack(self, msgid, topic): """ack this message id""" pass @abstractmethod def incr_counter(self, key, amount): """incr the counter of a given key in the managed state""" pass @abstractmethod def get_counter(self, key): """get the counter of a given key in the managed state""" pass @abstractmethod def del_counter(self, key): """delete the counter of a given key in the managed state""" pass @abstractmethod def put_state(self, key, value): """update the value of a given key in the managed state""" pass @abstractmethod def get_state(self, key): """get the value of a given key in the managed state""" pass pulsar-client-python-3.12.0-candidate-0/pulsar/functions/function.py000066400000000000000000000040141520101105200254440ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # """ This is the core interface of the function api. The process method is called for every message of the input topic of the function. The incoming input bytes are deserialized using the serde. The process function can optionally emit an output """ from abc import abstractmethod class Function(object): """Interface for Pulsar Function""" @abstractmethod def process(self, input, context): """Process input message""" pass pulsar-client-python-3.12.0-candidate-0/pulsar/functions/serde.py000066400000000000000000000061101520101105200247200ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # -*- encoding: utf-8 -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # """ SerDe defines the interface for serialization/deserialization. Every time a message is read from pulsar topic, the serde is invoked to serialize the bytes into an object before invoking the process method. Anytime a python object needs to be written back to pulsar, it is serialized into bytes before writing. """ import pickle from abc import abstractmethod class SerDe(object): """Interface for Serialization/Deserialization""" @abstractmethod def serialize(self, input): """Serialize input message into bytes""" pass @abstractmethod def deserialize(self, input_bytes): """Serialize input_bytes into an object""" pass class PickleSerDe(SerDe): """Pickle based serializer""" def serialize(self, input): return pickle.dumps(input) def deserialize(self, input_bytes): return pickle.loads(input_bytes) class IdentitySerDe(SerDe): """Simple Serde that just conversion to string and back""" def __init__(self): self._types = [int, float, complex, str] def serialize(self, input): if type(input) in self._types: return str(input).encode('utf-8') if type(input) == bytes: return input raise TypeError("IdentitySerde cannot serialize object of type %s" % type(input)) def deserialize(self, input_bytes): for typ in self._types: try: return typ(input_bytes.decode('utf-8')) except: pass return input_bytes pulsar-client-python-3.12.0-candidate-0/pulsar/schema/000077500000000000000000000000001520101105200224765ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/pulsar/schema/__init__.py000066400000000000000000000020671520101105200246140ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from .definition import Record, Field, Null, Boolean, Integer, Long, \ Float, Double, Bytes, String, Array, Map, CustomEnum from .schema import Schema, BytesSchema, StringSchema, JsonSchema from .schema_avro import AvroSchema from .schema_protobuf import ProtobufNativeSchema pulsar-client-python-3.12.0-candidate-0/pulsar/schema/definition.py000066400000000000000000000376721520101105200252170ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import copy from abc import abstractmethod from collections import OrderedDict from enum import Enum, EnumMeta def _string_representation(x): if hasattr(x, "__name__"): return x.__name__ else: return str(x) def _check_record_or_field(x): if (type(x) is type and not issubclass(x, Record)) \ and not isinstance(x, Field): raise Exception('Argument ' + _string_representation(x) + ' is not a Record or a Field') class RecordMeta(type): def __new__(metacls, name, parents, dct): if name != 'Record': # Do not apply this logic to the base class itself dct['_fields'] = RecordMeta._get_fields(dct) dct['_required'] = False return type.__new__(metacls, name, parents, dct) @classmethod def _get_fields(cls, dct): # Build a set of valid fields for this record fields = OrderedDict() for name, value in dct.items(): if issubclass(type(value), EnumMeta): value = CustomEnum(value) elif type(value) == RecordMeta: # We expect an instance of a record rather than the class itself value = value() if isinstance(value, Record) or isinstance(value, Field): fields[name] = value return fields class Record(metaclass=RecordMeta): # This field is used to set namespace for Avro Record schema. _avro_namespace = None # Generate a schema where fields are sorted alphabetically _sorted_fields = False def __init__(self, default=None, required_default=False, required=False, *args, **kwargs): self._required_default = required_default self._default = default self._required = required for k, value in self._fields.items(): if k in kwargs: if isinstance(value, Record) and isinstance(kwargs[k], dict): # Use dict init Record object copied = copy.copy(value) copied.__init__(**kwargs[k]) self.__setattr__(k, copied) elif isinstance(value, Array) and isinstance(kwargs[k], list) and len(kwargs[k]) > 0 \ and isinstance(value.array_type, Record) and isinstance(kwargs[k][0], dict): arr = [] for item in kwargs[k]: copied = copy.copy(value.array_type) copied.__init__(**item) arr.append(copied) self.__setattr__(k, arr) elif isinstance(value, Map) and isinstance(kwargs[k], dict) and len(kwargs[k]) > 0 \ and isinstance(value.value_type, Record) and isinstance(list(kwargs[k].values())[0], dict): dic = {} for mapKey, mapValue in kwargs[k].items(): copied = copy.copy(value.value_type) copied.__init__(**mapValue) dic[mapKey] = copied self.__setattr__(k, dic) else: # Value was overridden at constructor self.__setattr__(k, kwargs[k]) elif isinstance(value, Record): # Value is a subrecord self.__setattr__(k, value) else: # Set field to default value, without revalidating the default value type super(Record, self).__setattr__(k, value.default()) @classmethod def schema(cls): return cls.schema_info(set()) @classmethod def schema_info(cls, defined_names): namespace_prefix = '' if cls._avro_namespace is not None: namespace_prefix = cls._avro_namespace + '.' namespace_name = namespace_prefix + cls.__name__ if namespace_name in defined_names: return namespace_name defined_names.add(namespace_name) schema = { 'type': 'record', 'name': str(cls.__name__) } if cls._avro_namespace is not None: schema['namespace'] = cls._avro_namespace schema['fields'] = [] def get_filed_default_value(value): if isinstance(value, Enum): return value.name else: return value if cls._sorted_fields: fields = sorted(cls._fields.keys()) else: fields = cls._fields.keys() for name in fields: field = cls._fields[name] field_type = field.schema_info(defined_names) \ if field._required else ['null', field.schema_info(defined_names)] schema['fields'].append({ 'name': name, 'default': get_filed_default_value(field.default()), 'type': field_type }) if field.required_default() else schema['fields'].append({ 'name': name, 'type': field_type, }) return schema def __setattr__(self, key, value): if key == '_default': super(Record, self).__setattr__(key, value) elif key == '_required_default': super(Record, self).__setattr__(key, value) elif key == '_required': super(Record, self).__setattr__(key, value) else: if key not in self._fields: raise AttributeError('Cannot set undeclared field ' + key + ' on record') # Check that type of value matches the field type field = self._fields[key] value = field.validate_type(key, value) super(Record, self).__setattr__(key, value) def __eq__(self, other): for field in self._fields: if self.__getattribute__(field) != other.__getattribute__(field): return False return True def __ne__(self, other): return not self.__eq__(other) def __str__(self): return str(self.__dict__) def type(self): return str(self.__class__.__name__) def python_type(self): return self.__class__ def validate_type(self, name, val): if val is None and not self._required: return self.default() if not isinstance(val, self.__class__): raise TypeError("Invalid type '%s' for sub-record field '%s'. Expected: %s" % ( type(val), name, _string_representation(self.__class__))) return val def default(self): if self._default is not None: return self._default else: return None def required_default(self): return self._required_default class Field(object): def __init__(self, default=None, required=False, required_default=False): if default is not None: default = self.validate_type('default', default) self._default = default self._required_default = required_default self._required = required @abstractmethod def type(self): pass @abstractmethod def python_type(self): pass def validate_type(self, name, val): if val is None and not self._required: return self.default() if not isinstance(val, self.python_type()): raise TypeError("Invalid type '%s' for field '%s'. Expected: %s" % (type(val), name, _string_representation(self.python_type()))) return val def schema(self): # For primitive types, the schema would just be the type itself return self.type() def schema_info(self, defined_names): return self.type() def default(self): return self._default def required_default(self): return self._required_default # All types class Null(Field): def type(self): return 'null' def python_type(self): return type(None) def validate_type(self, name, val): if val is not None: raise TypeError('Field ' + name + ' is set to be None') return val class Boolean(Field): def type(self): return 'boolean' def python_type(self): return bool def default(self): if self._default is not None: return self._default else: return False class Integer(Field): def type(self): return 'int' def python_type(self): return int def default(self): if self._default is not None: return self._default else: return None class Long(Field): def type(self): return 'long' def python_type(self): return int def default(self): if self._default is not None: return self._default else: return None class Float(Field): def type(self): return 'float' def python_type(self): return float, int def default(self): if self._default is not None: return self._default else: return None class Double(Field): def type(self): return 'double' def python_type(self): return float, int def default(self): if self._default is not None: return self._default else: return None class Bytes(Field): def type(self): return 'bytes' def python_type(self): return bytes, str def default(self): if self._default is not None: return self._default else: return None def validate_type(self, name, val): if isinstance(val, str): return val.encode() return val class String(Field): def type(self): return 'string' def python_type(self): return str, bytes def validate_type(self, name, val): t = type(val) if val is None and not self._required: return self.default() if not (isinstance(val, (str, bytes)) or t.__name__ == 'unicode'): raise TypeError("Invalid type '%s' for field '%s'. Expected a string" % (t, name)) if isinstance(val, bytes): return val.decode() return val def default(self): if self._default is not None: return self._default else: return None # Complex types class CustomEnum(Field): def __init__(self, enum_type, default=None, required=False, required_default=False): if not issubclass(enum_type, Enum): raise Exception(_string_representation(enum_type) + " is not a valid Enum type") self.enum_type = enum_type self.values = {} for x in enum_type.__members__.values(): self.values[x.value] = x super(CustomEnum, self).__init__(default, required, required_default) def type(self): return 'enum' def python_type(self): return self.enum_type def validate_type(self, name, val): if val is None: return None if type(val) is str: # The enum was passed as a string, we need to check it against the possible values if val in self.enum_type.__members__: return self.enum_type.__members__[val] else: raise TypeError( "Invalid enum value '%s' for field '%s'. Expected: %s" % (val, name, self.enum_type.__members__.keys())) elif type(val) is int: # The enum was passed as an int, we need to check it against the possible values if val in self.values: return self.values[val] else: raise TypeError( "Invalid enum value '%s' for field '%s'. Expected: %s" % (val, name, self.values.keys())) elif not isinstance(val, self.python_type()): raise TypeError("Invalid type '%s' for field '%s'. Expected: %s" % (type(val), name, _string_representation(self.python_type()))) else: return val def schema(self): return self.schema_info(set()) def schema_info(self, defined_names): if self.enum_type.__name__ in defined_names: return self.enum_type.__name__ defined_names.add(self.enum_type.__name__) return { 'type': self.type(), 'name': self.enum_type.__name__, 'symbols': [x.name for x in self.enum_type] } def default(self): if self._default is not None: return self._default else: return None class Array(Field): def __init__(self, array_type, default=None, required=False, required_default=False): _check_record_or_field(array_type) self.array_type = array_type super(Array, self).__init__(default=default, required=required, required_default=required_default) def type(self): return 'array' def python_type(self): return list def validate_type(self, name, val): if val is None: return None super(Array, self).validate_type(name, val) for x in val: if not isinstance(x, self.array_type.python_type()): raise TypeError('Array field ' + name + ' items should all be of type ' + _string_representation(self.array_type.type())) return val def schema(self): return self.schema_info(set()) def schema_info(self, defined_names): return { 'type': self.type(), 'items': self.array_type.schema_info(defined_names) if isinstance(self.array_type, (Array, Map, Record)) else self.array_type.type() } def default(self): if self._default is not None: return self._default else: return None class Map(Field): def __init__(self, value_type, default=None, required=False, required_default=False): _check_record_or_field(value_type) self.value_type = value_type super(Map, self).__init__(default=default, required=required, required_default=required_default) def type(self): return 'map' def python_type(self): return dict def validate_type(self, name, val): if val is None: return None super(Map, self).validate_type(name, val) for k, v in val.items(): if type(k) != str and not is_unicode(k): raise TypeError('Map keys for field ' + name + ' should all be strings') if not isinstance(v, self.value_type.python_type()): raise TypeError('Map values for field ' + name + ' should all be of type ' + _string_representation(self.value_type.python_type())) return val def schema(self): return self.schema_info(set()) def schema_info(self, defined_names): return { 'type': self.type(), 'values': self.value_type.schema_info(defined_names) if isinstance(self.value_type, (Array, Map, Record)) else self.value_type.type() } def default(self): if self._default is not None: return self._default else: return None # Python3 has no `unicode` type, so here we use a tricky way to check if the type of `x` is `unicode` in Python2 # and also make it work well with Python3. def is_unicode(x): return 'encode' in dir(x) and type(x.encode()) == str pulsar-client-python-3.12.0-candidate-0/pulsar/schema/schema.py000066400000000000000000000070321520101105200243120ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from abc import abstractmethod import json import _pulsar import enum class Schema(object): def __init__(self, record_cls, schema_type, schema_definition, schema_name): self._record_cls = record_cls self._schema_info = _pulsar.SchemaInfo(schema_type, schema_name, json.dumps(schema_definition, indent=True)) @abstractmethod def encode(self, obj): pass @abstractmethod def decode(self, data): pass def decode_message(self, msg: _pulsar.Message): return self.decode(msg.data()) def schema_info(self): return self._schema_info def attach_client(self, client: _pulsar.Client): self._client = client def _validate_object_type(self, obj): if not isinstance(obj, self._record_cls): raise TypeError('Invalid record obj of type ' + str(type(obj)) + ' - expected type is ' + str(self._record_cls)) class BytesSchema(Schema): def __init__(self): super(BytesSchema, self).__init__(bytes, _pulsar.SchemaType.BYTES, None, 'BYTES') def encode(self, data): self._validate_object_type(data) return data def decode(self, data): return data def __str__(self): return 'BytesSchema' class StringSchema(Schema): def __init__(self): super(StringSchema, self).__init__(str, _pulsar.SchemaType.STRING, None, 'STRING') def encode(self, obj): self._validate_object_type(obj) return obj.encode('utf-8') def decode(self, data): return data.decode('utf-8') def __str__(self): return 'StringSchema' def remove_reserved_key(data): if '_default' in data: del data['_default'] if '_required' in data: del data['_required'] if '_required_default' in data: del data['_required_default'] class JsonSchema(Schema): def __init__(self, record_cls): super(JsonSchema, self).__init__(record_cls, _pulsar.SchemaType.JSON, record_cls.schema(), 'JSON') def _get_serialized_value(self, o): if isinstance(o, enum.Enum): return o.value elif isinstance(o, bytes): return o.decode() else: data = o.__dict__.copy() remove_reserved_key(data) return data def encode(self, obj): self._validate_object_type(obj) # Copy the dict of the object as to not modify the provided object via the reference provided data = obj.__dict__.copy() remove_reserved_key(data) return json.dumps(data, default=self._get_serialized_value, indent=True).encode('utf-8') def decode(self, data): return self._record_cls(**json.loads(data)) pulsar-client-python-3.12.0-candidate-0/pulsar/schema/schema_avro.py000066400000000000000000000127341520101105200253460ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import _pulsar import io import json import logging import enum from . import Record from .schema import Schema try: import fastavro HAS_AVRO = True except ImportError: HAS_AVRO = False if HAS_AVRO: class AvroSchema(Schema): def __init__(self, record_cls, schema_definition=None): if record_cls is None and schema_definition is None: raise AssertionError("The param record_cls and schema_definition shouldn't be both None.") if record_cls is not None: self._schema = record_cls.schema() else: self._schema = schema_definition self._writer_schemas = dict() self._logger = logging.getLogger() super(AvroSchema, self).__init__(record_cls, _pulsar.SchemaType.AVRO, self._schema, 'AVRO') def _get_serialized_value(self, x): if isinstance(x, enum.Enum): return x.name elif isinstance(x, Record): return self.encode_dict(x.__dict__) elif isinstance(x, list): arr = [] for item in x: arr.append(self._get_serialized_value(item)) return arr elif isinstance(x, dict): return self.encode_dict(x) else: return x def encode(self, obj): buffer = io.BytesIO() m = obj if self._record_cls is not None: self._validate_object_type(obj) m = self.encode_dict(obj.__dict__) elif not isinstance(obj, dict): raise ValueError('If using the custom schema, the record data should be dict type.') fastavro.schemaless_writer(buffer, self._schema, m) return buffer.getvalue() def encode_dict(self, d): obj = {} for k, v in d.items(): obj[k] = self._get_serialized_value(v) return obj def decode(self, data): return self._decode_bytes(data, self._schema) def decode_message(self, msg: _pulsar.Message): if self._client is None: return self.decode(msg.data()) topic = msg.topic_name() version = msg.int_schema_version() try: writer_schema = self._get_writer_schema(topic, version) return self._decode_bytes(msg.data(), writer_schema) except Exception as e: msg_id = msg.message_id() self._logger.warn(f'Failed to decode {msg_id} with schema {topic} version {version}: {e}') return self._decode_bytes(msg.data(), self._schema) def _get_writer_schema(self, topic: str, version: int) -> 'dict': if self._writer_schemas.get(topic) is None: self._writer_schemas[topic] = dict() writer_schema = self._writer_schemas[topic].get(version) if writer_schema is not None: return writer_schema if self._client is None: return self._schema self._logger.info('Downloading schema of %s version %d...', topic, version) info = self._client.get_schema_info(topic, version) self._logger.info('Downloaded schema of %s version %d', topic, version) if info.schema_type() != _pulsar.SchemaType.AVRO: raise RuntimeError(f'The schema type of topic "{topic}" and version {version}' f' is {info.schema_type()}') writer_schema = json.loads(info.schema()) self._writer_schemas[topic][version] = writer_schema return writer_schema def _decode_bytes(self, data: bytes, writer_schema: dict): buffer = io.BytesIO(data) # If the record names are different between the writer schema and the reader schema, # schemaless_reader will fail with fastavro._read_common.SchemaResolutionError. # So we make the record name fields consistent here. reader_schema: dict = self._schema writer_schema['name'] = reader_schema['name'] d = fastavro.schemaless_reader(buffer, writer_schema, reader_schema) if self._record_cls is not None: return self._record_cls(**d) else: return d else: class AvroSchema(Schema): def __init__(self, _record_cls, _schema_definition=None): raise Exception("Avro library support was not found. Make sure to install Pulsar client " + "with Avro support: pip3 install 'pulsar-client[avro]'") def encode(self, obj): pass def decode(self, data): pass pulsar-client-python-3.12.0-candidate-0/pulsar/schema/schema_protobuf.py000066400000000000000000000117031520101105200262320ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import base64 import _pulsar from .schema import Schema try: from google.protobuf import descriptor_pb2 from google.protobuf.message import Message as ProtobufMessage HAS_PROTOBUF = True except ImportError: HAS_PROTOBUF = False def _collect_file_descriptors(file_descriptor, visited, file_descriptor_set): """Recursively collect all FileDescriptorProto objects into file_descriptor_set.""" if file_descriptor.name in visited: return for dep in file_descriptor.dependencies: _collect_file_descriptors(dep, visited, file_descriptor_set) visited.add(file_descriptor.name) proto = descriptor_pb2.FileDescriptorProto() file_descriptor.CopyToProto(proto) file_descriptor_set.file.append(proto) def _build_schema_definition(descriptor): """ Build the schema definition dict used by Java's ``ProtobufNativeSchemaData``. The returned mapping has these keys: .. code-block:: text fileDescriptorSet rootMessageTypeName rootFileDescriptorName ``fileDescriptorSet`` contains base64-encoded ``FileDescriptorSet`` bytes. This mirrors ``ProtobufNativeSchemaUtils.serialize()`` in the Java client. """ file_descriptor_set = descriptor_pb2.FileDescriptorSet() _collect_file_descriptors(descriptor.file, set(), file_descriptor_set) file_descriptor_set_bytes = file_descriptor_set.SerializeToString() return { "fileDescriptorSet": base64.b64encode(file_descriptor_set_bytes).decode('utf-8'), "rootMessageTypeName": descriptor.full_name, "rootFileDescriptorName": descriptor.file.name, } if HAS_PROTOBUF: class ProtobufNativeSchema(Schema): """ Schema for protobuf messages using the native protobuf binary encoding. The schema definition is stored as a JSON-encoded ProtobufNativeSchemaData (fileDescriptorSet, rootMessageTypeName, rootFileDescriptorName), which is compatible with the Java client's ProtobufNativeSchema. Parameters ---------- record_cls: A generated protobuf message class (subclass of google.protobuf.message.Message). Example ------- .. code-block:: python import pulsar from pulsar.schema import ProtobufNativeSchema from my_proto_pb2 import MyMessage client = pulsar.Client('pulsar://localhost:6650') schema = ProtobufNativeSchema(MyMessage) producer = client.create_producer('my-topic', schema=schema) consumer = client.subscribe('my-topic', 'my-sub', schema=schema) message = MyMessage() message.field = 'value' producer.send(message) received = consumer.receive(timeout_millis=5000) typed_value = received.value() consumer.acknowledge(received) assert isinstance(typed_value, MyMessage) assert typed_value.field == 'value' consumer.close() producer.close() client.close() """ def __init__(self, record_cls): if not (isinstance(record_cls, type) and issubclass(record_cls, ProtobufMessage)): raise TypeError( f'record_cls must be a protobuf Message subclass, got {record_cls!r}' ) schema_definition = _build_schema_definition(record_cls.DESCRIPTOR) super(ProtobufNativeSchema, self).__init__( record_cls, _pulsar.SchemaType.PROTOBUF_NATIVE, schema_definition, 'PROTOBUF_NATIVE' ) def encode(self, obj): self._validate_object_type(obj) return obj.SerializeToString() def decode(self, data): return self._record_cls.FromString(data) def __str__(self): return f'ProtobufNativeSchema({self._record_cls.__name__})' else: class ProtobufNativeSchema(Schema): def __init__(self, _record_cls=None): raise Exception( "protobuf library support was not found. " "Install it with: pip install protobuf" ) def encode(self, obj): pass def decode(self, data): pass pulsar-client-python-3.12.0-candidate-0/pulsar/tableview.py000066400000000000000000000064041520101105200235760ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # """ The TableView implementation. """ from typing import Any, Callable, Optional from pulsar.schema.schema import Schema import _pulsar class TableView(): def __init__(self, table_view: _pulsar.TableView, topic: str, subscription: Optional[str], schema: Schema) -> None: self._table_view = table_view self._topic = topic self._subscription = subscription self._schema = schema def get(self, key: str) -> Optional[Any]: """ Return the value associated with the given key in the table view. Parameters ---------- key: str The message key Returns ------- Optional[Any] The value associated with the key, or None if the key does not exist. """ pair = self._table_view.get(key) if pair[0]: return self._schema.decode(pair[1]) else: return None def for_each(self, callback: Callable[[str, Any], None]) -> None: """ Iterate over all entries in the table view and call the callback function with the key and value for each entry. Parameters ---------- callback: Callable[[str, Any], None] The callback function to call for each entry. """ self._table_view.for_each(lambda k, v: callback(k, self._schema.decode(v))) def for_each_and_listen(self, callback: Callable[[str, Any], None]) -> None: """ Iterate over all entries in the table view and call the callback function with the key and value for each entry, then listen for changes. The callback will be called when a new entry is added or an existing entry is updated. Parameters ---------- callback: Callable[[str, Any], None] The callback function to call for each entry. """ self._table_view.for_each_and_listen(lambda k, v: callback(k, self._schema.decode(v))) def close(self) -> None: """ Close the table view. """ self._table_view.close() def __len__(self) -> int: """ Return the number of entries in the table view. """ return self._table_view.size() def __str__(self) -> str: if self._subscription is None: return f"TableView(topic={self._topic})" else: return f"TableView(topic={self._topic}, subscription={self._subscription})" def __repr__(self) -> str: return self.__str__() pulsar-client-python-3.12.0-candidate-0/pybind11/000077500000000000000000000000001520101105200213575ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/setup.py000077500000000000000000000062631520101105200214540ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from setuptools import setup from distutils.core import Extension from os import environ, path import platform from distutils.command import build_ext def get_version(): root = path.dirname(path.realpath(__file__)) version_file = path.join(root, 'pulsar', '__about__.py') version = {} with open(version_file) as fp: exec(fp.read(), version) return version['__version__'] def get_name(): postfix = environ.get('NAME_POSTFIX', '') base = 'pulsar-client' return base + postfix VERSION = get_version() NAME = get_name() print('NAME: %s' % NAME) print('VERSION: %s' % VERSION) # This is a workaround to have setuptools to include # the already compiled _pulsar.so library class my_build_ext(build_ext.build_ext): def build_extension(self, ext): import shutil import os.path try: os.makedirs(os.path.dirname(self.get_ext_fullpath(ext.name))) except OSError as e: if e.errno != 17: # already exists raise if 'Windows' in platform.platform(): shutil.copyfile('_pulsar.pyd', self.get_ext_fullpath(ext.name)) else: try: shutil.copyfile('_pulsar.so', self.get_ext_fullpath(ext.name)) except FileNotFoundError: shutil.copyfile('lib_pulsar.so', self.get_ext_fullpath(ext.name)) # Core Client dependencies dependencies = [ 'certifi', ] extras_require = {} # protobuf schema dependencies extras_require["protobuf"] = sorted( { "protobuf>=6.33.6", } ) # functions dependencies extras_require["functions"] = sorted( { "grpcio>=1.59.3", "apache-bookkeeper-client>=4.16.1", "prometheus_client", "ratelimit", *extras_require["protobuf"], } ) # avro dependencies extras_require["avro"] = sorted( { "fastavro>=1.9.2" } ) # all dependencies extras_require["all"] = sorted(set(sum(extras_require.values(), []))) setup( name=NAME, version=VERSION, packages=['pulsar', 'pulsar.schema', 'pulsar.functions'], cmdclass={'build_ext': my_build_ext}, ext_modules=[Extension('_pulsar', [])], author="Pulsar Devs", author_email="dev@pulsar.apache.org", description="Apache Pulsar Python client library", license="Apache License v2.0", url="https://pulsar.apache.org/", install_requires=dependencies, extras_require=extras_require, ) pulsar-client-python-3.12.0-candidate-0/src/000077500000000000000000000000001520101105200205175ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/src/authentication.cc000066400000000000000000000060121520101105200240440ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include #include #include #include namespace py = pybind11; using namespace pulsar; void export_authentication(py::module_& m) { using namespace py; class_>(m, "Authentication") .def("getAuthMethodName", &Authentication::getAuthMethodName) .def("getAuthData", &Authentication::getAuthData) .def_static("create", static_cast( &AuthFactory::create)); class_>(m, "AuthenticationTLS") .def(init()) .def_static("create", static_cast( &AuthTls::create)); class_>(m, "AuthenticationToken") .def(init()) .def_static("create", static_cast(&AuthToken::create)) .def_static("create", static_cast(&AuthToken::create)); class_>(m, "AuthenticationAthenz") .def(init()) .def_static("create", static_cast(&AuthAthenz::create)); class_>(m, "AuthenticationOauth2") .def(init()) .def_static("create", static_cast(&AuthOauth2::create)); class_>(m, "AuthenticationBasic") .def(init()) .def_static("create", static_cast(&AuthBasic::create)) .def_static("create", static_cast(&AuthBasic::create)); } pulsar-client-python-3.12.0-candidate-0/src/client.cc000066400000000000000000000231361520101105200223110ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include #include #include #include #include #include #include namespace py = pybind11; static ServiceInfo unwrapPythonServiceInfo(const py::handle& object) { auto serviceInfoObject = py::reinterpret_borrow(object); try { return serviceInfoObject.cast(); } catch (const py::cast_error&) { } if (py::hasattr(serviceInfoObject, "_service_info")) { try { return serviceInfoObject.attr("_service_info").cast(); } catch (const py::cast_error&) { } } throw py::value_error("Expected a pulsar.ServiceInfo or _pulsar.ServiceInfo instance"); } class PythonServiceInfoProvider : public ServiceInfoProvider { public: explicit PythonServiceInfoProvider(py::object provider) : provider_(std::move(provider)) {} ~PythonServiceInfoProvider() override { if (!Py_IsInitialized()) { return; } py::gil_scoped_acquire acquire; try { if (py::hasattr(provider_, "close")) { provider_.attr("close")(); } } catch (const py::error_already_set&) { PyErr_Print(); } } ServiceInfo initialServiceInfo() override { py::gil_scoped_acquire acquire; return unwrapPythonServiceInfo(provider_.attr("initial_service_info")()); } void initialize(std::function onServiceInfoUpdate) override { py::gil_scoped_acquire acquire; provider_.attr("initialize")(py::cpp_function( [onServiceInfoUpdate = std::move(onServiceInfoUpdate)](py::object serviceInfo) mutable { onServiceInfoUpdate(unwrapPythonServiceInfo(serviceInfo)); })); } private: py::object provider_; }; Producer Client_createProducer(Client& client, const std::string& topic, const ProducerConfiguration& conf) { return waitForAsyncValue( [&](CreateProducerCallback callback) { client.createProducerAsync(topic, conf, callback); }); } void Client_createProducerAsync(Client& client, const std::string& topic, ProducerConfiguration conf, CreateProducerCallback callback) { py::gil_scoped_release release; client.createProducerAsync(topic, conf, callback); } Consumer Client_subscribe(Client& client, const std::string& topic, const std::string& subscriptionName, const ConsumerConfiguration& conf) { return waitForAsyncValue( [&](SubscribeCallback callback) { client.subscribeAsync(topic, subscriptionName, conf, callback); }); } Consumer Client_subscribe_topics(Client& client, const std::vector& topics, const std::string& subscriptionName, const ConsumerConfiguration& conf) { return waitForAsyncValue( [&](SubscribeCallback callback) { client.subscribeAsync(topics, subscriptionName, conf, callback); }); } Consumer Client_subscribe_pattern(Client& client, const std::string& topic_pattern, const std::string& subscriptionName, const ConsumerConfiguration& conf) { return waitForAsyncValue([&](SubscribeCallback callback) { client.subscribeWithRegexAsync(topic_pattern, subscriptionName, conf, callback); }); } Reader Client_createReader(Client& client, const std::string& topic, const MessageId& startMessageId, const ReaderConfiguration& conf) { return waitForAsyncValue( [&](ReaderCallback callback) { client.createReaderAsync(topic, startMessageId, conf, callback); }); } std::vector Client_getTopicPartitions(Client& client, const std::string& topic) { return waitForAsyncValue>( [&](GetPartitionsCallback callback) { client.getPartitionsForTopicAsync(topic, callback); }); } void Client_getTopicPartitionsAsync(Client& client, const std::string& topic, GetPartitionsCallback callback) { py::gil_scoped_release release; client.getPartitionsForTopicAsync(topic, callback); } SchemaInfo Client_getSchemaInfo(Client& client, const std::string& topic, int64_t version) { return waitForAsyncValue([&](std::function callback) { client.getSchemaInfoAsync(topic, version, callback); }); } std::shared_ptr Client_createAutoClusterFailover(ServiceInfo primary, std::vector secondary, int64_t checkIntervalMs, uint32_t failoverThreshold, uint32_t switchBackThreshold, const ClientConfiguration& conf) { AutoClusterFailover::Config autoClusterFailoverConfig(std::move(primary), std::move(secondary)); autoClusterFailoverConfig.checkInterval = std::chrono::milliseconds(checkIntervalMs); autoClusterFailoverConfig.failoverThreshold = failoverThreshold; autoClusterFailoverConfig.switchBackThreshold = switchBackThreshold; return std::make_shared( Client::create(std::make_unique(std::move(autoClusterFailoverConfig)), conf)); } std::shared_ptr Client_createServiceInfoProvider(py::object provider, const ClientConfiguration& conf) { return std::make_shared( Client::create(std::make_unique(std::move(provider)), conf)); } void Client_close(Client& client) { waitForAsyncResult([&](ResultCallback callback) { client.closeAsync(callback); }); } void Client_closeAsync(Client& client, ResultCallback callback) { py::gil_scoped_release release; client.closeAsync(callback); } void Client_subscribeAsync(Client& client, const std::string& topic, const std::string& subscriptionName, ConsumerConfiguration conf, SubscribeCallback callback) { py::gil_scoped_release release; client.subscribeAsync(topic, subscriptionName, conf, callback); } void Client_subscribeAsync_topics(Client& client, const std::vector& topics, const std::string& subscriptionName, ConsumerConfiguration conf, SubscribeCallback callback) { py::gil_scoped_release release; client.subscribeAsync(topics, subscriptionName, conf, callback); } void Client_subscribeAsync_pattern(Client& client, const std::string& topic_pattern, const std::string& subscriptionName, ConsumerConfiguration conf, SubscribeCallback callback) { py::gil_scoped_release release; client.subscribeWithRegexAsync(topic_pattern, subscriptionName, conf, callback); } void export_client(py::module_& m) { py::class_>(m, "Client") .def(py::init()) .def_static("create_auto_cluster_failover", &Client_createAutoClusterFailover, py::arg("primary"), py::arg("secondary"), py::arg("check_interval_ms"), py::arg("failover_threshold"), py::arg("switch_back_threshold"), py::arg("client_configuration")) .def_static("create_service_info_provider", &Client_createServiceInfoProvider, py::arg("provider"), py::arg("client_configuration")) .def("create_producer", &Client_createProducer) .def("create_producer_async", &Client_createProducerAsync) .def("subscribe", &Client_subscribe) .def("subscribe_topics", &Client_subscribe_topics) .def("subscribe_pattern", &Client_subscribe_pattern) .def("create_reader", &Client_createReader) .def("create_table_view", [](Client& client, const std::string& topic, const TableViewConfiguration& config) { return waitForAsyncValue([&](TableViewCallback callback) { client.createTableViewAsync(topic, config, callback); }); }) .def("get_topic_partitions", &Client_getTopicPartitions) .def("get_service_info", &Client::getServiceInfo) .def("get_schema_info", &Client_getSchemaInfo) .def("close", &Client_close) .def("close_async", &Client_closeAsync) .def("get_topic_partitions_async", &Client_getTopicPartitionsAsync) .def("subscribe_async", &Client_subscribeAsync) .def("subscribe_async_topics", &Client_subscribeAsync_topics) .def("subscribe_async_pattern", &Client_subscribeAsync_pattern) .def("shutdown", &Client::shutdown); } pulsar-client-python-3.12.0-candidate-0/src/config.cc000066400000000000000000000540021520101105200222740ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include #include #include #include #include #include #include #include #include #include #include #include namespace py = pybind11; #ifdef __GNUC__ #define HIDDEN __attribute__((visibility("hidden"))) #else #define HIDDEN #endif class HIDDEN LoggerWrapper : public Logger, public CaptivePythonObjectMixin { const std::unique_ptr _fallbackLogger; py::object _pyLogger; public: LoggerWrapper(PyObject* pyLoggerPtr, Logger* fallbackLogger, py::object pyLogger) : CaptivePythonObjectMixin(pyLoggerPtr), _fallbackLogger(fallbackLogger), _pyLogger(pyLogger) {} LoggerWrapper(const LoggerWrapper&) = delete; LoggerWrapper(LoggerWrapper&&) noexcept = delete; LoggerWrapper& operator=(const LoggerWrapper&) = delete; LoggerWrapper& operator=(LoggerWrapper&&) = delete; bool isEnabled(Level level) { return true; // Python loggers are always enabled; they decide internally whether or not to log. } void log(Level level, int line, const std::string& message) { if (!Py_IsInitialized()) { // Python logger is unavailable - fallback to console logger _fallbackLogger->log(level, line, message); } else { PyGILState_STATE state = PyGILState_Ensure(); PyObject *type, *value, *traceback; PyErr_Fetch(&type, &value, &traceback); try { switch (level) { case Logger::LEVEL_DEBUG: _pyLogger(py::str("DEBUG"), message); break; case Logger::LEVEL_INFO: _pyLogger(py::str("INFO"), message); break; case Logger::LEVEL_WARN: _pyLogger(py::str("WARNING"), message); break; case Logger::LEVEL_ERROR: _pyLogger(py::str("ERROR"), message); break; } } catch (const py::error_already_set& e) { PyErr_Print(); _fallbackLogger->log(level, line, message); } PyErr_Restore(type, value, traceback); PyGILState_Release(state); } } }; class HIDDEN LoggerWrapperFactory : public LoggerFactory, public CaptivePythonObjectMixin { py::object _pyLogger; std::unique_ptr _fallbackLoggerFactory{new ConsoleLoggerFactory}; public: LoggerWrapperFactory(py::object pyLogger) : CaptivePythonObjectMixin(pyLogger.ptr()), _pyLogger(pyLogger) {} Logger* getLogger(const std::string& fileName) { const auto fallbackLogger = _fallbackLoggerFactory->getLogger(fileName); if (_captive == py::object().ptr()) { return fallbackLogger; } else { return new LoggerWrapper(_captive, fallbackLogger, _pyLogger); } } }; using MessageRouterFunc = std::function; class HIDDEN MessageRouter : public pulsar::MessageRoutingPolicy { public: explicit MessageRouter(MessageRouterFunc func) : func_(std::move(func)) {} int getPartition(const Message& msg, const TopicMetadata& topicMetadata) final { return func_(msg, topicMetadata.getNumPartitions()); } private: MessageRouterFunc func_; }; static ClientConfiguration& ClientConfiguration_setLogger(ClientConfiguration& conf, py::object logger) { conf.setLogger(new LoggerWrapperFactory(logger)); return conf; } static ClientConfiguration& ClientConfiguration_setConsoleLogger(ClientConfiguration& conf, Logger::Level level) { conf.setLogger(new ConsoleLoggerFactory(level)); return conf; } static ClientConfiguration& ClientConfiguration_setFileLogger(ClientConfiguration& conf, Logger::Level level, const std::string& logFile) { conf.setLogger(new FileLoggerFactory(level, logFile)); return conf; } static ServiceInfo ServiceInfo_init(const std::string& serviceUrl, AuthenticationPtr authentication, std::optional tlsTrustCertsFilePath) { return ServiceInfo(serviceUrl, authentication ? std::move(authentication) : AuthFactory::Disabled(), std::move(tlsTrustCertsFilePath)); } void export_config(py::module_& m) { using namespace py; class_(m, "ServiceInfo") .def(init(&ServiceInfo_init), arg("service_url"), arg("authentication") = nullptr, arg("tls_trust_certs_file_path") = py::none()) .def_property_readonly("service_url", [](const ServiceInfo& serviceInfo) { return serviceInfo.serviceUrl(); }) .def_property_readonly("use_tls", &ServiceInfo::useTls) .def_property_readonly("tls_trust_certs_file_path", [](const ServiceInfo& serviceInfo) { return serviceInfo.tlsTrustCertsFilePath(); }); class_>(m, "KeySharedPolicy") .def(init<>()) .def("set_key_shared_mode", &KeySharedPolicy::setKeySharedMode, return_value_policy::reference) .def("get_key_shared_mode", &KeySharedPolicy::getKeySharedMode) .def("set_allow_out_of_order_delivery", &KeySharedPolicy::setAllowOutOfOrderDelivery, return_value_policy::reference) .def("is_allow_out_of_order_delivery", &KeySharedPolicy::isAllowOutOfOrderDelivery) .def("set_sticky_ranges", static_cast( &KeySharedPolicy::setStickyRanges), return_value_policy::reference) .def("get_sticky_ranges", &KeySharedPolicy::getStickyRanges); class_>(m, "AbstractCryptoKeyReader") .def("getPublicKey", &CryptoKeyReader::getPublicKey) .def("getPrivateKey", &CryptoKeyReader::getPrivateKey); class_>( m, "CryptoKeyReader") .def(init()); class_>(m, "ClientConfiguration") .def(init<>()) .def("authentication", &ClientConfiguration::setAuth, return_value_policy::reference) .def("operation_timeout_seconds", &ClientConfiguration::getOperationTimeoutSeconds) .def("operation_timeout_seconds", &ClientConfiguration::setOperationTimeoutSeconds, return_value_policy::reference) .def("connection_timeout", &ClientConfiguration::getConnectionTimeout) .def("connection_timeout", &ClientConfiguration::setConnectionTimeout, return_value_policy::reference) .def("io_threads", &ClientConfiguration::getIOThreads) .def("io_threads", &ClientConfiguration::setIOThreads, return_value_policy::reference) .def("message_listener_threads", &ClientConfiguration::getMessageListenerThreads) .def("message_listener_threads", &ClientConfiguration::setMessageListenerThreads, return_value_policy::reference) .def("concurrent_lookup_requests", &ClientConfiguration::getConcurrentLookupRequest) .def("concurrent_lookup_requests", &ClientConfiguration::setConcurrentLookupRequest, return_value_policy::reference) .def("stats_interval_in_seconds", &ClientConfiguration::getStatsIntervalInSeconds) .def("stats_interval_in_seconds", &ClientConfiguration::setStatsIntervalInSeconds, return_value_policy::reference) .def("tls_trust_certs_file_path", &ClientConfiguration::setTlsTrustCertsFilePath, return_value_policy::reference) .def("tls_private_key_file_path", &ClientConfiguration::getTlsPrivateKeyFilePath, return_value_policy::copy) .def("tls_private_key_file_path", &ClientConfiguration::setTlsPrivateKeyFilePath, return_value_policy::reference) .def("tls_certificate_file_path", &ClientConfiguration::getTlsCertificateFilePath, return_value_policy::copy) .def("tls_certificate_file_path", &ClientConfiguration::setTlsCertificateFilePath, return_value_policy::reference) .def("tls_allow_insecure_connection", &ClientConfiguration::isTlsAllowInsecureConnection) .def("tls_allow_insecure_connection", &ClientConfiguration::setTlsAllowInsecureConnection, return_value_policy::reference) .def("tls_validate_hostname", &ClientConfiguration::setValidateHostName, return_value_policy::reference) .def("listener_name", &ClientConfiguration::setListenerName, return_value_policy::reference) .def("set_logger", &ClientConfiguration_setLogger, return_value_policy::reference) .def("set_console_logger", &ClientConfiguration_setConsoleLogger, return_value_policy::reference) .def("set_file_logger", &ClientConfiguration_setFileLogger, return_value_policy::reference); class_>(m, "ProducerConfiguration") .def(init<>()) .def("producer_name", &ProducerConfiguration::getProducerName, return_value_policy::copy) .def("producer_name", &ProducerConfiguration::setProducerName, return_value_policy::reference) .def("schema", &ProducerConfiguration::getSchema, return_value_policy::copy) .def("schema", &ProducerConfiguration::setSchema, return_value_policy::reference) .def("send_timeout_millis", &ProducerConfiguration::getSendTimeout) .def("send_timeout_millis", &ProducerConfiguration::setSendTimeout, return_value_policy::reference) .def("initial_sequence_id", &ProducerConfiguration::getInitialSequenceId) .def("initial_sequence_id", &ProducerConfiguration::setInitialSequenceId, return_value_policy::reference) .def("compression_type", &ProducerConfiguration::getCompressionType) .def("compression_type", &ProducerConfiguration::setCompressionType, return_value_policy::reference) .def("max_pending_messages", &ProducerConfiguration::getMaxPendingMessages) .def("max_pending_messages", &ProducerConfiguration::setMaxPendingMessages, return_value_policy::reference) .def("max_pending_messages_across_partitions", &ProducerConfiguration::getMaxPendingMessagesAcrossPartitions) .def("max_pending_messages_across_partitions", &ProducerConfiguration::setMaxPendingMessagesAcrossPartitions, return_value_policy::reference) .def("block_if_queue_full", &ProducerConfiguration::getBlockIfQueueFull) .def("block_if_queue_full", &ProducerConfiguration::setBlockIfQueueFull, return_value_policy::reference) .def("partitions_routing_mode", &ProducerConfiguration::getPartitionsRoutingMode) .def("partitions_routing_mode", &ProducerConfiguration::setPartitionsRoutingMode, return_value_policy::reference) .def("lazy_start_partitioned_producers", &ProducerConfiguration::getLazyStartPartitionedProducers) .def("lazy_start_partitioned_producers", &ProducerConfiguration::setLazyStartPartitionedProducers, return_value_policy::reference) .def("batching_enabled", &ProducerConfiguration::getBatchingEnabled, return_value_policy::copy) .def("batching_enabled", &ProducerConfiguration::setBatchingEnabled, return_value_policy::reference) .def("batching_max_messages", &ProducerConfiguration::getBatchingMaxMessages, return_value_policy::copy) .def("batching_max_messages", &ProducerConfiguration::setBatchingMaxMessages, return_value_policy::reference) .def("batching_max_allowed_size_in_bytes", &ProducerConfiguration::getBatchingMaxAllowedSizeInBytes, return_value_policy::copy) .def("batching_max_allowed_size_in_bytes", &ProducerConfiguration::setBatchingMaxAllowedSizeInBytes, return_value_policy::reference) .def("batching_max_publish_delay_ms", &ProducerConfiguration::getBatchingMaxPublishDelayMs, return_value_policy::copy) .def("batching_max_publish_delay_ms", &ProducerConfiguration::setBatchingMaxPublishDelayMs, return_value_policy::reference) .def("chunking_enabled", &ProducerConfiguration::isChunkingEnabled) .def("chunking_enabled", &ProducerConfiguration::setChunkingEnabled, return_value_policy::reference) .def("property", &ProducerConfiguration::setProperty, return_value_policy::reference) .def("batching_type", &ProducerConfiguration::setBatchingType, return_value_policy::reference) .def("batching_type", &ProducerConfiguration::getBatchingType) .def("encryption_key", &ProducerConfiguration::addEncryptionKey, return_value_policy::reference) .def("crypto_key_reader", &ProducerConfiguration::setCryptoKeyReader, return_value_policy::reference) .def("access_mode", &ProducerConfiguration::setAccessMode, return_value_policy::reference) .def("access_mode", &ProducerConfiguration::getAccessMode, return_value_policy::copy) .def("message_router", [](ProducerConfiguration& config, MessageRouterFunc func) { config.setMessageRouter(std::make_shared(std::move(func))); }); class_(m, "BatchReceivePolicy") .def(init()) .def("getTimeoutMs", &BatchReceivePolicy::getTimeoutMs) .def("getMaxNumMessages", &BatchReceivePolicy::getMaxNumMessages) .def("getMaxNumBytes", &BatchReceivePolicy::getMaxNumBytes); class_(m, "DeadLetterPolicy") .def(init<>()) .def("getDeadLetterTopic", &DeadLetterPolicy::getDeadLetterTopic) .def("getMaxRedeliverCount", &DeadLetterPolicy::getMaxRedeliverCount) .def("getInitialSubscriptionName", &DeadLetterPolicy::getInitialSubscriptionName); class_(m, "DeadLetterPolicyBuilder") .def(init<>()) .def("deadLetterTopic", &DeadLetterPolicyBuilder::deadLetterTopic, return_value_policy::reference) .def("maxRedeliverCount", &DeadLetterPolicyBuilder::maxRedeliverCount, return_value_policy::reference) .def("initialSubscriptionName", &DeadLetterPolicyBuilder::initialSubscriptionName, return_value_policy::reference) .def("build", &DeadLetterPolicyBuilder::build, return_value_policy::reference) .def("build", &DeadLetterPolicyBuilder::build, return_value_policy::reference); class_>(m, "ConsumerConfiguration") .def(init<>()) .def("key_shared_policy", &ConsumerConfiguration::getKeySharedPolicy) .def("key_shared_policy", &ConsumerConfiguration::setKeySharedPolicy, return_value_policy::reference) .def("consumer_type", &ConsumerConfiguration::getConsumerType) .def("consumer_type", &ConsumerConfiguration::setConsumerType, return_value_policy::reference) .def("schema", &ConsumerConfiguration::getSchema, return_value_policy::copy) .def("schema", &ConsumerConfiguration::setSchema, return_value_policy::reference) .def("message_listener", &ConsumerConfiguration::setMessageListener, return_value_policy::reference) .def("receiver_queue_size", &ConsumerConfiguration::getReceiverQueueSize) .def("receiver_queue_size", &ConsumerConfiguration::setReceiverQueueSize) .def("max_total_receiver_queue_size_across_partitions", &ConsumerConfiguration::getMaxTotalReceiverQueueSizeAcrossPartitions) .def("max_total_receiver_queue_size_across_partitions", &ConsumerConfiguration::setMaxTotalReceiverQueueSizeAcrossPartitions) .def("batch_receive_policy", &ConsumerConfiguration::getBatchReceivePolicy, return_value_policy::copy) .def("batch_receive_policy", &ConsumerConfiguration::setBatchReceivePolicy) .def("consumer_name", &ConsumerConfiguration::getConsumerName, return_value_policy::copy) .def("consumer_name", &ConsumerConfiguration::setConsumerName) .def("unacked_messages_timeout_ms", &ConsumerConfiguration::getUnAckedMessagesTimeoutMs) .def("unacked_messages_timeout_ms", &ConsumerConfiguration::setUnAckedMessagesTimeoutMs) .def("negative_ack_redelivery_delay_ms", &ConsumerConfiguration::getNegativeAckRedeliveryDelayMs) .def("negative_ack_redelivery_delay_ms", &ConsumerConfiguration::setNegativeAckRedeliveryDelayMs) .def("broker_consumer_stats_cache_time_ms", &ConsumerConfiguration::getBrokerConsumerStatsCacheTimeInMs) .def("broker_consumer_stats_cache_time_ms", &ConsumerConfiguration::setBrokerConsumerStatsCacheTimeInMs) .def("pattern_auto_discovery_period", &ConsumerConfiguration::getPatternAutoDiscoveryPeriod) .def("pattern_auto_discovery_period", &ConsumerConfiguration::setPatternAutoDiscoveryPeriod) .def("read_compacted", &ConsumerConfiguration::isReadCompacted) .def("read_compacted", &ConsumerConfiguration::setReadCompacted) .def("property", &ConsumerConfiguration::setProperty, return_value_policy::reference) .def("subscription_initial_position", &ConsumerConfiguration::getSubscriptionInitialPosition) .def("subscription_initial_position", &ConsumerConfiguration::setSubscriptionInitialPosition) .def("regex_subscription_mode", &ConsumerConfiguration::setRegexSubscriptionMode) .def("regex_subscription_mode", &ConsumerConfiguration::getRegexSubscriptionMode, return_value_policy::reference) .def("crypto_key_reader", &ConsumerConfiguration::setCryptoKeyReader, return_value_policy::reference) .def("replicate_subscription_state_enabled", &ConsumerConfiguration::setReplicateSubscriptionStateEnabled) .def("replicate_subscription_state_enabled", &ConsumerConfiguration::isReplicateSubscriptionStateEnabled) .def("max_pending_chunked_message", &ConsumerConfiguration::getMaxPendingChunkedMessage) .def("max_pending_chunked_message", &ConsumerConfiguration::setMaxPendingChunkedMessage, return_value_policy::reference) .def("auto_ack_oldest_chunked_message_on_queue_full", &ConsumerConfiguration::isAutoAckOldestChunkedMessageOnQueueFull) .def("auto_ack_oldest_chunked_message_on_queue_full", &ConsumerConfiguration::setAutoAckOldestChunkedMessageOnQueueFull, return_value_policy::reference) .def("start_message_id_inclusive", &ConsumerConfiguration::isStartMessageIdInclusive) .def("start_message_id_inclusive", &ConsumerConfiguration::setStartMessageIdInclusive, return_value_policy::reference) .def("batch_index_ack_enabled", &ConsumerConfiguration::isBatchIndexAckEnabled) .def("batch_index_ack_enabled", &ConsumerConfiguration::setBatchIndexAckEnabled, return_value_policy::reference) .def("dead_letter_policy", &ConsumerConfiguration::setDeadLetterPolicy) .def("dead_letter_policy", &ConsumerConfiguration::getDeadLetterPolicy, return_value_policy::copy) .def("crypto_failure_action", &ConsumerConfiguration::getCryptoFailureAction, return_value_policy::copy) .def("crypto_failure_action", &ConsumerConfiguration::setCryptoFailureAction, return_value_policy::reference); class_>(m, "ReaderConfiguration") .def(init<>()) .def("reader_listener", &ReaderConfiguration::setReaderListener, return_value_policy::reference) .def("schema", &ReaderConfiguration::getSchema, return_value_policy::copy) .def("schema", &ReaderConfiguration::setSchema, return_value_policy::reference) .def("receiver_queue_size", &ReaderConfiguration::getReceiverQueueSize) .def("receiver_queue_size", &ReaderConfiguration::setReceiverQueueSize) .def("reader_name", &ReaderConfiguration::getReaderName, return_value_policy::copy) .def("reader_name", &ReaderConfiguration::setReaderName) .def("subscription_role_prefix", &ReaderConfiguration::getSubscriptionRolePrefix, return_value_policy::copy) .def("subscription_role_prefix", &ReaderConfiguration::setSubscriptionRolePrefix) .def("read_compacted", &ReaderConfiguration::isReadCompacted) .def("read_compacted", &ReaderConfiguration::setReadCompacted) .def("crypto_key_reader", &ReaderConfiguration::setCryptoKeyReader, return_value_policy::reference) .def("start_message_id_inclusive", &ReaderConfiguration::isStartMessageIdInclusive) .def("start_message_id_inclusive", &ReaderConfiguration::setStartMessageIdInclusive, return_value_policy::reference) .def("crypto_failure_action", &ReaderConfiguration::getCryptoFailureAction, return_value_policy::copy) .def("crypto_failure_action", &ReaderConfiguration::setCryptoFailureAction, return_value_policy::reference); } pulsar-client-python-3.12.0-candidate-0/src/consumer.cc000066400000000000000000000205711520101105200226660ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include #include #include #include namespace py = pybind11; void Consumer_unsubscribe(Consumer& consumer) { waitForAsyncResult([&consumer](ResultCallback callback) { consumer.unsubscribeAsync(callback); }); } Message Consumer_receive(Consumer& consumer) { return waitForAsyncValue([&](ReceiveCallback callback) { consumer.receiveAsync(callback); }); } Message Consumer_receive_timeout(Consumer& consumer, int timeoutMs) { Message msg; Result res; Py_BEGIN_ALLOW_THREADS res = consumer.receive(msg, timeoutMs); Py_END_ALLOW_THREADS CHECK_RESULT(res); return msg; } Messages Consumer_batch_receive(Consumer& consumer) { Messages msgs; Result res; Py_BEGIN_ALLOW_THREADS res = consumer.batchReceive(msgs); Py_END_ALLOW_THREADS CHECK_RESULT(res); return msgs; } void Consumer_acknowledge(Consumer& consumer, const Message& msg) { waitForAsyncResult([&](ResultCallback callback) { consumer.acknowledgeAsync(msg, callback); }); } void Consumer_acknowledge_message_id(Consumer& consumer, const MessageId& msgId) { waitForAsyncResult([&](ResultCallback callback) { consumer.acknowledgeAsync(msgId, callback); }); } void Consumer_negative_acknowledge(Consumer& consumer, const Message& msg) { Py_BEGIN_ALLOW_THREADS consumer.negativeAcknowledge(msg); Py_END_ALLOW_THREADS } void Consumer_negative_acknowledge_message_id(Consumer& consumer, const MessageId& msgId) { Py_BEGIN_ALLOW_THREADS consumer.negativeAcknowledge(msgId); Py_END_ALLOW_THREADS } void Consumer_acknowledge_cumulative(Consumer& consumer, const Message& msg) { waitForAsyncResult([&](ResultCallback callback) { consumer.acknowledgeCumulativeAsync(msg, callback); }); } void Consumer_acknowledge_cumulative_message_id(Consumer& consumer, const MessageId& msgId) { waitForAsyncResult( [&](ResultCallback callback) { consumer.acknowledgeCumulativeAsync(msgId, callback); }); } void Consumer_close(Consumer& consumer) { waitForAsyncResult([&consumer](ResultCallback callback) { consumer.closeAsync(callback); }); } void Consumer_pauseMessageListener(Consumer& consumer) { CHECK_RESULT(consumer.pauseMessageListener()); } void Consumer_resumeMessageListener(Consumer& consumer) { CHECK_RESULT(consumer.resumeMessageListener()); } void Consumer_seek(Consumer& consumer, const MessageId& msgId) { waitForAsyncResult([msgId, &consumer](ResultCallback callback) { consumer.seekAsync(msgId, callback); }); } void Consumer_seek_timestamp(Consumer& consumer, uint64_t timestamp) { waitForAsyncResult( [timestamp, &consumer](ResultCallback callback) { consumer.seekAsync(timestamp, callback); }); } bool Consumer_is_connected(Consumer& consumer) { return consumer.isConnected(); } MessageId Consumer_get_last_message_id(Consumer& consumer) { MessageId msgId; Result res; Py_BEGIN_ALLOW_THREADS res = consumer.getLastMessageId(msgId); Py_END_ALLOW_THREADS; CHECK_RESULT(res); return msgId; } void Consumer_get_last_message_id_async(Consumer& consumer, GetLastMessageIdCallback callback) { py::gil_scoped_release release; consumer.getLastMessageIdAsync(callback); } void Consumer_receiveAsync(Consumer& consumer, ReceiveCallback callback) { py::gil_scoped_release release; consumer.receiveAsync(callback); } void Consumer_acknowledgeAsync(Consumer& consumer, const Message& msg, ResultCallback callback) { py::gil_scoped_release release; consumer.acknowledgeAsync(msg, callback); } void Consumer_acknowledgeAsync_message_id(Consumer& consumer, const MessageId& msgId, ResultCallback callback) { py::gil_scoped_release release; consumer.acknowledgeAsync(msgId, callback); } void Consumer_acknowledgeCumulativeAsync(Consumer& consumer, const Message& msg, ResultCallback callback) { py::gil_scoped_release release; consumer.acknowledgeCumulativeAsync(msg, callback); } void Consumer_acknowledgeCumulativeAsync_message_id(Consumer& consumer, const MessageId& msgId, ResultCallback callback) { py::gil_scoped_release release; consumer.acknowledgeCumulativeAsync(msgId, callback); } void Consumer_negative_acknowledgeAsync(Consumer& consumer, const Message& msg, ResultCallback callback) { py::gil_scoped_release release; consumer.negativeAcknowledge(msg); } void Consumer_negative_acknowledgeAsync_message_id(Consumer& consumer, const MessageId& msgId, ResultCallback callback) { py::gil_scoped_release release; consumer.negativeAcknowledge(msgId); } void Consumer_closeAsync(Consumer& consumer, ResultCallback callback) { py::gil_scoped_release release; consumer.closeAsync(callback); } void Consumer_unsubscribeAsync(Consumer& consumer, ResultCallback callback) { py::gil_scoped_release release; consumer.unsubscribeAsync(callback); } void Consumer_seekAsync(Consumer& consumer, const MessageId& msgId, ResultCallback callback) { py::gil_scoped_release release; consumer.seekAsync(msgId, callback); } void Consumer_seekAsync_timestamp(Consumer& consumer, uint64_t timestamp, ResultCallback callback) { py::gil_scoped_release release; consumer.seekAsync(timestamp, callback); } void export_consumer(py::module_& m) { py::class_(m, "Consumer") .def(py::init<>()) .def("topic", &Consumer::getTopic, "return the topic this consumer is subscribed to", py::return_value_policy::copy) .def("subscription_name", &Consumer::getSubscriptionName, py::return_value_policy::copy) .def("consumer_name", &Consumer::getConsumerName, py::return_value_policy::copy) .def("unsubscribe", &Consumer_unsubscribe) .def("receive", &Consumer_receive) .def("receive", &Consumer_receive_timeout) .def("batch_receive", &Consumer_batch_receive) .def("acknowledge", &Consumer_acknowledge) .def("acknowledge", &Consumer_acknowledge_message_id) .def("acknowledge_cumulative", &Consumer_acknowledge_cumulative) .def("acknowledge_cumulative", &Consumer_acknowledge_cumulative_message_id) .def("negative_acknowledge", &Consumer_negative_acknowledge) .def("negative_acknowledge", &Consumer_negative_acknowledge_message_id) .def("close", &Consumer_close) .def("pause_message_listener", &Consumer_pauseMessageListener) .def("resume_message_listener", &Consumer_resumeMessageListener) .def("redeliver_unacknowledged_messages", &Consumer::redeliverUnacknowledgedMessages) .def("seek", &Consumer_seek) .def("seek", &Consumer_seek_timestamp) .def("is_connected", &Consumer_is_connected) .def("get_last_message_id", &Consumer_get_last_message_id) .def("receive_async", &Consumer_receiveAsync) .def("acknowledge_async", &Consumer_acknowledgeAsync) .def("acknowledge_async", &Consumer_acknowledgeAsync_message_id) .def("acknowledge_cumulative_async", &Consumer_acknowledgeCumulativeAsync) .def("acknowledge_cumulative_async", &Consumer_acknowledgeCumulativeAsync_message_id) .def("negative_acknowledge_async", &Consumer_negative_acknowledgeAsync) .def("negative_acknowledge_async", &Consumer_negative_acknowledgeAsync_message_id) .def("get_last_message_id_async", &Consumer_get_last_message_id_async) .def("close_async", &Consumer_closeAsync) .def("unsubscribe_async", &Consumer_unsubscribeAsync) .def("seek_async", &Consumer_seekAsync) .def("seek_async", &Consumer_seekAsync_timestamp); } pulsar-client-python-3.12.0-candidate-0/src/enums.cc000066400000000000000000000164371520101105200221700ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include #include #include #include #include #include using namespace pulsar; namespace py = pybind11; void export_enums(py::module_& m) { using namespace py; enum_(m, "KeySharedMode") .value("AutoSplit", AUTO_SPLIT) .value("Sticky", STICKY); enum_(m, "PartitionsRoutingMode") .value("UseSinglePartition", ProducerConfiguration::UseSinglePartition) .value("RoundRobinDistribution", ProducerConfiguration::RoundRobinDistribution) .value("CustomPartition", ProducerConfiguration::CustomPartition); enum_(m, "CompressionType") .value("NONE", CompressionNone) // Don't use 'None' since it's a keyword in py3 .value("LZ4", CompressionLZ4) .value("ZLib", CompressionZLib) .value("ZSTD", CompressionZSTD) .value("SNAPPY", CompressionSNAPPY); enum_(m, "ConsumerType") .value("Exclusive", ConsumerExclusive) .value("Shared", ConsumerShared) .value("Failover", ConsumerFailover) .value("KeyShared", ConsumerKeyShared); enum_(m, "Result", "Collection of return codes") .value("Ok", ResultOk) .value("UnknownError", ResultUnknownError) .value("InvalidConfiguration", ResultInvalidConfiguration) .value("Timeout", ResultTimeout) .value("LookupError", ResultLookupError) .value("ConnectError", ResultConnectError) .value("ReadError", ResultReadError) .value("AuthenticationError", ResultAuthenticationError) .value("AuthorizationError", ResultAuthorizationError) .value("ErrorGettingAuthenticationData", ResultErrorGettingAuthenticationData) .value("BrokerMetadataError", ResultBrokerMetadataError) .value("BrokerPersistenceError", ResultBrokerPersistenceError) .value("ChecksumError", ResultChecksumError) .value("ConsumerBusy", ResultConsumerBusy) .value("NotConnected", ResultNotConnected) .value("AlreadyClosed", ResultAlreadyClosed) .value("InvalidMessage", ResultInvalidMessage) .value("ConsumerNotInitialized", ResultConsumerNotInitialized) .value("ProducerNotInitialized", ResultProducerNotInitialized) .value("ProducerBusy", ResultProducerBusy) .value("TooManyLookupRequestException", ResultTooManyLookupRequestException) .value("InvalidTopicName", ResultInvalidTopicName) .value("InvalidUrl", ResultInvalidUrl) .value("ServiceUnitNotReady", ResultServiceUnitNotReady) .value("OperationNotSupported", ResultOperationNotSupported) .value("ProducerBlockedQuotaExceededError", ResultProducerBlockedQuotaExceededError) .value("ProducerBlockedQuotaExceededException", ResultProducerBlockedQuotaExceededException) .value("ProducerQueueIsFull", ResultProducerQueueIsFull) .value("MessageTooBig", ResultMessageTooBig) .value("TopicNotFound", ResultTopicNotFound) .value("SubscriptionNotFound", ResultSubscriptionNotFound) .value("ConsumerNotFound", ResultConsumerNotFound) .value("UnsupportedVersionError", ResultUnsupportedVersionError) .value("TopicTerminated", ResultTopicTerminated) .value("CryptoError", ResultCryptoError) .value("IncompatibleSchema", ResultIncompatibleSchema) .value("ConsumerAssignError", ResultConsumerAssignError) .value("CumulativeAcknowledgementNotAllowedError", ResultCumulativeAcknowledgementNotAllowedError) .value("TransactionCoordinatorNotFoundError", ResultTransactionCoordinatorNotFoundError) .value("InvalidTxnStatusError", ResultInvalidTxnStatusError) .value("NotAllowedError", ResultNotAllowedError) .value("TransactionConflict", ResultTransactionConflict) .value("TransactionNotFound", ResultTransactionNotFound) .value("ProducerFenced", ResultProducerFenced) .value("MemoryBufferIsFull", ResultMemoryBufferIsFull) .value("Interrupted", pulsar::ResultInterrupted); enum_(m, "SchemaType", "Supported schema types") .value("NONE", pulsar::NONE) .value("STRING", pulsar::STRING) .value("INT8", pulsar::INT8) .value("INT16", pulsar::INT16) .value("INT32", pulsar::INT32) .value("INT64", pulsar::INT64) .value("FLOAT", pulsar::FLOAT) .value("DOUBLE", pulsar::DOUBLE) .value("BYTES", pulsar::BYTES) .value("JSON", pulsar::JSON) .value("PROTOBUF", pulsar::PROTOBUF) .value("AVRO", pulsar::AVRO) .value("AUTO_CONSUME", pulsar::AUTO_CONSUME) .value("AUTO_PUBLISH", pulsar::AUTO_PUBLISH) .value("KEY_VALUE", pulsar::KEY_VALUE) .value("PROTOBUF_NATIVE", pulsar::PROTOBUF_NATIVE); enum_(m, "InitialPosition", "Supported initial position") .value("Latest", InitialPositionLatest) .value("Earliest", InitialPositionEarliest); enum_(m, "RegexSubscriptionMode", "Regex subscription mode") .value("PersistentOnly", PersistentOnly) .value("NonPersistentOnly", NonPersistentOnly) .value("AllTopics", AllTopics); enum_(m, "BatchingType", "Supported batching types") .value("Default", ProducerConfiguration::DefaultBatching) .value("KeyBased", ProducerConfiguration::KeyBasedBatching); enum_(m, "ProducerAccessMode", "Producer Access Mode") .value("Shared", ProducerConfiguration::ProducerAccessMode::Shared) .value("Exclusive", ProducerConfiguration::ProducerAccessMode::Exclusive) .value("WaitForExclusive", ProducerConfiguration::ProducerAccessMode::WaitForExclusive) .value("ExclusiveWithFencing", ProducerConfiguration::ProducerAccessMode::ExclusiveWithFencing); enum_(m, "LoggerLevel") .value("Debug", Logger::LEVEL_DEBUG) .value("Info", Logger::LEVEL_INFO) .value("Warn", Logger::LEVEL_WARN) .value("Error", Logger::LEVEL_ERROR); enum_(m, "ConsumerCryptoFailureAction") .value("FAIL", ConsumerCryptoFailureAction::FAIL) .value("DISCARD", ConsumerCryptoFailureAction::DISCARD) .value("CONSUME", ConsumerCryptoFailureAction::CONSUME); } pulsar-client-python-3.12.0-candidate-0/src/exceptions.cc000066400000000000000000000172621520101105200232170ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "exceptions.h" #include #include using namespace pulsar; namespace py = pybind11; #define CASE_RESULT(className) \ case pulsar::Result##className: \ throw className{pulsar::Result##className}; void raiseException(pulsar::Result result) { switch (result) { CASE_RESULT(UnknownError) CASE_RESULT(InvalidConfiguration) CASE_RESULT(Timeout) CASE_RESULT(LookupError) CASE_RESULT(ConnectError) CASE_RESULT(ReadError) CASE_RESULT(AuthenticationError) CASE_RESULT(AuthorizationError) CASE_RESULT(ErrorGettingAuthenticationData) CASE_RESULT(BrokerMetadataError) CASE_RESULT(BrokerPersistenceError) CASE_RESULT(ChecksumError) CASE_RESULT(ConsumerBusy) CASE_RESULT(NotConnected) CASE_RESULT(AlreadyClosed) CASE_RESULT(InvalidMessage) CASE_RESULT(ConsumerNotInitialized) CASE_RESULT(ProducerNotInitialized) CASE_RESULT(ProducerBusy) CASE_RESULT(TooManyLookupRequestException) CASE_RESULT(InvalidTopicName) CASE_RESULT(InvalidUrl) CASE_RESULT(ServiceUnitNotReady) CASE_RESULT(OperationNotSupported) CASE_RESULT(ProducerBlockedQuotaExceededError) CASE_RESULT(ProducerBlockedQuotaExceededException) CASE_RESULT(ProducerQueueIsFull) CASE_RESULT(MessageTooBig) CASE_RESULT(TopicNotFound) CASE_RESULT(SubscriptionNotFound) CASE_RESULT(ConsumerNotFound) CASE_RESULT(UnsupportedVersionError) CASE_RESULT(TopicTerminated) CASE_RESULT(CryptoError) CASE_RESULT(IncompatibleSchema) CASE_RESULT(ConsumerAssignError) CASE_RESULT(CumulativeAcknowledgementNotAllowedError) CASE_RESULT(TransactionCoordinatorNotFoundError) CASE_RESULT(InvalidTxnStatusError) CASE_RESULT(NotAllowedError) CASE_RESULT(TransactionConflict) CASE_RESULT(TransactionNotFound) CASE_RESULT(ProducerFenced) CASE_RESULT(MemoryBufferIsFull) CASE_RESULT(Interrupted) default: return; } } // There is no std::hash specification for an enum in Clang compiler of macOS for C++11 template <> struct std::hash { std::size_t operator()(const Result& result) const noexcept { return std::hash()(static_cast(result)); } }; using PythonExceptionMap = std::unordered_map>; static PythonExceptionMap createPythonExceptionMap(py::module_& m, py::exception& base) { PythonExceptionMap exceptions; exceptions[ResultUnknownError] = {m, "UnknownError", base}; exceptions[ResultInvalidConfiguration] = {m, "InvalidConfiguration", base}; exceptions[ResultTimeout] = {m, "Timeout", base}; exceptions[ResultLookupError] = {m, "LookupError", base}; exceptions[ResultConnectError] = {m, "ConnectError", base}; exceptions[ResultReadError] = {m, "ReadError", base}; exceptions[ResultAuthenticationError] = {m, "AuthenticationError", base}; exceptions[ResultAuthorizationError] = {m, "AuthorizationError", base}; exceptions[ResultErrorGettingAuthenticationData] = {m, "ErrorGettingAuthenticationData", base}; exceptions[ResultBrokerMetadataError] = {m, "BrokerMetadataError", base}; exceptions[ResultBrokerPersistenceError] = {m, "BrokerPersistenceError", base}; exceptions[ResultChecksumError] = {m, "ChecksumError", base}; exceptions[ResultConsumerBusy] = {m, "ConsumerBusy", base}; exceptions[ResultNotConnected] = {m, "NotConnected", base}; exceptions[ResultAlreadyClosed] = {m, "AlreadyClosed", base}; exceptions[ResultInvalidMessage] = {m, "InvalidMessage", base}; exceptions[ResultConsumerNotInitialized] = {m, "ConsumerNotInitialized", base}; exceptions[ResultProducerNotInitialized] = {m, "ProducerNotInitialized", base}; exceptions[ResultProducerBusy] = {m, "ProducerBusy", base}; exceptions[ResultTooManyLookupRequestException] = {m, "TooManyLookupRequestException", base}; exceptions[ResultInvalidTopicName] = {m, "InvalidTopicName", base}; exceptions[ResultInvalidUrl] = {m, "InvalidUrl", base}; exceptions[ResultServiceUnitNotReady] = {m, "ServiceUnitNotReady", base}; exceptions[ResultOperationNotSupported] = {m, "OperationNotSupported", base}; exceptions[ResultProducerBlockedQuotaExceededError] = {m, "ProducerBlockedQuotaExceededError", base}; exceptions[ResultProducerBlockedQuotaExceededException] = {m, "ProducerBlockedQuotaExceededException", base}; exceptions[ResultProducerQueueIsFull] = {m, "ProducerQueueIsFull", base}; exceptions[ResultMessageTooBig] = {m, "MessageTooBig", base}; exceptions[ResultTopicNotFound] = {m, "TopicNotFound", base}; exceptions[ResultSubscriptionNotFound] = {m, "SubscriptionNotFound", base}; exceptions[ResultConsumerNotFound] = {m, "ConsumerNotFound", base}; exceptions[ResultUnsupportedVersionError] = {m, "UnsupportedVersionError", base}; exceptions[ResultTopicTerminated] = {m, "TopicTerminated", base}; exceptions[ResultCryptoError] = {m, "CryptoError", base}; exceptions[ResultIncompatibleSchema] = {m, "IncompatibleSchema", base}; exceptions[ResultConsumerAssignError] = {m, "ConsumerAssignError", base}; exceptions[ResultCumulativeAcknowledgementNotAllowedError] = { m, "CumulativeAcknowledgementNotAllowedError", base}; exceptions[ResultTransactionCoordinatorNotFoundError] = {m, "TransactionCoordinatorNotFoundError", base}; exceptions[ResultInvalidTxnStatusError] = {m, "InvalidTxnStatusError", base}; exceptions[ResultNotAllowedError] = {m, "NotAllowedError", base}; exceptions[ResultTransactionConflict] = {m, "TransactionConflict", base}; exceptions[ResultTransactionNotFound] = {m, "TransactionNotFound", base}; exceptions[ResultProducerFenced] = {m, "ProducerFenced", base}; exceptions[ResultMemoryBufferIsFull] = {m, "MemoryBufferIsFull", base}; exceptions[ResultInterrupted] = {m, "Interrupted", base}; return exceptions; } void export_exceptions(py::module_& m) { static py::exception base{m, "PulsarException"}; static auto exceptions = createPythonExceptionMap(m, base); py::register_exception_translator([](std::exception_ptr e) { try { if (e) { std::rethrow_exception(e); } } catch (const PulsarException& e) { auto it = exceptions.find(e._result); if (it != exceptions.end()) { PyErr_SetString(it->second.ptr(), e.what()); } else { base(e.what()); } } catch (const std::invalid_argument& e) { PyErr_SetString(PyExc_ValueError, e.what()); } catch (const std::exception& e) { PyErr_SetString(PyExc_RuntimeError, e.what()); } }); } pulsar-client-python-3.12.0-candidate-0/src/exceptions.h000066400000000000000000000065611520101105200230610ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #pragma once #include #include #include struct PulsarException : std::exception { const pulsar::Result _result; std::string _msg = "Pulsar error: "; PulsarException(pulsar::Result res) : _result(res) { _msg += strResult(res); } const char* what() const noexcept override { return _msg.c_str(); } }; void raiseException(pulsar::Result result); #define INHERIT_PULSAR_EXCEPTION(name) \ struct name : PulsarException { \ using PulsarException::PulsarException; \ }; INHERIT_PULSAR_EXCEPTION(UnknownError) INHERIT_PULSAR_EXCEPTION(InvalidConfiguration) INHERIT_PULSAR_EXCEPTION(Timeout) INHERIT_PULSAR_EXCEPTION(LookupError) INHERIT_PULSAR_EXCEPTION(ConnectError) INHERIT_PULSAR_EXCEPTION(ReadError) INHERIT_PULSAR_EXCEPTION(AuthenticationError) INHERIT_PULSAR_EXCEPTION(AuthorizationError) INHERIT_PULSAR_EXCEPTION(ErrorGettingAuthenticationData) INHERIT_PULSAR_EXCEPTION(BrokerMetadataError) INHERIT_PULSAR_EXCEPTION(BrokerPersistenceError) INHERIT_PULSAR_EXCEPTION(ChecksumError) INHERIT_PULSAR_EXCEPTION(ConsumerBusy) INHERIT_PULSAR_EXCEPTION(NotConnected) INHERIT_PULSAR_EXCEPTION(AlreadyClosed) INHERIT_PULSAR_EXCEPTION(InvalidMessage) INHERIT_PULSAR_EXCEPTION(ConsumerNotInitialized) INHERIT_PULSAR_EXCEPTION(ProducerNotInitialized) INHERIT_PULSAR_EXCEPTION(ProducerBusy) INHERIT_PULSAR_EXCEPTION(TooManyLookupRequestException) INHERIT_PULSAR_EXCEPTION(InvalidTopicName) INHERIT_PULSAR_EXCEPTION(InvalidUrl) INHERIT_PULSAR_EXCEPTION(ServiceUnitNotReady) INHERIT_PULSAR_EXCEPTION(OperationNotSupported) INHERIT_PULSAR_EXCEPTION(ProducerBlockedQuotaExceededError) INHERIT_PULSAR_EXCEPTION(ProducerBlockedQuotaExceededException) INHERIT_PULSAR_EXCEPTION(ProducerQueueIsFull) INHERIT_PULSAR_EXCEPTION(MessageTooBig) INHERIT_PULSAR_EXCEPTION(TopicNotFound) INHERIT_PULSAR_EXCEPTION(SubscriptionNotFound) INHERIT_PULSAR_EXCEPTION(ConsumerNotFound) INHERIT_PULSAR_EXCEPTION(UnsupportedVersionError) INHERIT_PULSAR_EXCEPTION(TopicTerminated) INHERIT_PULSAR_EXCEPTION(CryptoError) INHERIT_PULSAR_EXCEPTION(IncompatibleSchema) INHERIT_PULSAR_EXCEPTION(ConsumerAssignError) INHERIT_PULSAR_EXCEPTION(CumulativeAcknowledgementNotAllowedError) INHERIT_PULSAR_EXCEPTION(TransactionCoordinatorNotFoundError) INHERIT_PULSAR_EXCEPTION(InvalidTxnStatusError) INHERIT_PULSAR_EXCEPTION(NotAllowedError) INHERIT_PULSAR_EXCEPTION(TransactionConflict) INHERIT_PULSAR_EXCEPTION(TransactionNotFound) INHERIT_PULSAR_EXCEPTION(ProducerFenced) INHERIT_PULSAR_EXCEPTION(MemoryBufferIsFull) INHERIT_PULSAR_EXCEPTION(Interrupted) #undef INHERIT_PULSAR_EXCEPTION pulsar-client-python-3.12.0-candidate-0/src/message.cc000066400000000000000000000146311520101105200224570ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include #include #include #include namespace py = pybind11; void export_message(py::module_& m) { using namespace py; PyDateTime_IMPORT; MessageBuilder& (MessageBuilder::*MessageBuilderSetContentString)(const std::string&) = &MessageBuilder::setContent; class_(m, "MessageBuilder") .def(init<>()) .def("content", MessageBuilderSetContentString, return_value_policy::reference) .def("property", &MessageBuilder::setProperty, return_value_policy::reference) .def("properties", &MessageBuilder::setProperties, return_value_policy::reference) .def("sequence_id", &MessageBuilder::setSequenceId, return_value_policy::reference) .def("deliver_after", &MessageBuilder::setDeliverAfter, return_value_policy::reference) .def("deliver_at", &MessageBuilder::setDeliverAt, return_value_policy::reference) .def("partition_key", &MessageBuilder::setPartitionKey, return_value_policy::reference) .def("ordering_key", &MessageBuilder::setOrderingKey, return_value_policy::reference) .def("event_timestamp", &MessageBuilder::setEventTimestamp, return_value_policy::reference) .def("replication_clusters", &MessageBuilder::setReplicationClusters, return_value_policy::reference) .def("disable_replication", &MessageBuilder::disableReplication, return_value_policy::reference) .def("build", &MessageBuilder::build); class_(m, "MessageId") .def(init()) .def("__str__", [](const MessageId& msgId) { std::ostringstream oss; oss << msgId; return oss.str(); }) .def("__repr__", [](const MessageId& msgId) { std::ostringstream oss; oss << msgId; return oss.str(); }) .def("__eq__", &MessageId::operator==) .def("__ne__", &MessageId::operator!=) .def("__le__", &MessageId::operator<=) .def("__lt__", &MessageId::operator<) .def("__ge__", &MessageId::operator>=) .def("__gt__", &MessageId::operator>) .def("ledger_id", &MessageId::ledgerId) .def("entry_id", &MessageId::entryId) .def("batch_index", &MessageId::batchIndex) .def("partition", &MessageId::partition) .def( "topic_name", [](MessageId& msgId, const std::string& topicName) { msgId.setTopicName(topicName); }, return_value_policy::copy) .def_property_readonly_static("earliest", [](object) { return MessageId::earliest(); }) .def_property_readonly_static("latest", [](object) { return MessageId::latest(); }) .def("serialize", [](const MessageId& msgId) { std::string serialized; msgId.serialize(serialized); return bytes(serialized); }) .def_static("deserialize", &MessageId::deserialize); class_(m, "EncryptionKey") .def_readonly("key", &EncryptionKey::key) .def("value", [](const EncryptionKey& key) { return bytes(key.value); }) .def_readonly("metadata", &EncryptionKey::metadata); class_(m, "EncryptionContext") .def("keys", &EncryptionContext::keys) .def("param", [](const EncryptionContext& context) { return bytes(context.param()); }) .def("algorithm", &EncryptionContext::algorithm, return_value_policy::copy) .def("compression_type", &EncryptionContext::compressionType) .def("uncompressed_message_size", &EncryptionContext::uncompressedMessageSize) .def("batch_size", &EncryptionContext::batchSize) .def("is_decryption_failed", &EncryptionContext::isDecryptionFailed); class_(m, "Message") .def(init<>()) .def("properties", &Message::getProperties) .def("data", [](const Message& msg) { return bytes(msg.getDataAsString()); }) .def("length", &Message::getLength) .def("partition_key", &Message::getPartitionKey, return_value_policy::copy) .def("ordering_key", &Message::getOrderingKey, return_value_policy::copy) .def("publish_timestamp", &Message::getPublishTimestamp) .def("event_timestamp", &Message::getEventTimestamp) .def("message_id", &Message::getMessageId, return_value_policy::copy) .def("__str__", [](const Message& msg) { std::ostringstream oss; oss << msg; return oss.str(); }) .def("topic_name", &Message::getTopicName, return_value_policy::copy) .def("redelivery_count", &Message::getRedeliveryCount) .def("int_schema_version", &Message::getLongSchemaVersion) .def("schema_version", &Message::getSchemaVersion, return_value_policy::copy) .def("producer_name", &Message::getProducerName, return_value_policy::copy) .def("encryption_context", &Message::getEncryptionContext, return_value_policy::reference); MessageBatch& (MessageBatch::*MessageBatchParseFromString)(const std::string& payload, uint32_t batchSize) = &MessageBatch::parseFrom; class_(m, "MessageBatch") .def(init<>()) .def("with_message_id", &MessageBatch::withMessageId, return_value_policy::reference) .def("parse_from", MessageBatchParseFromString, return_value_policy::reference) .def("messages", &MessageBatch::messages, return_value_policy::copy); } pulsar-client-python-3.12.0-candidate-0/src/producer.cc000066400000000000000000000066551520101105200226650ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include #include #include namespace py = pybind11; MessageId Producer_send(Producer& producer, const Message& message) { return waitForAsyncValue( [&](SendCallback callback) { producer.sendAsync(message, callback); }); } void Producer_sendAsync(Producer& producer, const Message& msg, SendCallback callback) { Py_BEGIN_ALLOW_THREADS producer.sendAsync(msg, callback); Py_END_ALLOW_THREADS if (PyErr_CheckSignals() == -1) { PyErr_SetInterrupt(); } } void Producer_flush(Producer& producer) { waitForAsyncResult([&](ResultCallback callback) { producer.flushAsync(callback); }); } void Producer_close(Producer& producer) { waitForAsyncResult([&](ResultCallback callback) { producer.closeAsync(callback); }); } void Producer_closeAsync(Producer& producer, ResultCallback callback) { py::gil_scoped_release release; producer.closeAsync(callback); } void export_producer(py::module_& m) { using namespace py; class_(m, "Producer") .def(init<>()) .def("topic", &Producer::getTopic, "return the topic to which producer is publishing to", return_value_policy::copy) .def("producer_name", &Producer::getProducerName, "return the producer name which could have been assigned by the system or specified by the " "client", return_value_policy::copy) .def("last_sequence_id", &Producer::getLastSequenceId) .def("send", &Producer_send, "Publish a message on the topic associated with this Producer.\n" "\n" "This method will block until the message will be accepted and persisted\n" "by the broker. In case of errors, the client library will try to\n" "automatically recover and use a different broker.\n" "\n" "If it wasn't possible to successfully publish the message within the sendTimeout,\n" "an error will be returned.\n" "\n" "This method is equivalent to asyncSend() and wait until the callback is triggered.\n" "\n" "@param msg message to publish\n") .def("send_async", &Producer_sendAsync) .def("flush", &Producer_flush, "Flush all the messages buffered in the client and wait until all messages have been\n" "successfully persisted\n") .def("close", &Producer_close) .def("close_async", &Producer_closeAsync) .def("is_connected", &Producer::isConnected) .def("flush_async", &Producer::flushAsync); } pulsar-client-python-3.12.0-candidate-0/src/pulsar.cc000066400000000000000000000030421520101105200223330ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include namespace py = pybind11; using Module = py::module_; void export_client(Module& m); void export_message(Module& m); void export_producer(Module& m); void export_consumer(Module& m); void export_reader(Module& m); void export_config(Module& m); void export_enums(Module& m); void export_authentication(Module& m); void export_schema(Module& m); void export_exceptions(Module& m); void export_table_view(Module& m); PYBIND11_MODULE(_pulsar, m) { export_exceptions(m); export_client(m); export_message(m); export_producer(m); export_consumer(m); export_reader(m); export_config(m); export_enums(m); export_authentication(m); export_schema(m); export_table_view(m); } pulsar-client-python-3.12.0-candidate-0/src/reader.cc000066400000000000000000000046551520101105200223020ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include namespace py = pybind11; Message Reader_readNext(Reader& reader) { return waitForAsyncValue([&](ReadNextCallback callback) { reader.readNextAsync(callback); }); } Message Reader_readNextTimeout(Reader& reader, int timeoutMs) { Message msg; Result res; Py_BEGIN_ALLOW_THREADS res = reader.readNext(msg, timeoutMs); Py_END_ALLOW_THREADS CHECK_RESULT(res); return msg; } bool Reader_hasMessageAvailable(Reader& reader) { return waitForAsyncValue( [&](HasMessageAvailableCallback callback) { reader.hasMessageAvailableAsync(callback); }); } void Reader_close(Reader& reader) { waitForAsyncResult([&](ResultCallback callback) { reader.closeAsync(callback); }); } void Reader_seek(Reader& reader, const MessageId& msgId) { waitForAsyncResult([&](ResultCallback callback) { reader.seekAsync(msgId, callback); }); } void Reader_seek_timestamp(Reader& reader, uint64_t timestamp) { waitForAsyncResult([&](ResultCallback callback) { reader.seekAsync(timestamp, callback); }); } bool Reader_is_connected(Reader& reader) { return reader.isConnected(); } void export_reader(py::module_& m) { using namespace py; class_(m, "Reader") .def("topic", &Reader::getTopic, return_value_policy::copy) .def("read_next", &Reader_readNext) .def("read_next", &Reader_readNextTimeout) .def("has_message_available", &Reader_hasMessageAvailable) .def("close", &Reader_close) .def("seek", &Reader_seek) .def("seek", &Reader_seek_timestamp) .def("is_connected", &Reader_is_connected); } pulsar-client-python-3.12.0-candidate-0/src/schema.cc000066400000000000000000000023601520101105200222670ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" #include namespace py = pybind11; void export_schema(py::module_& m) { using namespace py; class_(m, "SchemaInfo") .def(init()) .def("schema_type", &SchemaInfo::getSchemaType) .def("name", &SchemaInfo::getName, return_value_policy::copy) .def("schema", &SchemaInfo::getSchema, return_value_policy::copy); } pulsar-client-python-3.12.0-candidate-0/src/table_view.cc000066400000000000000000000061731520101105200231560ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include #include #include #include #include #include #include #include #include "utils.h" namespace py = pybind11; using namespace pulsar; void export_table_view(py::module_& m) { py::class_(m, "TableViewConfiguration") .def(py::init<>()) .def("subscription_name", [](TableViewConfiguration& config, const std::string& name) { config.subscriptionName = name; }) .def("schema", [](TableViewConfiguration& config, const SchemaInfo& schema) { config.schemaInfo = schema; }); py::class_(m, "TableView") .def(py::init<>()) .def("get", [](const TableView& view, const std::string& key) -> std::pair { py::gil_scoped_release release; std::string value; bool available = view.getValue(key, value); py::gil_scoped_acquire acquire; if (available) { return std::make_pair(true, py::bytes(std::move(value))); } else { return std::make_pair(false, py::bytes()); } }) .def("size", &TableView::size, py::call_guard()) .def("for_each", [](TableView& view, std::function callback) { py::gil_scoped_release release; view.forEach([callback](const std::string& key, const std::string& value) { py::gil_scoped_acquire acquire; callback(key, py::bytes(value)); }); }) .def("for_each_and_listen", [](TableView& view, std::function callback) { py::gil_scoped_release release; view.forEachAndListen([callback](const std::string& key, const std::string& value) { py::gil_scoped_acquire acquire; callback(key, py::bytes(value)); }); }) .def("close", [](TableView& view) { waitForAsyncResult([&view](ResultCallback callback) { view.closeAsync(callback); }); }); } pulsar-client-python-3.12.0-candidate-0/src/utils.cc000066400000000000000000000037211520101105200221710ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #include "utils.h" void waitForAsyncResult(std::function func) { auto promise = std::make_shared>(); { // Always call the Pulsar C++ client methods without holding // the GIL. This avoids deadlocks due the sequence of acquiring // mutexes by different threads. eg: // Thread-1: GIL -> producer.lock // Thread-2: producer.lock -> GIL (In a callback) py::gil_scoped_release release; func([promise](Result result) { promise->set_value(result); }); } internal::waitForResult(*promise); } namespace internal { void waitForResult(std::promise& promise) { auto future = promise.get_future(); while (true) { { py::gil_scoped_release release; auto status = future.wait_for(std::chrono::milliseconds(100)); if (status == std::future_status::ready) { CHECK_RESULT(future.get()); return; } } py::gil_scoped_acquire acquire; if (PyErr_CheckSignals() != 0) { raiseException(ResultInterrupted); } } } } // namespace internal pulsar-client-python-3.12.0-candidate-0/src/utils.h000066400000000000000000000050431520101105200220320ustar00rootroot00000000000000/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ #pragma once #include #include #include #include #include #include #include "exceptions.h" using namespace pulsar; namespace py = pybind11; inline void CHECK_RESULT(Result res) { if (res != ResultOk) { raiseException(res); } } namespace internal { void waitForResult(std::promise& promise); } // namespace internal void waitForAsyncResult(std::function func); template inline T waitForAsyncValue(std::function)> func) { auto resultPromise = std::make_shared>(); auto valuePromise = std::make_shared>(); { py::gil_scoped_release release; func([resultPromise, valuePromise](Result result, const T& value) { valuePromise->set_value(value); resultPromise->set_value(result); }); } internal::waitForResult(*resultPromise); return valuePromise->get_future().get(); } struct CryptoKeyReaderWrapper { CryptoKeyReaderPtr cryptoKeyReader; CryptoKeyReaderWrapper(); CryptoKeyReaderWrapper(const std::string& publicKeyPath, const std::string& privateKeyPath); }; class CaptivePythonObjectMixin { protected: PyObject* _captive; CaptivePythonObjectMixin(PyObject* captive) { _captive = captive; PyGILState_STATE state = PyGILState_Ensure(); Py_XINCREF(_captive); PyGILState_Release(state); } ~CaptivePythonObjectMixin() { if (Py_IsInitialized()) { PyGILState_STATE state = PyGILState_Ensure(); Py_XDECREF(_captive); PyGILState_Release(state); } } }; pulsar-client-python-3.12.0-candidate-0/tests/000077500000000000000000000000001520101105200210725ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/tests/.gitignore000066400000000000000000000000171520101105200230600ustar00rootroot00000000000000.test-token.txtpulsar-client-python-3.12.0-candidate-0/tests/asyncio_test.py000066400000000000000000000473641520101105200241660ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # """ Unit tests for asyncio Pulsar client API. """ # pylint: disable=missing-function-docstring import asyncio import time from typing import List from unittest import ( main, IsolatedAsyncioTestCase, ) import pulsar # pylint: disable=import-error import _pulsar # pylint: disable=import-error from pulsar.asyncio import ( # pylint: disable=import-error Client, Consumer, Producer, PulsarException, _set_future, ) from pulsar.schema import ( # pylint: disable=import-error AvroSchema, Integer, Record, String, ) from urllib.request import urlopen, Request SERVICE_URL = 'pulsar://localhost:6650' ADMIN_URL = "http://localhost:8080" TIMEOUT_MS = 10000 # Do not wait forever in tests def doHttpPost(url, data): req = Request(url, data.encode()) req.add_header("Content-Type", "application/json") urlopen(req) def doHttpPut(url, data): try: req = Request(url, data.encode()) req.add_header("Content-Type", "application/json") req.get_method = lambda: "PUT" urlopen(req) except Exception as ex: # ignore conflicts exception to have test idempotency if "409" in str(ex): pass else: raise ex class AsyncioTest(IsolatedAsyncioTestCase): """Test cases for asyncio Pulsar client.""" async def asyncSetUp(self) -> None: self._client = Client(SERVICE_URL, operation_timeout_seconds=5) async def asyncTearDown(self) -> None: await self._client.close() async def test_batch_end_to_end(self): topic = f'asyncio-test-batch-e2e-{time.time()}' producer = await self._client.create_producer(topic, producer_name="my-producer") self.assertEqual(producer.topic(), f'persistent://public/default/{topic}') self.assertEqual(producer.producer_name(), "my-producer") tasks = [] for i in range(5): tasks.append(asyncio.create_task(producer.send(f'msg-{i}'.encode()))) msg_ids = await asyncio.gather(*tasks) self.assertEqual(len(msg_ids), 5) # pylint: disable=fixme self.assertEqual(producer.last_sequence_id(), 4) ledger_id = msg_ids[0].ledger_id() entry_id = msg_ids[0].entry_id() # These messages should be in the same entry for i in range(5): msg_id = msg_ids[i] print(f'{i} was sent to {msg_id}') self.assertIsInstance(msg_id, pulsar.MessageId) self.assertEqual(msg_ids[i].ledger_id(), ledger_id) self.assertEqual(msg_ids[i].entry_id(), entry_id) self.assertEqual(msg_ids[i].batch_index(), i) consumer = await self._client.subscribe(topic, 'sub', initial_position=pulsar.InitialPosition.Earliest) for i in range(5): msg = await consumer.receive() self.assertEqual(msg.data(), f'msg-{i}'.encode()) await consumer.close() # create a different subscription to verify initial position is latest by default consumer = await self._client.subscribe(topic, 'sub2') await producer.send(b'final-message') msg = await consumer.receive() self.assertEqual(msg.data(), b'final-message') async def test_send_keyed_message(self): topic = f'asyncio-test-send-keyed-message-{time.time()}' producer = await self._client.create_producer(topic) consumer = await self._client.subscribe(topic, 'sub') await producer.send(b'msg', partition_key='key0', ordering_key="key1", properties={'my-prop': 'my-value'}) msg = await consumer.receive() self.assertEqual(msg.data(), b'msg') self.assertEqual(msg.partition_key(), 'key0') self.assertEqual(msg.ordering_key(), 'key1') self.assertEqual(msg.properties(), {'my-prop': 'my-value'}) async def test_flush(self): topic = f'asyncio-test-flush-{time.time()}' producer = await self._client.create_producer(topic, batching_max_messages=3, batching_max_publish_delay_ms=60000) tasks = [] tasks.append(asyncio.create_task(producer.send(b'msg-0'))) tasks.append(asyncio.create_task(producer.send(b'msg-1'))) done, pending = await asyncio.wait(tasks, timeout=1, return_when=asyncio.FIRST_COMPLETED) self.assertEqual(len(done), 0) self.assertEqual(len(pending), 2) # flush will trigger sending the batched messages await producer.flush() for task in pending: self.assertTrue(task.done()) msg_id0 = tasks[0].result() msg_id1 = tasks[1].result() self.assertEqual(msg_id0.ledger_id(), msg_id1.ledger_id()) self.assertEqual(msg_id0.entry_id(), msg_id1.entry_id()) self.assertEqual(msg_id0.batch_index(), 0) self.assertEqual(msg_id1.batch_index(), 1) async def test_get_topics_partitions(self): topic_partitioned = "persistent://public/default/test_get_topics_partitions_async" topic_non_partitioned = "persistent://public/default/test_get_topics_async_not-partitioned" url1 = ADMIN_URL + "/admin/v2/persistent/public/default/test_get_topics_partitions_async/partitions" doHttpPut(url1, "3") self.assertEqual( await self._client.get_topic_partitions(topic_partitioned), [ "persistent://public/default/test_get_topics_partitions_async-partition-0", "persistent://public/default/test_get_topics_partitions_async-partition-1", "persistent://public/default/test_get_topics_partitions_async-partition-2", ], ) self.assertEqual(await self._client.get_topic_partitions(topic_non_partitioned), [topic_non_partitioned]) async def test_get_partitioned_topic_name(self): url1 = ADMIN_URL + "/admin/v2/persistent/public/default/partitioned_topic_name_test/partitions" doHttpPut(url1, "3") partitions = [ "persistent://public/default/partitioned_topic_name_test-partition-0", "persistent://public/default/partitioned_topic_name_test-partition-1", "persistent://public/default/partitioned_topic_name_test-partition-2", ] self.assertEqual( await self._client.get_topic_partitions("persistent://public/default/partitioned_topic_name_test"), partitions ) consumer = await self._client.subscribe( "persistent://public/default/partitioned_topic_name_test", "partitioned_topic_name_test_sub", consumer_type=pulsar.ConsumerType.Shared, ) producer = await self._client.create_producer("persistent://public/default/partitioned_topic_name_test") await producer.send(b"hello") msg = await asyncio.wait_for(consumer.receive(), TIMEOUT_MS / 1000) self.assertTrue(msg.topic_name() in partitions) async def test_create_producer_failure(self): try: await self._client.create_producer('tenant/ns/asyncio-test-send-failure') self.fail() except PulsarException as e: self.assertEqual(e.error(), pulsar.Result.Timeout) async def test_send_failure(self): producer = await self._client.create_producer('asyncio-test-send-failure') try: await producer.send(('x' * 1024 * 1024 * 10).encode()) self.fail() except PulsarException as e: self.assertEqual(e.error(), pulsar.Result.MessageTooBig) async def test_close_producer(self): producer = await self._client.create_producer('asyncio-test-close-producer') await producer.close() try: await producer.close() self.fail() except PulsarException as e: self.assertEqual(e.error(), pulsar.Result.AlreadyClosed) async def test_producer_is_connected(self): topic = f'asyncio-test-producer-is-connected-{time.time()}' producer = await self._client.create_producer(topic) self.assertTrue(producer.is_connected()) await producer.close() self.assertFalse(producer.is_connected()) async def test_shutdown_client(self): producer = await self._client.create_producer("persistent://public/default/partitioned_topic_name_test") await producer.send(b"hello") self._client.shutdown() try: await producer.send(b"hello") self.fail("Expected AlreadyClosed exception after client shutdown") except PulsarException as e: self.assertEqual(e.error(), pulsar.Result.AlreadyClosed) async def _prepare_messages(self, producer: Producer) -> List[pulsar.MessageId]: msg_ids = [] for i in range(5): msg_ids.append(await producer.send(f'msg-{i}'.encode())) return msg_ids async def test_consumer_cumulative_acknowledge(self): topic = f'asyncio-test-consumer-cumulative-ack-{time.time()}' sub = 'sub' consumer = await self._client.subscribe(topic, sub) producer = await self._client.create_producer(topic) await self._prepare_messages(producer) last_msg = None for _ in range(5): last_msg = await consumer.receive() await consumer.acknowledge_cumulative(last_msg) await consumer.close() consumer = await self._client.subscribe(topic, sub) await producer.send(b'final-message') msg = await consumer.receive() self.assertEqual(msg.data(), b'final-message') async def test_consumer_individual_acknowledge(self): topic = f'asyncio-test-consumer-individual-ack-{time.time()}' sub = 'sub' consumer = await self._client.subscribe(topic, sub, consumer_type=pulsar.ConsumerType.Shared) producer = await self._client.create_producer(topic) await self._prepare_messages(producer) msgs = [] for _ in range(5): msg = await consumer.receive() msgs.append(msg) await consumer.acknowledge(msgs[0]) await consumer.acknowledge(msgs[2]) await consumer.acknowledge(msgs[4]) await consumer.close() consumer = await self._client.subscribe(topic, sub, consumer_type=pulsar.ConsumerType.Shared) msg = await consumer.receive() self.assertEqual(msg.data(), b'msg-1') msg = await consumer.receive() self.assertEqual(msg.data(), b'msg-3') async def test_consumer_negative_acknowledge(self): topic = f'asyncio-test-consumer-negative-ack-{time.time()}' sub = 'sub' consumer = await self._client.subscribe(topic, sub, consumer_type=pulsar.ConsumerType.Shared, negative_ack_redelivery_delay_ms=100) producer = await self._client.create_producer(topic) await self._prepare_messages(producer) msgs = [] for _ in range(5): msg = await consumer.receive() msgs.append(msg) await consumer.acknowledge(msgs[1]) await consumer.acknowledge(msgs[3]) await consumer.negative_acknowledge(msgs[0]) await consumer.negative_acknowledge(msgs[2]) await consumer.negative_acknowledge(msgs[4]) await asyncio.sleep(0.2) received = [] for _ in range(3): msg = await consumer.receive() received.append(msg.data()) self.assertEqual(sorted(received), [b'msg-0', b'msg-2', b'msg-4']) await consumer.close() async def test_multi_topic_consumer(self): topics = ['asyncio-test-multi-topic-1', 'asyncio-test-multi-topic-2'] producers = [] for topic in topics: producer = await self._client.create_producer(topic) producers.append(producer) consumer = await self._client.subscribe(topics, 'test-multi-subscription') await producers[0].send(b'message-from-topic-1') await producers[1].send(b'message-from-topic-2') async def verify_receive(consumer: Consumer): received_messages = {} for _ in range(2): msg = await consumer.receive() received_messages[msg.data()] = None await consumer.acknowledge(msg.message_id()) self.assertEqual(received_messages, { b'message-from-topic-1': None, b'message-from-topic-2': None }) await verify_receive(consumer) await consumer.close() consumer = await self._client.subscribe('public/default/asyncio-test-multi-topic-.*', 'test-multi-subscription-2', is_pattern_topic=True, initial_position=pulsar.InitialPosition.Earliest) await verify_receive(consumer) await consumer.close() async def test_consumer_get_last_message_id(self): topic = f'asyncio-test-get-last-message-id-{time.time()}' sub = 'sub' consumer = await self._client.subscribe(topic, sub, consumer_type=pulsar.ConsumerType.Shared) producer = await self._client.create_producer(topic) for i in range(5): msg = f'msg-{i}'.encode() await producer.send(msg) last_msg_id = await consumer.get_last_message_id() assert isinstance(last_msg_id, _pulsar.MessageId) assert last_msg_id.entry_id() == i await consumer.acknowledge(last_msg_id) await consumer.close() async def test_async_dead_letter_policy(self): topic = f'asyncio-test-dlq-{time.time()}' dlq_topic = 'dlq-' + topic max_redeliver_count = 5 dlq_consumer = await self._client.subscribe(dlq_topic, "my-sub", consumer_type=pulsar.ConsumerType.Shared) consumer = await self._client.subscribe(topic, "my-sub", consumer_type=pulsar.ConsumerType.Shared, dead_letter_policy=pulsar.ConsumerDeadLetterPolicy(max_redeliver_count, dlq_topic, 'init-sub')) producer = await self._client.create_producer(topic) # Sen num msgs. num = 10 for i in range(num): await producer.send(b"hello-%d" % i) await producer.flush() # Redelivery all messages maxRedeliverCountNum time. for i in range(1, num * max_redeliver_count + num + 1): msg = await consumer.receive() if i % num == 0: consumer.redeliver_unacknowledged_messages() print(f"Start redeliver msgs '{i}'") with self.assertRaises(asyncio.TimeoutError): await asyncio.wait_for(consumer.receive(), 0.1) for i in range(num): msg = await dlq_consumer.receive() self.assertTrue(msg) self.assertEqual(msg.data(), b"hello-%d" % i) dlq_consumer.acknowledge(msg) with self.assertRaises(asyncio.TimeoutError): await asyncio.wait_for(dlq_consumer.receive(), 0.1) await consumer.close() await dlq_consumer.close() async def test_unsubscribe(self): topic = f'asyncio-test-unsubscribe-{time.time()}' sub = 'sub' consumer = await self._client.subscribe(topic, sub) await consumer.unsubscribe() # Verify the consumer can be created successfully with the same subscription name consumer = await self._client.subscribe(topic, sub) await consumer.close() async def test_seek_message_id(self): topic = f'asyncio-test-seek-message-id-{time.time()}' sub = 'sub' producer = await self._client.create_producer(topic) msg_ids = await self._prepare_messages(producer) consumer = await self._client.subscribe( topic, sub, initial_position=pulsar.InitialPosition.Earliest ) await consumer.seek(msg_ids[2]) msg = await consumer.receive() self.assertEqual(msg.data(), b'msg-3') await consumer.close() consumer = await self._client.subscribe( topic, sub, initial_position=pulsar.InitialPosition.Earliest, start_message_id_inclusive=True ) await consumer.seek(msg_ids[2]) msg = await consumer.receive() self.assertEqual(msg.data(), b'msg-2') await consumer.close() async def test_seek_timestamp(self): topic = f'asyncio-test-seek-timestamp-{time.time()}' sub = 'sub' consumer = await self._client.subscribe( topic, sub, initial_position=pulsar.InitialPosition.Earliest ) producer = await self._client.create_producer(topic) # Send first 3 messages for i in range(3): await producer.send(f'msg-{i}'.encode()) seek_time = int(time.time() * 1000) # Send 2 more messages for i in range(3, 5): await producer.send(f'msg-{i}'.encode()) # Consume all messages first for i in range(5): msg = await consumer.receive() self.assertEqual(msg.data(), f'msg-{i}'.encode()) # Seek to the timestamp (should start from msg-3) await consumer.seek(seek_time) msg = await consumer.receive() self.assertEqual(msg.data(), b'msg-3') async def test_schema(self): class ExampleRecord(Record): # pylint: disable=too-few-public-methods """Example record schema for testing.""" str_field = String() int_field = Integer() topic = f'asyncio-test-schema-{time.time()}' producer = await self._client.create_producer( topic, schema=AvroSchema(ExampleRecord) ) consumer = await self._client.subscribe( topic, 'sub', schema=AvroSchema(ExampleRecord) ) await producer.send(ExampleRecord(str_field='test', int_field=42)) msg = await consumer.receive() self.assertIsInstance(msg.value(), ExampleRecord) self.assertEqual(msg.value().str_field, 'test') self.assertEqual(msg.value().int_field, 42) class AsyncioSetFutureTest(IsolatedAsyncioTestCase): """Tests for asyncio bridge helpers (no live Pulsar broker).""" async def test_set_future_noop_when_future_cancelled(self): loop = asyncio.get_running_loop() fut = loop.create_future() fut.cancel() _set_future(fut, _pulsar.Result.Ok, None) await asyncio.sleep(0) self.assertTrue(fut.cancelled()) async def test_set_future_noop_when_future_already_resolved(self): loop = asyncio.get_running_loop() fut = loop.create_future() fut.set_result("first") _set_future(fut, _pulsar.Result.Ok, "late") await asyncio.sleep(0) self.assertEqual(fut.result(), "first") if __name__ == '__main__': main() pulsar-client-python-3.12.0-candidate-0/tests/auto_cluster_failover_test.py000066400000000000000000000214771520101105200271160ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import shutil import time from unittest import SkipTest, TestCase, main from urllib.error import URLError from urllib.request import Request, urlopen import pulsar from pulsar import AutoClusterFailover, Client, MessageId, ServiceInfo try: from testcontainers.core.container import DockerContainer except ImportError: DockerContainer = None class AutoClusterFailoverDockerTest(TestCase): PRIMARY_URL = "pulsar://localhost:16650" PRIMARY_ADMIN_URL = "http://localhost:18080" SECONDARY_URL = "pulsar://localhost:26650" SECONDARY_ADMIN_URL = "http://localhost:28080" RECEIVE_TIMEOUT_MS = 10000 FAILOVER_WAIT_SECONDS = 30 @classmethod def setUpClass(cls): if shutil.which("docker") is None: raise SkipTest("docker is required for auto_cluster_failover_test") if DockerContainer is None: raise SkipTest("testcontainers is required for auto_cluster_failover_test") try: cls.primary_container = cls._create_container( service_port=16650, admin_port=18080, cluster_name="standalone-0", ) cls.secondary_container = cls._create_container( service_port=26650, admin_port=28080, cluster_name="standalone-1", ) cls.primary_container.start() cls.secondary_container.start() cls._wait_for_http(cls.PRIMARY_ADMIN_URL + "/metrics") cls._wait_for_http(cls.SECONDARY_ADMIN_URL + "/metrics") cls._configure_cluster( cls.PRIMARY_ADMIN_URL, cls.PRIMARY_URL, "standalone-0", ) cls._configure_cluster( cls.SECONDARY_ADMIN_URL, cls.SECONDARY_URL, "standalone-1", ) except Exception: cls._print_container_logs() raise @classmethod def tearDownClass(cls): for container in ( getattr(cls, "primary_container", None), getattr(cls, "secondary_container", None), ): if container is not None: container.stop() @classmethod def _create_container(cls, service_port, admin_port, cluster_name): return ( DockerContainer("apachepulsar/pulsar:latest") .with_env("clusterName", cluster_name) .with_env("advertisedAddress", "localhost") .with_env("advertisedListeners", f"external:pulsar://localhost:{service_port}") .with_env("PULSAR_MEM", "-Xms512m -Xmx512m -XX:MaxDirectMemorySize=256m") .with_bind_ports(6650, service_port) .with_bind_ports(8080, admin_port) .with_command( 'bash -c "bin/apply-config-from-env.py conf/standalone.conf && ' 'exec bin/pulsar standalone -nss -nfw"' ) ) @classmethod def _print_container_logs(cls): for container in ( getattr(cls, "primary_container", None), getattr(cls, "secondary_container", None), ): if container is None: continue wrapped = container.get_wrapped_container() try: print(wrapped.logs().decode("utf-8", errors="replace")) except Exception: pass @classmethod def _wait_for_http(cls, url, timeout_seconds=180): deadline = time.time() + timeout_seconds last_error = None while time.time() < deadline: try: with urlopen(url, timeout=5) as response: if response.status == 200: return except (URLError, OSError) as e: last_error = e time.sleep(1) raise AssertionError(f"Timed out waiting for {url}: {last_error}") @staticmethod def _http_put(url, data): request = Request(url, data.encode("utf-8")) request.add_header("Content-Type", "application/json") request.get_method = lambda: "PUT" try: with urlopen(request, timeout=10): return except URLError as e: if "409" in str(e): return raise @classmethod def _configure_cluster(cls, admin_url, service_url, cluster_name): cls._http_put( f"{admin_url}/admin/v2/clusters/{cluster_name}", """ { "serviceUrl": "%s/", "brokerServiceUrl": "%s/" } """ % (admin_url, service_url), ) cls._http_put( f"{admin_url}/admin/v2/tenants/public", """ { "adminRoles": ["anonymous"], "allowedClusters": ["%s"] } """ % cluster_name, ) cls._http_put( f"{admin_url}/admin/v2/namespaces/public/default", """ { "replication_clusters": ["%s"] } """ % cluster_name, ) @staticmethod def _wait_until(predicate, timeout_seconds, description): deadline = time.time() + timeout_seconds while time.time() < deadline: if predicate(): return time.sleep(0.2) raise AssertionError(f"Timed out waiting for {description}") @staticmethod def _ensure_topic_exists(service_url, topic): client = Client(service_url) producer = client.create_producer(topic) producer.close() client.close() def test_producer_failover_between_two_standalones(self): topic = "test-auto-cluster-failover-%d" % int(time.time() * 1000) message_before_failover = b"before-failover" message_after_failover = b"after-failover" self._ensure_topic_exists(self.PRIMARY_URL, topic) self._ensure_topic_exists(self.SECONDARY_URL, topic) primary_client = Client(self.PRIMARY_URL) primary_reader = primary_client.create_reader(topic, MessageId.earliest) secondary_client = Client(self.SECONDARY_URL) secondary_reader = secondary_client.create_reader(topic, MessageId.earliest) failover_client = Client( AutoClusterFailover( ServiceInfo(self.PRIMARY_URL), [ServiceInfo(self.SECONDARY_URL)], check_interval_ms=200, failover_threshold=1, switch_back_threshold=1, ), operation_timeout_seconds=10, ) producer = failover_client.create_producer( topic, send_timeout_millis=3000, batching_enabled=False, ) self.assertEqual(failover_client.get_service_info().service_url, self.PRIMARY_URL) producer.send(message_before_failover) self.assertEqual(primary_reader.read_next(self.RECEIVE_TIMEOUT_MS).data(), message_before_failover) primary_reader.close() primary_client.close() self.primary_container.get_wrapped_container().kill(signal="SIGTERM") self._wait_until( lambda: failover_client.get_service_info().service_url == self.SECONDARY_URL, self.FAILOVER_WAIT_SECONDS, "client service info to switch to the secondary broker", ) last_error = None deadline = time.time() + self.FAILOVER_WAIT_SECONDS while time.time() < deadline: try: producer.send(message_after_failover) break except pulsar.PulsarException as e: last_error = e time.sleep(0.5) else: raise AssertionError(f"Producer did not recover after failover: {last_error}") self.assertEqual(secondary_reader.read_next(self.RECEIVE_TIMEOUT_MS).data(), message_after_failover) self.assertEqual(failover_client.get_service_info().service_url, self.SECONDARY_URL) producer.close() failover_client.close() secondary_reader.close() secondary_client.close() if __name__ == "__main__": main() pulsar-client-python-3.12.0-candidate-0/tests/custom_logger_test.py000077500000000000000000000063521520101105200253650ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from unittest import TestCase, main import asyncio import logging import threading from pulsar import Client class CustomLoggingTest(TestCase): serviceUrl = 'pulsar://localhost:6650' def test_async_func_with_custom_logger(self): # boost::python::call may fail in C++ destructors, even worse, calls # to PyErr_Print could corrupt the Python interpreter. # See https://github.com/boostorg/python/issues/374 for details. # This test is to verify these functions won't be called in C++ destructors # so that Python's async function works well. client = Client( self.serviceUrl, logger=logging.getLogger('custom-logger') ) async def async_get(value): consumer = client.subscribe('test_async_get', 'sub') consumer.close() return value value = 'foo' result = asyncio.run(async_get(value)) self.assertEqual(value, result) client.close() def test_logger_thread_leaks(self): def _do_connect(close): logger = logging.getLogger(str(threading.current_thread().ident)) logger.setLevel(logging.INFO) client = Client( service_url="pulsar://localhost:6650", io_threads=4, message_listener_threads=4, operation_timeout_seconds=1, log_conf_file_path=None, authentication=None, logger=logger, ) client.get_topic_partitions("persistent://public/default/partitioned_topic_name_test") if close: client.close() for should_close in (True, False): self.assertEqual(threading.active_count(), 1, "Explicit close: {}; baseline is 1 thread".format(should_close)) _do_connect(should_close) self.assertEqual(threading.active_count(), 1, "Explicit close: {}; synchronous connect doesn't leak threads".format(should_close)) threads = [] for _ in range(10): threads.append(threading.Thread(target=_do_connect, args=(should_close))) threads[-1].start() for thread in threads: thread.join() assert threading.active_count() == 1, "Explicit close: {}; threaded connect in parallel doesn't leak threads".format(should_close) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG) main() pulsar-client-python-3.12.0-candidate-0/tests/debug_logger_test.py000066400000000000000000000030711520101105200251310ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from unittest import TestCase, main import pulsar class DebugLoggerTest(TestCase): def test_configure_log_level(self): client = pulsar.Client( service_url="pulsar://localhost:6650", logger=pulsar.ConsoleLogger(pulsar.LoggerLevel.Debug) ) producer = client.create_producer( topic='test_log_level' ) producer.send(b'hello') def test_configure_log_to_file(self): client = pulsar.Client( service_url="pulsar://localhost:6650", logger=pulsar.FileLogger(pulsar.LoggerLevel.Debug, 'test.log') ) producer = client.create_producer( topic='test_log_to_file' ) producer.send(b'hello') if __name__ == "__main__": main() pulsar-client-python-3.12.0-candidate-0/tests/interrupted_test.py000066400000000000000000000031641520101105200250540ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from unittest import TestCase, main import pulsar import signal import time import threading class InterruptedTest(TestCase): service_url = 'pulsar://localhost:6650' def test_sigint(self): def thread_function(): time.sleep(1) signal.raise_signal(signal.SIGINT) client = pulsar.Client(self.service_url) consumer = client.subscribe('test-sigint', "my-sub") thread = threading.Thread(target=thread_function) thread.start() start = time.time() with self.assertRaises(pulsar.Interrupted): consumer.receive() finish = time.time() print(f"time: {finish - start}") self.assertGreater(finish - start, 1) self.assertLess(finish - start, 1.5) client.close() if __name__ == '__main__': main() pulsar-client-python-3.12.0-candidate-0/tests/oauth2_test.py000066400000000000000000000072451520101105200237150ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from unittest import TestCase, main from pulsar import AuthenticationOauth2, AuthenticationError, Client import base64 import os # This test should run against the standalone that is set up with # build-support/docker-compose-pulsar-oauth2.yml class Oauth2Test(TestCase): service_url = 'pulsar://localhost:6650' def test_invalid_private_key(self): def test_create_client(auth_params_string): client = Client(self.service_url, authentication=AuthenticationOauth2(auth_params_string)) with self.assertRaises(AuthenticationError): client.create_producer('oauth2-test-base64') client.close() test_create_client('{"private_key":"xxx:yyy"}') test_create_client('{"private_key":"data:"}') test_create_client('{"private_key":"data:application/x-pem"}') test_create_client('{"private_key":"data:application/json;xxx"}') def test_key_file(self): path = (os.path.dirname(os.path.abspath(__file__)) + '/test-conf/cpp_credentials_file.json') auth = AuthenticationOauth2(f'''{{ "issuer_url": "https://dev-kt-aa9ne.us.auth0.com", "private_key": "{path}", "audience": "https://dev-kt-aa9ne.us.auth0.com/api/v2/" }}''') client = Client(self.service_url, authentication=auth) producer = client.create_producer('oauth2-test-base64') producer.close() client.close() def test_base64(self): credentials = '''{ "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb" }''' base64_credentials = base64.b64encode(credentials.encode()).decode() auth = AuthenticationOauth2(f'''{{ "issuer_url": "https://dev-kt-aa9ne.us.auth0.com", "private_key": "data:application/json;base64,{base64_credentials}", "audience": "https://dev-kt-aa9ne.us.auth0.com/api/v2/" }}''') client = Client(self.service_url, authentication=auth) producer = client.create_producer('oauth2-test-base64') producer.close() client.close() def test_wrong_secret(self): credentials = '''{ "client_id": "my-id", "client_secret":"my-secret" }''' base64_credentials = base64.b64encode(credentials.encode()).decode() auth = AuthenticationOauth2(f'''{{ "issuer_url": "https://dev-kt-aa9ne.us.auth0.com", "private_key": "data:application/json;base64,{base64_credentials}", "audience": "https://dev-kt-aa9ne.us.auth0.com/api/v2/" }}''') client = Client(self.service_url, authentication=auth) with self.assertRaises(AuthenticationError): client.create_producer('oauth2-test-base64') client.close() if __name__ == '__main__': main() pulsar-client-python-3.12.0-candidate-0/tests/pulsar_test.py000077500000000000000000002451701520101105200240250ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import random from typing import Optional from unittest import TestCase, main import time import os import re import pulsar import uuid from datetime import timedelta from pulsar import ( Client, MessageId, CompressionType, ConsumerType, KeySharedMode, ConsumerKeySharedPolicy, PartitionsRoutingMode, AuthenticationBasic, AuthenticationTLS, Authentication, AuthenticationToken, InitialPosition, CryptoKeyReader, ConsumerBatchReceivePolicy, ProducerAccessMode, ConsumerDeadLetterPolicy, ServiceInfoProvider, ServiceInfo, AutoClusterFailover, ) from pulsar.schema import JsonSchema, Record, Integer from _pulsar import ProducerConfiguration, ConsumerConfiguration, RegexSubscriptionMode from urllib.request import urlopen, Request TM = 10000 # Do not wait forever in tests TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) TOKEN_PATH = TESTS_DIR + "/.test-token.txt" CERTS_DIR = TESTS_DIR + "/test-conf/" def doHttpPost(url, data): req = Request(url, data.encode()) req.add_header("Content-Type", "application/json") urlopen(req) def doHttpPut(url, data): try: req = Request(url, data.encode()) req.add_header("Content-Type", "application/json") req.get_method = lambda: "PUT" urlopen(req) except Exception as ex: # ignore conflicts exception to have test idempotency if "409" in str(ex): pass else: raise ex def doHttpGet(url): req = Request(url) req.add_header("Accept", "application/json") return urlopen(req).read() class TestRecord(Record): a = Integer() b = Integer() class PulsarTest(TestCase): serviceUrl = "pulsar://localhost:6650" adminUrl = "http://localhost:8080" serviceUrlTls = "pulsar+ssl://localhost:6651" class StaticServiceInfoProvider(ServiceInfoProvider): def __init__(self, initial_service_info): self.initial = initial_service_info self.callback = None self.closed = False def initial_service_info(self): return self.initial def initialize(self, on_service_info_update): self.callback = on_service_info_update def close(self): self.closed = True class InvalidServiceInfoProvider(ServiceInfoProvider): def initial_service_info(self): return "invalid" def initialize(self, on_service_info_update): pass def test_producer_config(self): conf = ProducerConfiguration() conf.send_timeout_millis(12) self.assertEqual(conf.send_timeout_millis(), 12) self.assertEqual(conf.compression_type(), CompressionType.NONE) conf.compression_type(CompressionType.LZ4) self.assertEqual(conf.compression_type(), CompressionType.LZ4) conf.max_pending_messages(120) self.assertEqual(conf.max_pending_messages(), 120) def test_consumer_config(self): conf = ConsumerConfiguration() self.assertEqual(conf.consumer_type(), ConsumerType.Exclusive) conf.consumer_type(ConsumerType.Shared) self.assertEqual(conf.consumer_type(), ConsumerType.Shared) self.assertEqual(conf.consumer_name(), "") conf.consumer_name("my-name") self.assertEqual(conf.consumer_name(), "my-name") self.assertEqual(conf.replicate_subscription_state_enabled(), False) conf.replicate_subscription_state_enabled(True) self.assertEqual(conf.replicate_subscription_state_enabled(), True) def test_connect_error(self): with self.assertRaises(ValueError): Client("fakeServiceUrl") def test_exception_inheritance(self): assert issubclass(pulsar.ConnectError, pulsar.PulsarException) assert issubclass(pulsar.PulsarException, Exception) def test_simple_producer(self): client = Client(self.serviceUrl) producer = client.create_producer("my-python-topic") producer.send(b"hello") producer.close() client.close() def test_producer_send_async(self): client = Client(self.serviceUrl) producer = client.create_producer("my-python-topic") sent_messages = [] def send_callback(producer, msg): sent_messages.append(msg) producer.send_async(b"hello", send_callback) producer.send_async(b"hello", send_callback) producer.send_async(b"hello", send_callback) i = 0 while len(sent_messages) < 3 and i < 100: time.sleep(0.1) i += 1 self.assertEqual(len(sent_messages), 3) client.close() def test_producer_send(self): client = Client(self.serviceUrl) topic = "test_producer_send" producer = client.create_producer(topic) consumer = client.subscribe(topic, "sub-name") msg_id = producer.send(b"hello") print("send to {}".format(msg_id)) msg = consumer.receive(TM) consumer.acknowledge(msg) print("receive from {}".format(msg.message_id())) self.assertEqual(msg_id, msg.message_id()) self.assertIsNone(msg.encryption_context()) client.close() def test_producer_access_mode_exclusive(self): client = Client(self.serviceUrl) topic_name = "test-access-mode-exclusive" client.create_producer(topic_name, producer_name="p1", access_mode=ProducerAccessMode.Exclusive) with self.assertRaises(pulsar.ProducerFenced): client.create_producer(topic_name, producer_name="p2", access_mode=ProducerAccessMode.Exclusive) client.close() def test_producer_access_mode_wait_exclusive(self): client = Client(self.serviceUrl) topic_name = "test_producer_access_mode_wait_exclusive" producer1 = client.create_producer( topic=topic_name, producer_name='p-1', access_mode=ProducerAccessMode.Exclusive ) assert producer1.producer_name() == 'p-1' # when p1 close, p2 success created. producer1.close() producer2 = client.create_producer( topic=topic_name, producer_name='p-2', access_mode=ProducerAccessMode.WaitForExclusive ) assert producer2.producer_name() == 'p-2' producer2.close() client.close() def test_producer_access_mode_exclusive_with_fencing(self): client = Client(self.serviceUrl) topic_name = 'test_producer_access_mode_exclusive_with_fencing' producer1 = client.create_producer( topic=topic_name, producer_name='p-1', access_mode=ProducerAccessMode.Exclusive ) assert producer1.producer_name() == 'p-1' producer2 = client.create_producer( topic=topic_name, producer_name='p-2', access_mode=ProducerAccessMode.ExclusiveWithFencing ) assert producer2.producer_name() == 'p-2' # producer1 will be fenced. with self.assertRaises((pulsar.ProducerFenced, pulsar.AlreadyClosed)): producer1.send('test-msg'.encode('utf-8')) # sleep 200ms to make sure producer1 is close done. time.sleep(0.2) producer2.close() client.close() def test_producer_is_connected(self): client = Client(self.serviceUrl) topic = "test_producer_is_connected" producer = client.create_producer(topic) self.assertTrue(producer.is_connected()) producer.close() self.assertFalse(producer.is_connected()) client.close() def test_producer_consumer(self): client = Client(self.serviceUrl) consumer = client.subscribe("my-python-topic-producer-consumer", "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer("my-python-topic-producer-consumer", producer_name="my-producer") producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") self.assertEqual(msg.producer_name(), "my-producer") with self.assertRaises(pulsar.Timeout): consumer.receive(100) consumer.unsubscribe() client.close() def test_ordering_key(self): client = Client(self.serviceUrl) consumer = client.subscribe( "my-python-topic-ordering-key", "my-sub", consumer_type=ConsumerType.KeyShared ) producer = client.create_producer("my-python-topic-ordering-key") producer.send(b"ordered-hello", ordering_key="random-ordering-key") # Message should be available immediately with ordering key set msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"ordered-hello") self.assertEqual(msg.ordering_key(), "random-ordering-key") consumer.unsubscribe() producer.close() client.close() def test_redelivery_count(self): client = Client(self.serviceUrl) consumer = client.subscribe( "my-python-topic-redelivery-count", "my-sub", consumer_type=ConsumerType.Shared, negative_ack_redelivery_delay_ms=500, ) producer = client.create_producer("my-python-topic-redelivery-count") producer.send(b"hello") redelivery_count = 0 for i in range(4): msg = consumer.receive(TM) print("Received message %s" % msg.data()) if i % 2 == 0: consumer.negative_acknowledge(msg) else: consumer.negative_acknowledge(msg.message_id()) redelivery_count = msg.redelivery_count() self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") self.assertEqual(3, redelivery_count) consumer.unsubscribe() producer.close() client.close() def test_deliver_at(self): client = Client(self.serviceUrl) consumer = client.subscribe("my-python-topic-deliver-at", "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer("my-python-topic-deliver-at") # Delay message in 1.1s producer.send(b"hello", deliver_at=int(round(time.time() * 1000)) + 1100) # Message should not be available in the next second with self.assertRaises(pulsar.Timeout): consumer.receive(1000) # Message should be published now msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") consumer.unsubscribe() producer.close() client.close() def test_deliver_after(self): client = Client(self.serviceUrl) consumer = client.subscribe("my-python-topic-deliver-after", "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer("my-python-topic-deliver-after") # Delay message in 1.1s producer.send(b"hello", deliver_after=timedelta(milliseconds=1100)) # Message should not be available in the next second with self.assertRaises(pulsar.Timeout): consumer.receive(1000) # Message should be published in the next 500ms msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") consumer.unsubscribe() producer.close() client.close() def test_consumer_initial_position(self): client = Client(self.serviceUrl) producer = client.create_producer("consumer-initial-position") # Sending 5 messages before consumer creation. # These should be received with initial_position set to Earliest but not with Latest. for i in range(5): producer.send(b"hello-%d" % i) consumer = client.subscribe( "consumer-initial-position", "my-sub", consumer_type=ConsumerType.Shared, initial_position=InitialPosition.Earliest, ) # Sending 5 other messages that should be received regardless of the initial_position. for i in range(5, 10): producer.send(b"hello-%d" % i) for i in range(10): msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello-%d" % i) with self.assertRaises(pulsar.Timeout): consumer.receive(100) consumer.unsubscribe() client.close() def test_consumer_queue_size_is_zero(self): client = Client(self.serviceUrl) consumer = client.subscribe( "my-python-topic-consumer-init-queue-size-is-zero", "my-sub", consumer_type=ConsumerType.Shared, receiver_queue_size=0, initial_position=InitialPosition.Earliest, ) producer = client.create_producer("my-python-topic-consumer-init-queue-size-is-zero") producer.send(b"hello") time.sleep(0.1) msg = consumer.receive() self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") consumer.unsubscribe() client.close() def test_message_properties(self): client = Client(self.serviceUrl) topic = "my-python-test-message-properties" consumer = client.subscribe( topic=topic, subscription_name="my-subscription", schema=pulsar.schema.StringSchema() ) producer = client.create_producer(topic=topic, schema=pulsar.schema.StringSchema()) producer.send("hello", properties={"a": "1", "b": "2"}) msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.value(), "hello") self.assertEqual(msg.properties(), {"a": "1", "b": "2"}) consumer.unsubscribe() client.close() def test_tls_auth(self): client = Client( self.serviceUrlTls, tls_trust_certs_file_path=CERTS_DIR + "cacert.pem", tls_allow_insecure_connection=False, authentication=AuthenticationTLS(CERTS_DIR + "client-cert.pem", CERTS_DIR + "client-key.pem"), ) topic = "my-python-topic-tls-auth-" + str(time.time()) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): consumer.receive(100) client.close() def test_tls_auth2(self): authPlugin = "org.apache.pulsar.client.impl.auth.AuthenticationTls" authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (CERTS_DIR, CERTS_DIR) client = Client( self.serviceUrlTls, tls_trust_certs_file_path=CERTS_DIR + "cacert.pem", tls_allow_insecure_connection=False, authentication=Authentication(authPlugin, authParams), ) topic = "my-python-topic-tls-auth-2-" + str(time.time()) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): consumer.receive(100) client.close() def test_encryption(self): publicKeyPath = CERTS_DIR + "public-key.client-rsa.pem" privateKeyPath = CERTS_DIR + "private-key.client-rsa.pem" crypto_key_reader = CryptoKeyReader(publicKeyPath, privateKeyPath) client = Client(self.serviceUrl) topic = "my-python-test-end-to-end-encryption" consumer = client.subscribe( topic=topic, subscription_name="my-subscription", crypto_key_reader=crypto_key_reader ) producer = client.create_producer( topic=topic, encryption_key="client-rsa.pem", crypto_key_reader=crypto_key_reader ) reader = client.create_reader( topic=topic, start_message_id=MessageId.earliest, crypto_key_reader=crypto_key_reader ) producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.value(), b"hello") consumer.unsubscribe() msg = reader.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): reader.read_next(100) reader.close() client.close() def test_encryption_failure(self): publicKeyPath = CERTS_DIR + "public-key.client-rsa.pem" privateKeyPath = CERTS_DIR + "private-key.client-rsa.pem" crypto_key_reader = CryptoKeyReader(publicKeyPath, privateKeyPath) client = Client(self.serviceUrl) topic = "my-python-test-end-to-end-encryption-failure-" + str(time.time()) producer = client.create_producer( topic=topic, encryption_key="client-rsa.pem", crypto_key_reader=crypto_key_reader, compression_type=CompressionType.LZ4 ) producer.send(b"msg-0") def verify_encryption_context(context: pulsar.EncryptionContext | None, failed: bool, batch_size: int): if context is None: self.fail("Encryption context is None") keys = context.keys() self.assertEqual(len(keys), 1) key = keys[0] self.assertEqual(key.key, "client-rsa.pem") self.assertGreater(len(key.value), 0) self.assertEqual(key.metadata, {}) self.assertGreater(len(context.param()), 0) self.assertEqual(context.algorithm(), "") self.assertEqual(context.compression_type(), CompressionType.LZ4) if batch_size == -1: self.assertEqual(context.uncompressed_message_size(), len(b"msg-0")) else: self.assertGreater(context.uncompressed_message_size(), len(b"msg-0")) self.assertEqual(context.batch_size(), batch_size) self.assertEqual(context.is_decryption_failed(), failed) def verify_next_message(value: bytes): consumer = client.subscribe(topic, subscription, crypto_key_reader=crypto_key_reader) msg = consumer.receive(3000) self.assertEqual(msg.data(), value) verify_encryption_context(msg.encryption_context(), False, -1) consumer.acknowledge(msg) consumer.close() subscription = "my-sub" consumer = client.subscribe(topic, subscription, initial_position=InitialPosition.Earliest, crypto_failure_action=pulsar.ConsumerCryptoFailureAction.FAIL) with self.assertRaises(pulsar.Timeout): consumer.receive(3000) consumer.close() producer.send(b"msg-1") verify_next_message(b"msg-0") # msg-0 won't be skipped consumer = client.subscribe(topic, subscription, initial_position=InitialPosition.Earliest, crypto_failure_action=pulsar.ConsumerCryptoFailureAction.DISCARD) with self.assertRaises(pulsar.Timeout): consumer.receive(3000) consumer.close() producer.send(b"msg-2") verify_next_message(b"msg-2") # msg-1 is skipped since the crypto failure action is DISCARD producer.close() # send batched messages producer = client.create_producer( topic=topic, encryption_key="client-rsa.pem", crypto_key_reader=crypto_key_reader, compression_type=CompressionType.LZ4, batching_enabled=True, ) producer.send_async(b"msg-3", None) producer.send_async(b"msg-4", None) producer.flush() def verify_undecrypted_message(msg: pulsar.Message, i: int): self.assertNotEqual(msg.data(), f"msg-{i}".encode()) self.assertGreater(len(msg.data()), 5, f"msg.data() is {msg.data()}") verify_encryption_context(msg.encryption_context(), True, 2 if i >= 3 else -1) # Encrypted messages will be consumed since the crypto failure action is CONSUME # Only 4 messages can be received because msg-3 and msg-4 are sent in batch and they are delivered # as a single message when decryption fails. consumer = client.subscribe(topic, 'another-sub', initial_position=InitialPosition.Earliest, crypto_failure_action=pulsar.ConsumerCryptoFailureAction.CONSUME) for i in range(4): msg = consumer.receive(3000) verify_undecrypted_message(msg, i) reader = client.create_reader(topic, MessageId.earliest, crypto_failure_action=pulsar.ConsumerCryptoFailureAction.CONSUME) for i in range(4): msg = reader.read_next(3000) verify_undecrypted_message(msg, i) client.close() def test_tls_auth3(self): authPlugin = "tls" authParams = "tlsCertFile:%s/client-cert.pem,tlsKeyFile:%s/client-key.pem" % (CERTS_DIR, CERTS_DIR) client = Client( self.serviceUrlTls, tls_trust_certs_file_path=CERTS_DIR + "cacert.pem", tls_allow_insecure_connection=False, authentication=Authentication(authPlugin, authParams), ) topic = "my-python-topic-tls-auth-3-" + str(time.time()) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): consumer.receive(100) client.close() def test_auth_junk_params(self): authPlugin = "someoldjunk.so" authParams = "blah" client = Client( self.serviceUrlTls, tls_trust_certs_file_path=CERTS_DIR + "cacert.pem", tls_allow_insecure_connection=False, authentication=Authentication(authPlugin, authParams), ) with self.assertRaises(pulsar.AuthenticationError): client.subscribe("my-python-topic-auth-junk-params", "my-sub", consumer_type=ConsumerType.Shared) def test_message_listener(self): client = Client(self.serviceUrl) received_messages = [] def listener(consumer, msg): print("Got message: %s" % msg) received_messages.append(msg) consumer.acknowledge(msg) client.subscribe( "my-python-topic-listener", "my-sub", consumer_type=ConsumerType.Exclusive, message_listener=listener ) producer = client.create_producer("my-python-topic-listener") producer.send(b"hello-1") producer.send(b"hello-2") producer.send(b"hello-3") time.sleep(0.1) self.assertEqual(len(received_messages), 3) self.assertEqual(received_messages[0].data(), b"hello-1") self.assertEqual(received_messages[1].data(), b"hello-2") self.assertEqual(received_messages[2].data(), b"hello-3") client.close() def test_consumer_is_connected(self): client = Client(self.serviceUrl) topic = "test_consumer_is_connected" sub = "sub" consumer = client.subscribe(topic, sub) self.assertTrue(consumer.is_connected()) consumer.close() self.assertFalse(consumer.is_connected()) client.close() def test_reader_simple(self): client = Client(self.serviceUrl) reader = client.create_reader("my-python-topic-reader-simple", MessageId.earliest) producer = client.create_producer("my-python-topic-reader-simple") producer.send(b"hello") msg = reader.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): reader.read_next(100) reader.close() client.close() def test_reader_on_last_message(self): client = Client(self.serviceUrl) producer = client.create_producer("my-python-topic-reader-on-last-message") for i in range(10): producer.send(b"hello-%d" % i) reader = client.create_reader("my-python-topic-reader-on-last-message", MessageId.latest) for i in range(10, 20): producer.send(b"hello-%d" % i) for i in range(10, 20): msg = reader.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello-%d" % i) reader.close() client.close() def test_reader_on_specific_message(self): num_of_msgs = 10 client = Client(self.serviceUrl) producer = client.create_producer("my-python-topic-reader-on-specific-message") for i in range(num_of_msgs): producer.send(b"hello-%d" % i) reader1 = client.create_reader("my-python-topic-reader-on-specific-message", MessageId.earliest) for i in range(num_of_msgs // 2): msg = reader1.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello-%d" % i) last_msg_id = msg.message_id() last_msg_idx = i reader2 = client.create_reader("my-python-topic-reader-on-specific-message", last_msg_id) # The reset would be effectively done on the next position relative to reset. # When available, we should test this behaviour with `startMessageIdInclusive` opt. from_msg_idx = last_msg_idx for i in range(from_msg_idx + 1, num_of_msgs): msg = reader2.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello-%d" % i) reader1.close() reader2.close() client.close() def test_reader_on_specific_message_with_batches(self): client = Client(self.serviceUrl) producer = client.create_producer( "my-python-topic-reader-on-specific-message-with-batches", batching_enabled=True, batching_max_publish_delay_ms=1000, ) for i in range(10): producer.send_async(b"hello-%d" % i, None) # Send one sync message to make sure everything was published producer.send(b"hello-10") reader1 = client.create_reader("my-python-topic-reader-on-specific-message-with-batches", MessageId.earliest) for i in range(5): msg = reader1.read_next(TM) last_msg_id = msg.message_id() reader2 = client.create_reader("my-python-topic-reader-on-specific-message-with-batches", last_msg_id) for i in range(5, 11): msg = reader2.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello-%d" % i) reader1.close() reader2.close() client.close() def test_reader_on_partitioned_topic(self): num_of_msgs = 100 topic_name = "public/default/my-python-topic-test_reader_on_partitioned_topic" url1 = self.adminUrl + "/admin/v2/persistent/" + topic_name + "/partitions" doHttpPut(url1, "4") client = Client(self.serviceUrl) producer = client.create_producer(topic_name) send_array = [] for i in range(num_of_msgs): data = b"hello-%d" % i producer.send(data) send_array.append(data) reader = client.create_reader(topic_name, MessageId.earliest) read_array = [] for i in range(num_of_msgs): msg = reader.read_next(TM) self.assertTrue(msg) read_array.append(msg.data()) self.assertListEqual(sorted(send_array), sorted(read_array)) reader.close() client.close() def test_reader_is_connected(self): client = Client(self.serviceUrl) topic = "test_reader_is_connected" reader = client.create_reader(topic, MessageId.earliest) self.assertTrue(reader.is_connected()) reader.close() self.assertFalse(reader.is_connected()) client.close() def test_reader_seek_for_message_id(self): client = pulsar.Client(self.serviceUrl) topic = "test-seek-for-message-id-" + str(int(time.time())) producer = client.create_producer(topic) readerExclusive = client.create_reader(topic, MessageId.latest) readerInclusive = client.create_reader(topic, MessageId.latest, start_message_id_inclusive=True) numMessages = 100 seekMessageId = None r = random.randint(0, numMessages - 2) for i in range(numMessages): msg_content = b"msg-%d" % i id = producer.send(msg_content) if i == r: seekMessageId = id readerExclusive.seek(seekMessageId) msg0 = readerExclusive.read_next(timeout_millis=3000) readerInclusive.seek(seekMessageId) msg1 = readerInclusive.read_next(timeout_millis=3000) self.assertEqual(msg0.data(), b"msg-%d" % (r + 1)) self.assertEqual(msg1.data(), b"msg-%d" % r) readerExclusive.close() readerInclusive.close() producer.close() client.close() def test_producer_sequence_after_reconnection(self): # Enable deduplication on namespace doHttpPost(self.adminUrl + "/admin/v2/namespaces/public/default/deduplication", "true") client = Client(self.serviceUrl) topic = "my-python-test-producer-sequence-after-reconnection-" + str(time.time()) producer = client.create_producer(topic, producer_name="my-producer-name") self.assertEqual(producer.last_sequence_id(), -1) for i in range(10): producer.send(b"hello-%d" % i) self.assertEqual(producer.last_sequence_id(), i) producer.close() producer = client.create_producer(topic, producer_name="my-producer-name") self.assertEqual(producer.last_sequence_id(), 9) for i in range(10, 20): producer.send(b"hello-%d" % i) self.assertEqual(producer.last_sequence_id(), i) client.close() doHttpPost(self.adminUrl + "/admin/v2/namespaces/public/default/deduplication", "false") def test_producer_deduplication(self): # Enable deduplication on namespace doHttpPost(self.adminUrl + "/admin/v2/namespaces/public/default/deduplication", "true") client = Client(self.serviceUrl) topic = "my-python-test-producer-deduplication-" + str(time.time()) producer = client.create_producer(topic, producer_name="my-producer-name") self.assertEqual(producer.last_sequence_id(), -1) consumer = client.subscribe(topic, "my-sub") producer.send(b"hello-0", sequence_id=0) producer.send(b"hello-1", sequence_id=1) producer.send(b"hello-2", sequence_id=2) self.assertEqual(producer.last_sequence_id(), 2) # Repeat the messages and verify they're not received by consumer producer.send(b"hello-1", sequence_id=1) producer.send(b"hello-2", sequence_id=2) self.assertEqual(producer.last_sequence_id(), 2) for i in range(3): msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-%d" % i) consumer.acknowledge(msg) with self.assertRaises(pulsar.Timeout): consumer.receive(100) producer.close() producer = client.create_producer(topic, producer_name="my-producer-name") self.assertEqual(producer.last_sequence_id(), 2) # Repeat the messages and verify they're not received by consumer producer.send(b"hello-1", sequence_id=1) producer.send(b"hello-2", sequence_id=2) self.assertEqual(producer.last_sequence_id(), 2) with self.assertRaises(pulsar.Timeout): consumer.receive(100) client.close() doHttpPost(self.adminUrl + "/admin/v2/namespaces/public/default/deduplication", "false") def test_producer_routing_mode(self): client = Client(self.serviceUrl) producer = client.create_producer( "my-python-test-producer", message_routing_mode=PartitionsRoutingMode.UseSinglePartition ) producer.send(b"test") client.close() def test_message_argument_errors(self): client = Client(self.serviceUrl) topic = "my-python-test-producer" producer = client.create_producer(topic) content = "test".encode("utf-8") self._check_type_error(lambda: producer.send(5)) self._check_value_error(lambda: producer.send(content, properties="test")) self._check_value_error(lambda: producer.send(content, partition_key=5)) self._check_value_error(lambda: producer.send(content, sequence_id="test")) self._check_value_error(lambda: producer.send(content, replication_clusters=5)) self._check_value_error(lambda: producer.send(content, disable_replication="test")) self._check_value_error(lambda: producer.send(content, event_timestamp="test")) self._check_value_error(lambda: producer.send(content, deliver_at="test")) self._check_value_error(lambda: producer.send(content, deliver_after="test")) client.close() def test_client_argument_errors(self): self._check_value_error(lambda: Client(None)) self._check_value_error(lambda: Client(self.serviceUrl, authentication="test")) self._check_value_error(lambda: Client(self.serviceUrl, operation_timeout_seconds="test")) self._check_value_error(lambda: Client(self.serviceUrl, stats_interval_in_seconds="test")) self._check_value_error(lambda: Client(self.serviceUrl, io_threads="test")) self._check_value_error(lambda: Client(self.serviceUrl, message_listener_threads="test")) self._check_value_error(lambda: Client(self.serviceUrl, concurrent_lookup_requests="test")) self._check_value_error(lambda: Client(self.serviceUrl, use_tls="test")) self._check_value_error(lambda: Client(self.serviceUrl, tls_trust_certs_file_path=5)) self._check_value_error(lambda: Client(self.serviceUrl, tls_allow_insecure_connection="test")) def test_service_info_argument_errors(self): self._check_value_error(lambda: ServiceInfo(None)) self._check_value_error(lambda: ServiceInfo(self.serviceUrl, authentication="test")) self._check_value_error(lambda: ServiceInfo(self.serviceUrl, tls_trust_certs_file_path=5)) def test_auto_cluster_failover_argument_errors(self): primary = ServiceInfo(self.serviceUrl) secondary = [ServiceInfo("pulsar://192.0.2.1:6650")] self._check_value_error(lambda: AutoClusterFailover("test", secondary)) self._check_value_error(lambda: AutoClusterFailover(primary, "test")) self._check_value_error(lambda: AutoClusterFailover(primary, [])) self._check_value_error(lambda: AutoClusterFailover(primary, ["test"])) self._check_value_error(lambda: AutoClusterFailover(primary, secondary, check_interval_ms=0)) self._check_value_error(lambda: AutoClusterFailover(primary, secondary, failover_threshold=0)) self._check_value_error(lambda: AutoClusterFailover(primary, secondary, switch_back_threshold=0)) self._check_value_error(lambda: Client(AutoClusterFailover(primary, secondary), authentication=AuthenticationToken("token"))) self._check_value_error(lambda: Client(AutoClusterFailover(primary, secondary), tls_trust_certs_file_path=CERTS_DIR + "cacert.pem")) self._check_value_error(lambda: Client(self.StaticServiceInfoProvider(primary), authentication=AuthenticationToken("token"))) self._check_value_error(lambda: Client(self.StaticServiceInfoProvider(primary), tls_trust_certs_file_path=CERTS_DIR + "cacert.pem")) self._check_value_error(lambda: Client(self.InvalidServiceInfoProvider())) def test_auto_cluster_failover_client(self): primary = ServiceInfo(self.serviceUrl) secondary = [ServiceInfo("pulsar://192.0.2.1:6650")] client = Client(AutoClusterFailover(primary, secondary, check_interval_ms=100)) self.assertEqual(client.get_service_info().service_url, self.serviceUrl) client.close() def test_service_info_provider_client(self): primary = ServiceInfo(self.serviceUrl) secondary = ServiceInfo("pulsar://192.0.2.1:6650") provider = self.StaticServiceInfoProvider(primary) client = Client(provider) self.assertEqual(client.get_service_info().service_url, self.serviceUrl) provider.callback(secondary) self.assertEqual(client.get_service_info().service_url, secondary.service_url) client.close() self.assertTrue(provider.closed) def test_producer_argument_errors(self): client = Client(self.serviceUrl) self._check_value_error(lambda: client.create_producer(None)) topic = "my-python-test-producer" self._check_value_error(lambda: client.create_producer(topic, producer_name=5)) self._check_value_error(lambda: client.create_producer(topic, initial_sequence_id="test")) self._check_value_error(lambda: client.create_producer(topic, send_timeout_millis="test")) self._check_value_error(lambda: client.create_producer(topic, compression_type=None)) self._check_value_error(lambda: client.create_producer(topic, max_pending_messages="test")) self._check_value_error(lambda: client.create_producer(topic, block_if_queue_full="test")) self._check_value_error(lambda: client.create_producer(topic, batching_enabled="test")) self._check_value_error(lambda: client.create_producer(topic, batching_enabled="test")) self._check_value_error(lambda: client.create_producer(topic, batching_max_allowed_size_in_bytes="test")) self._check_value_error(lambda: client.create_producer(topic, batching_max_publish_delay_ms="test")) client.close() def test_consumer_argument_errors(self): client = Client(self.serviceUrl) topic = "my-python-test-producer" sub_name = "my-sub-name" self._check_value_error(lambda: client.subscribe(None, sub_name)) self._check_value_error(lambda: client.subscribe(topic, None)) self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_type=None)) self._check_value_error(lambda: client.subscribe(topic, sub_name, receiver_queue_size="test")) self._check_value_error(lambda: client.subscribe(topic, sub_name, consumer_name=5)) self._check_value_error(lambda: client.subscribe(topic, sub_name, unacked_messages_timeout_ms="test")) self._check_value_error(lambda: client.subscribe(topic, sub_name, broker_consumer_stats_cache_time_ms="test")) client.close() def test_reader_argument_errors(self): client = Client(self.serviceUrl) topic = "my-python-test-producer" # This should not raise exception client.create_reader(topic, MessageId.earliest) self._check_value_error(lambda: client.create_reader(None, MessageId.earliest)) self._check_value_error(lambda: client.create_reader(topic, None)) self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, receiver_queue_size="test")) self._check_value_error(lambda: client.create_reader(topic, MessageId.earliest, reader_name=5)) client.close() def test_get_last_message_id(self): client = Client(self.serviceUrl) consumer = client.subscribe( "persistent://public/default/topic_name_test", "topic_name_test_sub", consumer_type=ConsumerType.Shared ) producer = client.create_producer("persistent://public/default/topic_name_test") msg_id = producer.send(b"hello") msg = consumer.receive(TM) self.assertEqual(msg.message_id(), msg_id) client.close() def test_publish_compact_and_consume(self): client = Client(self.serviceUrl) topic = "compaction_%s" % (uuid.uuid4()) producer = client.create_producer(topic, producer_name="my-producer-name", batching_enabled=False) self.assertEqual(producer.last_sequence_id(), -1) consumer = client.subscribe(topic, "my-sub1", is_read_compacted=True) consumer.close() consumer2 = client.subscribe(topic, "my-sub2", is_read_compacted=False) # producer create 2 messages with same key. producer.send(b"hello-0", partition_key="key0") producer.send(b"hello-1", partition_key="key0") producer.close() # issue compact command, and wait success url = "%s/admin/v2/persistent/public/default/%s/compaction" % (self.adminUrl, topic) doHttpPut(url, "") while True: s = doHttpGet(url).decode("utf-8") if "RUNNING" in s: print(s) print("Compact still running") time.sleep(0.2) else: print(s) print("Compact Complete now") self.assertTrue("SUCCESS" in s) break # after compaction completes the compacted ledger is recorded # as a property of a cursor. As persisting the cursor is async # and we don't wait for the acknowledgement of the acknowledgement, # there may be a race if we try to read the compacted ledger immediately. # therefore wait a second to allow the compacted ledger to be updated on # the broker. time.sleep(1.0) # after compact, consumer with `is_read_compacted=True`, expected read only the second message for same key. consumer1 = client.subscribe(topic, "my-sub1", is_read_compacted=True) msg0 = consumer1.receive(TM) self.assertEqual(msg0.data(), b"hello-1") consumer1.acknowledge(msg0) consumer1.close() # ditto for reader reader1 = client.create_reader(topic, MessageId.earliest, is_read_compacted=True) msg0 = reader1.read_next(TM) self.assertEqual(msg0.data(), b"hello-1") reader1.close() # after compact, consumer with `is_read_compacted=False`, expected read 2 messages for same key. msg0 = consumer2.receive(TM) self.assertEqual(msg0.data(), b"hello-0") consumer2.acknowledge(msg0) msg1 = consumer2.receive(TM) self.assertEqual(msg1.data(), b"hello-1") consumer2.acknowledge(msg1) consumer2.close() # ditto for reader reader2 = client.create_reader(topic, MessageId.earliest, is_read_compacted=False) msg0 = reader2.read_next(TM) self.assertEqual(msg0.data(), b"hello-0") msg1 = reader2.read_next(TM) self.assertEqual(msg1.data(), b"hello-1") reader2.close() client.close() def test_reader_has_message_available(self): # create client, producer, reader client = Client(self.serviceUrl) producer = client.create_producer("my-python-topic-reader-has-message-available") reader = client.create_reader("my-python-topic-reader-has-message-available", MessageId.latest) # before produce data, expected not has message available self.assertFalse(reader.has_message_available()) for i in range(10): producer.send(b"hello-%d" % i) # produced data, expected has message available self.assertTrue(reader.has_message_available()) for i in range(10): msg = reader.read_next(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello-%d" % i) # consumed all data, expected not has message available self.assertFalse(reader.has_message_available()) for i in range(10, 20): producer.send(b"hello-%d" % i) # produced data again, expected has message available self.assertTrue(reader.has_message_available()) reader.close() producer.close() client.close() def test_seek(self): client = Client(self.serviceUrl) topic = "my-python-topic-seek-" + str(time.time()) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) for i in range(100): if i > 0: time.sleep(0.02) producer.send(b"hello-%d" % i) ids = [] timestamps = [] for i in range(100): msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-%d" % i) ids.append(msg.message_id()) timestamps.append(msg.publish_timestamp()) consumer.acknowledge(msg) # seek, and after reconnect, expected receive first message. consumer.seek(MessageId.earliest) msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-0") # seek with wrong type with self.assertRaises(ValueError, msg="invalid seek_arg type "): consumer.seek(1.0) # seek on messageId consumer.seek(ids[50]) msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-51") # seek on a user provided MessageId msg_id = MessageId(ledger_id=ids[60].ledger_id(), entry_id=ids[60].entry_id()) consumer.seek(msg_id) msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-61") # ditto, but seek on timestamp consumer.seek(timestamps[42]) msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-42") # repeat with reader reader = client.create_reader(topic, MessageId.latest) with self.assertRaises(pulsar.Timeout): reader.read_next(100) # seek with wrong type with self.assertRaises(ValueError, msg="invalid seek_arg type "): consumer.seek(1.0) # earliest reader.seek(MessageId.earliest) msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-0") msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-1") # seek on messageId reader.seek(ids[33]) msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-34") msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-35") # seek on a user provided MessageId msg_id = MessageId(ledger_id=ids[44].ledger_id(), entry_id=ids[44].entry_id()) reader.seek(msg_id) msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-45") # seek on timestamp reader.seek(timestamps[79]) msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-79") msg = reader.read_next(TM) self.assertEqual(msg.data(), b"hello-80") reader.close() client.close() def test_seek_inclusive(self): client = Client(self.serviceUrl) topic = "my-python-topic-seek-inclusive-" + str(time.time()) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared, start_message_id_inclusive=True) producer = client.create_producer(topic) for i in range(100): if i > 0: time.sleep(0.02) producer.send(b"hello-%d" % i) ids = [] for i in range(100): msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-%d" % i) ids.append(msg.message_id()) consumer.acknowledge(msg) # seek, and after reconnect, expected receive first message. consumer.seek(MessageId.earliest) time.sleep(0.5) msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-0") # seek on messageId consumer.seek(ids[50]) time.sleep(0.5) msg = consumer.receive(TM) self.assertEqual(msg.data(), b"hello-50") client.close() def test_v2_topics(self): self._v2_topics(self.serviceUrl) def test_v2_topics_http(self): self._v2_topics(self.adminUrl) def _v2_topics(self, url): client = Client(url) consumer = client.subscribe("my-v2-topic-producer-consumer", "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer("my-v2-topic-producer-consumer") producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") consumer.acknowledge(msg) with self.assertRaises(pulsar.Timeout): consumer.receive(100) client.close() def test_topics_consumer(self): client = Client(self.serviceUrl) topic1 = "persistent://public/default/my-python-topics-consumer-1" topic2 = "persistent://public/default/my-python-topics-consumer-2" topic3 = "persistent://public/default-2/my-python-topics-consumer-3" # topic from different namespace topics = [topic1, topic2, topic3] url1 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-topics-consumer-1/partitions" url2 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-topics-consumer-2/partitions" url3 = self.adminUrl + "/admin/v2/persistent/public/default-2/my-python-topics-consumer-3/partitions" doHttpPut(url1, "2") doHttpPut(url2, "3") doHttpPut(url3, "4") producer1 = client.create_producer(topic1) producer2 = client.create_producer(topic2) producer3 = client.create_producer(topic3) consumer = client.subscribe( topics, "my-topics-consumer-sub", consumer_type=ConsumerType.Shared, receiver_queue_size=10 ) for i in range(100): producer1.send(b"hello-1-%d" % i) for i in range(100): producer2.send(b"hello-2-%d" % i) for i in range(100): producer3.send(b"hello-3-%d" % i) for i in range(300): msg = consumer.receive(TM) consumer.acknowledge(msg) with self.assertRaises(pulsar.Timeout): consumer.receive(100) client.close() def test_topics_pattern_consumer(self): client = Client(self.serviceUrl) topics_pattern = "persistent://public/default/my-python-pattern-consumer.*" topic1 = "persistent://public/default/my-python-pattern-consumer-1" topic2 = "persistent://public/default/my-python-pattern-consumer-2" topic3 = "persistent://public/default/my-python-pattern-consumer-3" url1 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-pattern-consumer-1/partitions" url2 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-pattern-consumer-2/partitions" url3 = self.adminUrl + "/admin/v2/persistent/public/default/my-python-pattern-consumer-3/partitions" doHttpPut(url1, "2") doHttpPut(url2, "3") doHttpPut(url3, "4") producer1 = client.create_producer(topic1) producer2 = client.create_producer(topic2) producer3 = client.create_producer(topic3) consumer = client.subscribe( re.compile(topics_pattern), "my-pattern-consumer-sub", consumer_type=ConsumerType.Shared, receiver_queue_size=10, pattern_auto_discovery_period=1, ) # wait enough time to trigger auto discovery time.sleep(2) for i in range(100): producer1.send(b"hello-1-%d" % i) for i in range(100): producer2.send(b"hello-2-%d" % i) for i in range(100): producer3.send(b"hello-3-%d" % i) for i in range(300): msg = consumer.receive(TM) consumer.acknowledge(msg) with self.assertRaises(pulsar.Timeout): consumer.receive(100) client.close() def test_batch_receive(self): client = Client(self.serviceUrl) topic = "my-python-topic-batch-receive-" + str(time.time()) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared, start_message_id_inclusive=True, batch_receive_policy=ConsumerBatchReceivePolicy(10, -1, -1)) producer = client.create_producer(topic) for i in range(10): if i > 0: time.sleep(0.02) producer.send(b"hello-%d" % i) msgs = consumer.batch_receive() i = 0 for msg in msgs: self.assertEqual(msg.data(), b"hello-%d" % i) i += 1 client.close() def test_message_id(self): s = MessageId.earliest.serialize() self.assertEqual(MessageId.deserialize(s), MessageId.earliest) s = MessageId.latest.serialize() self.assertEqual(MessageId.deserialize(s), MessageId.latest) client = Client(self.serviceUrl) topic = f'test-message-id-compare-{str(time.time())}' producer = client.create_producer(topic) consumer = client.subscribe(topic, 'sub') sent_ids = [] received_ids = [] for i in range(5): sent_ids.append(MessageId.wrap(producer.send(b'msg-%d' % i))) msg = consumer.receive(TM) received_ids.append(MessageId.wrap(msg.message_id())) self.assertEqual(sent_ids[i], received_ids[i]) consumer.acknowledge(received_ids[i]) consumer.acknowledge_cumulative(received_ids[4]) for i in range(4): self.assertLess(sent_ids[i], sent_ids[i + 1]) self.assertLessEqual(sent_ids[i], sent_ids[i + 1]) self.assertGreater(sent_ids[i + 1], sent_ids[i]) self.assertGreaterEqual(sent_ids[i + 1], sent_ids[i]) self.assertNotEqual(sent_ids[i], sent_ids[i + 1]) client.close() def test_get_topics_partitions(self): client = Client(self.serviceUrl) topic_partitioned = "persistent://public/default/test_get_topics_partitions" topic_non_partitioned = "persistent://public/default/test_get_topics_not-partitioned" url1 = self.adminUrl + "/admin/v2/persistent/public/default/test_get_topics_partitions/partitions" doHttpPut(url1, "3") self.assertEqual( client.get_topic_partitions(topic_partitioned), [ "persistent://public/default/test_get_topics_partitions-partition-0", "persistent://public/default/test_get_topics_partitions-partition-1", "persistent://public/default/test_get_topics_partitions-partition-2", ], ) self.assertEqual(client.get_topic_partitions(topic_non_partitioned), [topic_non_partitioned]) client.close() def test_token_auth(self): with open(TOKEN_PATH) as tf: token = tf.read().strip() # Use adminUrl to test both HTTP request and binary protocol client = Client(self.adminUrl, authentication=AuthenticationToken(token)) consumer = client.subscribe( "persistent://private/auth/my-python-topic-token-auth", "my-sub", consumer_type=ConsumerType.Shared ) producer = client.create_producer("persistent://private/auth/my-python-topic-token-auth") producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") client.close() def test_token_auth_supplier(self): def read_token(): with open(TOKEN_PATH) as tf: return tf.read().strip() client = Client(self.serviceUrl, authentication=AuthenticationToken(read_token)) consumer = client.subscribe( "persistent://private/auth/my-python-topic-token-auth", "my-sub", consumer_type=ConsumerType.Shared ) producer = client.create_producer("persistent://private/auth/my-python-topic-token-auth") producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") client.close() def test_producer_consumer_zstd(self): client = Client(self.serviceUrl) consumer = client.subscribe( "my-python-topic-producer-consumer-zstd", "my-sub", consumer_type=ConsumerType.Shared ) producer = client.create_producer( "my-python-topic-producer-consumer-zstd", compression_type=CompressionType.ZSTD ) producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") with self.assertRaises(pulsar.Timeout): consumer.receive(100) consumer.unsubscribe() client.close() def test_client_reference_deleted(self): def get_producer(): cl = Client(self.serviceUrl) return cl.create_producer(topic="foobar") producer = get_producer() producer.send(b"test_payload") ##### def test_get_topic_name(self): client = Client(self.serviceUrl) consumer = client.subscribe( "persistent://public/default/topic_name_test", "topic_name_test_sub", consumer_type=ConsumerType.Shared ) producer = client.create_producer("persistent://public/default/topic_name_test") producer.send(b"hello") msg = consumer.receive(TM) self.assertEqual(msg.topic_name(), "persistent://public/default/topic_name_test") client.close() def test_get_partitioned_topic_name(self): client = Client(self.serviceUrl) url1 = self.adminUrl + "/admin/v2/persistent/public/default/partitioned_topic_name_test/partitions" doHttpPut(url1, "3") partitions = [ "persistent://public/default/partitioned_topic_name_test-partition-0", "persistent://public/default/partitioned_topic_name_test-partition-1", "persistent://public/default/partitioned_topic_name_test-partition-2", ] self.assertEqual( client.get_topic_partitions("persistent://public/default/partitioned_topic_name_test"), partitions ) consumer = client.subscribe( "persistent://public/default/partitioned_topic_name_test", "partitioned_topic_name_test_sub", consumer_type=ConsumerType.Shared, ) producer = client.create_producer("persistent://public/default/partitioned_topic_name_test") producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg.topic_name() in partitions) client.close() def test_shutdown_client(self): client = Client(self.serviceUrl) producer = client.create_producer("persistent://public/default/partitioned_topic_name_test") producer.send(b"hello") client.shutdown() try: producer.send(b"hello") self.assertTrue(False) except pulsar.PulsarException: # Expected pass def test_listener_name_client(self): client = Client(self.serviceUrl, listener_name='test') try: producer = client.create_producer("persistent://public/default/partitioned_topic_name_test") self.fail() except pulsar.PulsarException: # Expected pass client.close() def test_negative_acks(self): client = Client(self.serviceUrl) consumer = client.subscribe( "test_negative_acks", "test", schema=pulsar.schema.StringSchema(), negative_ack_redelivery_delay_ms=1000 ) producer = client.create_producer("test_negative_acks", schema=pulsar.schema.StringSchema()) for i in range(10): producer.send_async("hello-%d" % i, callback=None) producer.flush() for i in range(10): msg = consumer.receive() self.assertEqual(msg.value(), "hello-%d" % i) consumer.negative_acknowledge(msg) for i in range(10): msg = consumer.receive() self.assertEqual(msg.value(), "hello-%d" % i) consumer.acknowledge(msg) with self.assertRaises(pulsar.Timeout): consumer.receive(100) client.close() def test_connect_timeout(self): client = pulsar.Client( service_url="pulsar://192.0.2.1:1234", connection_timeout_ms=1000, # 1 second ) t1 = time.time() try: producer = client.create_producer("test_connect_timeout") self.fail("create_producer should not succeed") except pulsar.ConnectError as expected: print("expected error: {} when create producer".format(expected)) t2 = time.time() self.assertGreater(t2 - t1, 1.0) self.assertLess(t2 - t1, 1.5) # 1.5 seconds is long enough client.close() def test_json_schema_encode(self): schema = JsonSchema(TestRecord) record = TestRecord(a=1, b=2) # Ensure that encoding a JsonSchema more than once works and produces the same result first_encode = schema.encode(record) second_encode = schema.encode(record) self.assertEqual(first_encode, second_encode) def test_chunking(self): client = Client(self.serviceUrl) data_size = 10 * 1024 * 1024 producer = client.create_producer( 'test_chunking', chunking_enabled=True ) consumer = client.subscribe('test_chunking', "my-subscription", max_pending_chunked_message=10, auto_ack_oldest_chunked_message_on_queue_full=False ) producer.send(bytes(bytearray(os.urandom(data_size))), None) msg = consumer.receive(TM) self.assertEqual(len(msg.data()), data_size) def test_invalid_chunking_config(self): client = Client(self.serviceUrl) self._check_value_error(lambda: client.create_producer( 'test_invalid_chunking_config', chunking_enabled=True, batching_enabled=True )) def _check_value_error(self, fun): with self.assertRaises(ValueError): fun() def _check_type_error(self, fun): with self.assertRaises(TypeError): fun() def _test_basic_auth(self, topic_id: int, auth, use_tls: bool = False, tls_private_key_file_path: Optional[str] = None, tls_certificate_file_path: Optional[str] = None) -> None: if use_tls: service_url = self.serviceUrlTls tls_trust_certs_file_path = CERTS_DIR + 'cacert.pem' else: service_url = self.adminUrl tls_trust_certs_file_path = None client = Client(service_url, authentication=auth, tls_trust_certs_file_path=tls_trust_certs_file_path, tls_private_key_file_path=tls_private_key_file_path, tls_certificate_file_path=tls_certificate_file_path) topic = "persistent://private/auth/my-python-topic-basic-auth-" + str(topic_id) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) producer = client.create_producer(topic) producer.send(b"hello") msg = consumer.receive(TM) self.assertTrue(msg) self.assertEqual(msg.data(), b"hello") client.close() def test_basic_auth(self): username = "admin" password = "123456" self._test_basic_auth(0, AuthenticationBasic(username, password)) self._test_basic_auth(1, AuthenticationBasic( auth_params_string='{{"username": "{}","password": "{}"}}'.format(username, password) )) def test_basic_auth_method(self): username = "admin" password = "123456" self._test_basic_auth(2, AuthenticationBasic(username, password, 'basic')) with self.assertRaises(pulsar.AuthorizationError): self._test_basic_auth(3, AuthenticationBasic(username, password, 'unknown')) self._test_basic_auth(4, AuthenticationBasic( auth_params_string='{{"username": "{}","password": "{}", "method": "basic"}}'.format(username, password) )) with self.assertRaises(pulsar.AuthorizationError): self._test_basic_auth(5, AuthenticationBasic( auth_params_string='{{"username": "{}","password": "{}", "method": "unknown"}}'.format(username, password) )) def test_tls_encryption_with_other_auth(self): self._test_basic_auth(6, AuthenticationBasic('admin', '123456'), use_tls=True, tls_private_key_file_path=CERTS_DIR + 'client-key.pem', tls_certificate_file_path=CERTS_DIR + 'client-cert.pem') with self.assertRaises(pulsar.ConnectError): self._test_basic_auth(7, AuthenticationBasic('admin', '123456'), use_tls=True, tls_private_key_file_path=CERTS_DIR + 'client-cert.pem', tls_certificate_file_path=CERTS_DIR + 'client-key.pem') def test_invalid_basic_auth(self): username = "invalid" password = "123456" client = Client(self.adminUrl, authentication=AuthenticationBasic(username, password)) topic = "persistent://private/auth/my-python-topic-invalid-basic-auth" with self.assertRaises(pulsar.ConnectError): client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) client = Client(self.adminUrl, authentication=AuthenticationBasic( auth_params_string='{{"username": "{}","password": "{}"}}'.format(username, password) )) with self.assertRaises(pulsar.ConnectError): client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared) with self.assertRaises(RuntimeError): AuthenticationBasic(auth_params_string='invalid auth params') def test_send_async_no_deadlock(self): client = Client(self.serviceUrl) producer = client.create_producer('test_send_async_no_deadlock') def send_callback(res, msg): print(f"Message '{msg}' published res={res}") for i in range(30): producer.send_async(f"Hello-{i}".encode('utf-8'), callback=send_callback) producer.flush() client.close() def test_keyshare_policy(self): with self.assertRaises(ValueError): # Raise error because sticky ranges are not provided. pulsar.ConsumerKeySharedPolicy( key_shared_mode=pulsar.KeySharedMode.Sticky, allow_out_of_order_delivery=False, ) expected_key_shared_mode = pulsar.KeySharedMode.Sticky expected_allow_out_of_order_delivery = True expected_sticky_ranges = [(0, 100), (101,200)] consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( key_shared_mode=expected_key_shared_mode, allow_out_of_order_delivery=expected_allow_out_of_order_delivery, sticky_ranges=expected_sticky_ranges ) self.assertEqual(consumer_key_shared_policy.key_shared_mode, expected_key_shared_mode) self.assertEqual(consumer_key_shared_policy.allow_out_of_order_delivery, expected_allow_out_of_order_delivery) self.assertEqual(consumer_key_shared_policy.sticky_ranges, expected_sticky_ranges) def test_keyshared_invalid_sticky_ranges(self): client = Client(self.serviceUrl) topic = "my-python-topic-keyshare-invalid-" + str(time.time()) with self.assertRaises(ValueError): consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( key_shared_mode=pulsar.KeySharedMode.Sticky, allow_out_of_order_delivery=False, sticky_ranges=[(0,65536)] ) client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, start_message_id_inclusive=True, key_shared_policy=consumer_key_shared_policy) with self.assertRaises(ValueError): consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( key_shared_mode=pulsar.KeySharedMode.Sticky, allow_out_of_order_delivery=False, sticky_ranges=[(0, 100), (50, 150)] ) client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, start_message_id_inclusive=True, key_shared_policy=consumer_key_shared_policy) def test_keyshared_autosplit(self): client = Client(self.serviceUrl) topic = "my-python-topic-keyshare-autosplit-" + str(time.time()) consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( key_shared_mode=pulsar.KeySharedMode.AutoSplit, allow_out_of_order_delivery=True, ) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, consumer_name = 'con-1', start_message_id_inclusive=True, key_shared_policy=consumer_key_shared_policy) consumer2 = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, consumer_name = 'con-2', start_message_id_inclusive=True, key_shared_policy=consumer_key_shared_policy) producer = client.create_producer(topic) for i in range(10): if i > 0: time.sleep(0.02) producer.send(b"hello-%d" % i) msgs = [] while True: try: msg = consumer.receive(100) except pulsar.Timeout: break msgs.append(msg) consumer.acknowledge(msg) while True: try: msg = consumer2.receive(100) except pulsar.Timeout: break msgs.append(msg) consumer2.acknowledge(msg) self.assertEqual(len(msgs), 10) client.close() def test_sticky_autosplit(self): client = Client(self.serviceUrl) topic = "my-python-topic-keyshare-sticky-" + str(time.time()) consumer_key_shared_policy = pulsar.ConsumerKeySharedPolicy( key_shared_mode=pulsar.KeySharedMode.Sticky, allow_out_of_order_delivery=True, sticky_ranges=[(0,30000)], ) consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, consumer_name='con-1', start_message_id_inclusive=True, key_shared_policy=consumer_key_shared_policy) consumer2_key_shared_policy = pulsar.ConsumerKeySharedPolicy( key_shared_mode=pulsar.KeySharedMode.Sticky, allow_out_of_order_delivery=True, sticky_ranges=[(30001, 65535)], ) consumer2 = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.KeyShared, consumer_name='con-2', start_message_id_inclusive=True, key_shared_policy=consumer2_key_shared_policy) producer = client.create_producer(topic) for i in range(10): if i > 0: time.sleep(0.02) producer.send(b"hello-%d" % i) msgs = [] while True: try: msg = consumer.receive(100) except pulsar.Timeout: break msgs.append(msg) consumer.acknowledge(msg) while True: try: msg = consumer2.receive(100) except pulsar.Timeout: break msgs.append(msg) consumer2.acknowledge(msg) self.assertEqual(len(msgs), 10) client.close() def test_acknowledge_failed(self): client = Client(self.serviceUrl) topic = 'test_acknowledge_failed' producer = client.create_producer(topic) consumer1 = client.subscribe(topic, 'sub1', consumer_type=ConsumerType.Shared) consumer2 = client.subscribe(topic, 'sub2', consumer_type=ConsumerType.KeyShared) msg_id = producer.send('hello'.encode()) msg1 = consumer1.receive() with self.assertRaises(pulsar.CumulativeAcknowledgementNotAllowedError): consumer1.acknowledge_cumulative(msg1) with self.assertRaises(pulsar.CumulativeAcknowledgementNotAllowedError): consumer1.acknowledge_cumulative(msg1.message_id()) msg2 = consumer2.receive() with self.assertRaises(pulsar.CumulativeAcknowledgementNotAllowedError): consumer2.acknowledge_cumulative(msg2) with self.assertRaises(pulsar.CumulativeAcknowledgementNotAllowedError): consumer2.acknowledge_cumulative(msg2.message_id()) consumer = client.subscribe([topic, topic + '-another'], 'sub') # The message id does not have a topic name with self.assertRaises(pulsar.OperationNotSupported): consumer.acknowledge(msg_id) client.close() def test_batch_index_ack(self): topic_name = 'test-batch-index-ack-3' client = pulsar.Client('pulsar://localhost:6650') producer = client.create_producer(topic_name, batching_enabled=True, batching_max_messages=100, batching_max_publish_delay_ms=10000) consumer = client.subscribe(topic_name, subscription_name='test-batch-index-ack', batch_index_ack_enabled=True) # Make sure send 0~5 is a batch msg. for i in range(5): producer.send_async(b"hello-%d" % i, callback=None) producer.flush() # Receive msgs and just ack 0, 1 msgs results = [] for i in range(5): msg = consumer.receive() print("receive from {}".format(msg.message_id())) results.append(msg) assert len(results) == 5 for i in range(2): consumer.acknowledge(results[i]) time.sleep(0.2) # Restart consumer after, just receive 2~5 msg. consumer.close() consumer = client.subscribe(topic_name, subscription_name='test-batch-index-ack', batch_index_ack_enabled=True) results2 = [] for i in range(2, 5): msg = consumer.receive() results2.append(msg) assert len(results2) == 3 # assert no more msgs. with self.assertRaises(pulsar.Timeout): consumer.receive(timeout_millis=1000) client.close() def test_dead_letter_policy_config(self): with self.assertRaises(ValueError): ConsumerDeadLetterPolicy(-1) policy = ConsumerDeadLetterPolicy(10) self.assertEqual(10, policy.max_redeliver_count) self.assertEqual("", policy.dead_letter_topic) self.assertEqual("", policy.initial_subscription_name) def test_dead_letter_policy(self): client = Client(self.serviceUrl) topic = "my-python-topic-test-dlq" + str(time.time()) dlq_topic = 'dlq-' + topic max_redeliver_count = 5 consumer = client.subscribe(topic, "my-sub", consumer_type=ConsumerType.Shared, dead_letter_policy=ConsumerDeadLetterPolicy(max_redeliver_count, dlq_topic, 'init-sub')) dlq_consumer = client.subscribe(dlq_topic, "my-sub", consumer_type=ConsumerType.Shared) # Sen num msgs. producer = client.create_producer(topic) num = 10 for i in range(num): producer.send(b"hello-%d" % i) producer.flush() # Redelivery all messages maxRedeliverCountNum time. for i in range(1, num * max_redeliver_count + num + 1): msg = consumer.receive() if i % num == 0: consumer.redeliver_unacknowledged_messages() print(f"Start redeliver msgs '{i}'") with self.assertRaises(pulsar.Timeout): consumer.receive(100) for i in range(num): msg = dlq_consumer.receive() self.assertTrue(msg) self.assertEqual(msg.data(), b"hello-%d" % i) dlq_consumer.acknowledge(msg) with self.assertRaises(pulsar.Timeout): dlq_consumer.receive(100) client.close() def test_regex_subscription(self): client = Client(self.serviceUrl) topic1 = "persistent://public/default/test-regex-sub-1" topic2 = "persistent://public/default/test-regex-sub-2" topic3 = "non-persistent://public/default/test-regex-sub-3" topic4 = "persistent://public/default/no-match-test-regex-sub-3" # no match pattern rule topic. producer1 = client.create_producer(topic1) producer2 = client.create_producer(topic2) producer3 = client.create_producer(topic3) producer4 = client.create_producer(topic4) consumer_all = client.subscribe( re.compile('public/default/test-regex-sub-.*'), "regex-sub-all", consumer_type=ConsumerType.Shared, regex_subscription_mode=RegexSubscriptionMode.AllTopics ) consumer_persistent = client.subscribe( re.compile('public/default/test-regex-sub-.*'), "regex-sub-persistent", consumer_type=ConsumerType.Shared, regex_subscription_mode=RegexSubscriptionMode.PersistentOnly ) consumer_non_persistent = client.subscribe( re.compile('public/default/test-regex-sub-.*'), "regex-sub-non-persistent", consumer_type=ConsumerType.Shared, regex_subscription_mode=RegexSubscriptionMode.NonPersistentOnly ) num = 10 for i in range(num): producer1.send(b"hello-1-%d" % i) producer2.send(b"hello-2-%d" % i) producer3.send(b"hello-3-%d" % i) producer4.send(b"hello-4-%d" % i) # Assert consumer_all. received_topics = set() for i in range(3 * num): msg = consumer_all.receive(TM) topic_name = msg.topic_name() self.assertIn(topic_name, [topic1, topic2, topic3]) received_topics.add(topic_name) consumer_all.acknowledge(msg) self.assertEqual(received_topics, {topic1, topic2, topic3}) with self.assertRaises(pulsar.Timeout): consumer_all.receive(100) # Assert consumer_persistent. received_topics.clear() for i in range(2 * num): msg = consumer_persistent.receive(TM) topic_name = msg.topic_name() self.assertIn(topic_name, [topic1, topic2]) received_topics.add(topic_name) consumer_persistent.acknowledge(msg) self.assertEqual(received_topics, {topic1, topic2}) with self.assertRaises(pulsar.Timeout): consumer_persistent.receive(100) # Assert consumer_non_persistent. received_topics.clear() for i in range(num): msg = consumer_non_persistent.receive(TM) topic_name = msg.topic_name() self.assertIn(topic_name, [topic3]) received_topics.add(topic_name) consumer_non_persistent.acknowledge(msg) self.assertEqual(received_topics, {topic3}) with self.assertRaises(pulsar.Timeout): consumer_non_persistent.receive(100) client.close() def test_consumer_name(self): client = Client(self.serviceUrl) name = 'my-consumer-name' consumer = client.subscribe('test_consumer_name', 'sub', consumer_name=name) self.assertEqual(consumer.consumer_name(), name) client.close() def test_deserialize_msg_id_with_topic(self): client = Client(self.serviceUrl) topic1 = "deserialize-msg-id-with-topic1-" + str(time.time()) topic2 = "deserialize-msg-id-with-topic2-" + str(time.time()) consumer = client.subscribe([topic1, topic2], 'sub') producer1 = client.create_producer(topic1) producer2 = client.create_producer(topic2) producer1.send(b"msg-1") producer2.send(b"msg-2") serialized_msg_ids = dict() for _ in range(2): msg = consumer.receive(TM) serialized_msg_ids[msg.topic_name()] = msg.message_id().serialize() for topic, serialized_msg_id in serialized_msg_ids.items(): deserialized_msg_id = MessageId.deserialize(serialized_msg_id, topic=topic) consumer.acknowledge_cumulative(deserialized_msg_id) consumer.close() consumer = client.subscribe([topic1, topic2], 'sub') producer1.send(b'msg-3') msg = consumer.receive(TM) self.assertEqual(msg.value(), b'msg-3') client.close() def test_message_router(self): topic_name = "public/default/test_message_router" + str(time.time()) url1 = self.adminUrl + "/admin/v2/persistent/" + topic_name + "/partitions" doHttpPut(url1, "5") client = Client(self.serviceUrl) def router(msg: pulsar.Message, num_partitions: int): s = msg.value().decode('utf-8') if s.startswith("hello-"): return 10 % num_partitions else: return 11 % num_partitions producer = client.create_producer(topic_name, message_router=router) producer.send(b"hello-0") producer.send(b"hello-1") producer.send(b"world-0") producer.send(b"world-1") consumer = client.subscribe(topic_name, 'sub', initial_position=InitialPosition.Earliest) partition_to_values = dict() for _ in range(4): msg = consumer.receive(TM) partition = msg.message_id().partition() if partition in partition_to_values: partition_to_values[partition].append(msg.value().decode('utf-8')) else: partition_to_values[partition] = [msg.value().decode('utf-8')] self.assertEqual(partition_to_values[0], ["hello-0", "hello-1"]) self.assertEqual(partition_to_values[1], ["world-0", "world-1"]) client.close() if __name__ == "__main__": main() pulsar-client-python-3.12.0-candidate-0/tests/reader_test.py000066400000000000000000000045041520101105200237500ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from unittest import TestCase, main import time from pulsar import Client, MessageId class ReaderTest(TestCase): def setUp(self): self._client: Client = Client('pulsar://localhost:6650') def tearDown(self) -> None: self._client.close() def test_has_message_available_after_seek(self): topic = f'test_has_message_available_after_seek-{time.time()}' producer = self._client.create_producer(topic) reader = self._client.create_reader(topic, start_message_id=MessageId.earliest) producer.send('msg-0'.encode()) self.assertTrue(reader.has_message_available()) reader.seek(MessageId.latest) self.assertFalse(reader.has_message_available()) producer.send('msg-1'.encode()) self.assertTrue(reader.has_message_available()) def test_seek_latest_message_id(self): topic = f'test_seek_latest_message_id-{time.time()}' producer = self._client.create_producer(topic) msg_id = producer.send('msg'.encode()) reader = self._client.create_reader(topic, start_message_id=MessageId.latest) self.assertFalse(reader.has_message_available()) reader.close() reader = self._client.create_reader(topic, start_message_id=MessageId.latest, start_message_id_inclusive=True) self.assertTrue(reader.has_message_available()) msg = reader.read_next(3000) self.assertEqual(msg.message_id(), msg_id) reader.close() if __name__ == "__main__": main() pulsar-client-python-3.12.0-candidate-0/tests/run-unit-tests.sh000077500000000000000000000022171520101105200243540ustar00rootroot00000000000000#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # set -e -x ROOT_DIR=$(git rev-parse --show-toplevel) cd $ROOT_DIR/tests python3 -m pip install testcontainers python3 custom_logger_test.py python3 debug_logger_test.py python3 interrupted_test.py python3 auto_cluster_failover_test.py python3 pulsar_test.py python3 schema_test.py python3 table_view_test.py python3 reader_test.py python3 asyncio_test.py pulsar-client-python-3.12.0-candidate-0/tests/schema_test.py000077500000000000000000001476101520101105200237570ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # import base64 import math import requests from typing import List from unittest import TestCase, main import fastavro import pulsar from pulsar.schema import * from enum import Enum import json from fastavro.schema import load_schema from google.protobuf import descriptor_pb2, descriptor_pool, message_factory def _add_protobuf_field(message, name, number, field_type, type_name=None): field = message.field.add() field.name = name field.number = number field.label = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL field.type = field_type if type_name: field.type_name = type_name def _get_message_classes(pool, message_names): if hasattr(message_factory, 'GetMessageClass'): return tuple( message_factory.GetMessageClass(pool.FindMessageTypeByName(message_name)) for message_name in message_names ) factory = message_factory.MessageFactory(pool) return tuple( factory.GetPrototype(pool.FindMessageTypeByName(message_name)) for message_name in message_names ) def _build_protobuf_test_messages(): file_proto = descriptor_pb2.FileDescriptorProto() file_proto.name = 'test_schema.proto' file_proto.package = 'test' file_proto.syntax = 'proto3' test_message = file_proto.message_type.add() test_message.name = 'TestMessage' _add_protobuf_field(test_message, 'name', 1, descriptor_pb2.FieldDescriptorProto.TYPE_STRING) _add_protobuf_field(test_message, 'value', 2, descriptor_pb2.FieldDescriptorProto.TYPE_INT32) nested_message = file_proto.message_type.add() nested_message.name = 'TestMessageWithNested' _add_protobuf_field(nested_message, 'str_field', 1, descriptor_pb2.FieldDescriptorProto.TYPE_STRING) _add_protobuf_field(nested_message, 'int_field', 2, descriptor_pb2.FieldDescriptorProto.TYPE_INT32) _add_protobuf_field(nested_message, 'double_field', 3, descriptor_pb2.FieldDescriptorProto.TYPE_DOUBLE) _add_protobuf_field( nested_message, 'nested', 4, descriptor_pb2.FieldDescriptorProto.TYPE_MESSAGE, '.test.TestInner' ) inner_message = file_proto.message_type.add() inner_message.name = 'TestInner' _add_protobuf_field(inner_message, 'inner_str', 1, descriptor_pb2.FieldDescriptorProto.TYPE_STRING) _add_protobuf_field(inner_message, 'inner_int', 2, descriptor_pb2.FieldDescriptorProto.TYPE_INT64) pool = descriptor_pool.DescriptorPool() pool.AddSerializedFile(file_proto.SerializeToString()) return _get_message_classes( pool, ('test.TestMessage', 'test.TestMessageWithNested', 'test.TestInner'), ) TestMessage, TestMessageWithNested, TestInner = _build_protobuf_test_messages() class ExampleRecord(Record): str_field = String() int_field = Integer() float_field = Float() bytes_field = Bytes() class SchemaTest(TestCase): serviceUrl = 'pulsar://localhost:6650' def test_simple(self): class Color(Enum): red = 1 green = 2 blue = 3 class Example(Record): _sorted_fields = True a = String() b = Integer() c = Array(String()) d = Color e = Boolean() f = Float() g = Double() h = Bytes() i = Map(String()) j = CustomEnum(Color) fastavro.parse_schema(Example.schema()) self.assertEqual(Example.schema(), { "name": "Example", "type": "record", "fields": [ {"name": "a", "type": ["null", "string"]}, {"name": "b", "type": ["null", "int"]}, {"name": "c", "type": ["null", { "type": "array", "items": "string"}] }, {"name": "d", "type": ["null", { "type": "enum", "name": "Color", "symbols": ["red", "green", "blue"]}] }, {"name": "e", "type": ["null", "boolean"]}, {"name": "f", "type": ["null", "float"]}, {"name": "g", "type": ["null", "double"]}, {"name": "h", "type": ["null", "bytes"]}, {"name": "i", "type": ["null", { "type": "map", "values": "string"}] }, {"name": "j", "type": ["null", "Color"]} ] }) def test_type_promotion(self): test_cases = [ (20, int, 20), # No promotion necessary: int => int (20, float, 20.0), # Promotion: int => float (20.0, float, 20.0), # No Promotion necessary: float => float ("Test text1", bytes, b"Test text1"), # Promotion: str => bytes (b"Test text1", str, "Test text1"), # Promotion: bytes => str ] for value_from, type_to, value_to in test_cases: if type_to == int: fieldType = Integer() elif type_to == float: fieldType = Double() elif type_to == str: fieldType = String() elif type_to == bytes: fieldType = Bytes() else: fieldType = String() field_value = fieldType.validate_type("test_field", value_from) self.assertEqual(value_to, field_value) def test_complex(self): class Color(Enum): red = 1 green = 2 blue = 3 class MySubRecord(Record): _sorted_fields = True x = Integer() y = Long() z = String() color = CustomEnum(Color) class Example(Record): _sorted_fields = True a = String() sub = MySubRecord # Test with class sub2 = MySubRecord() # Test with instance fastavro.parse_schema(Example.schema()) self.assertEqual(Example.schema(), { "name": "Example", "type": "record", "fields": [ {"name": "a", "type": ["null", "string"]}, {"name": "sub", "type": ["null", { "name": "MySubRecord", "type": "record", "fields": [ {'name': 'color', 'type': ['null', {'type': 'enum', 'name': 'Color', 'symbols': ['red', 'green', 'blue']}]}, {"name": "x", "type": ["null", "int"]}, {"name": "y", "type": ["null", "long"]}, {"name": "z", "type": ["null", "string"]}] }] }, {"name": "sub2", "type": ["null", 'MySubRecord'] } ] }) def test_complex_with_required_fields(self): class MySubRecord(Record): x = Integer(required=True) y = Long(required=True) z = String() class Example(Record): a = String(required=True) sub = MySubRecord(required=True) self.assertEqual(Example.schema(), { "name": "Example", "type": "record", "fields": [ {"name": "a", "type": "string"}, {"name": "sub", "type": { "name": "MySubRecord", "type": "record", "fields": [{"name": "x", "type": "int"}, {"name": "y", "type": "long"}, {"name": "z", "type": ["null", "string"]}] } }, ] }) def test_invalid_enum(self): class Color: red = 1 green = 2 blue = 3 class InvalidEnum(Record): a = Integer() b = Color # Enum will be ignored self.assertEqual(InvalidEnum.schema(), {'name': 'InvalidEnum', 'type': 'record', 'fields': [{'name': 'a', 'type': ["null", 'int']}]}) def test_initialization(self): class Example(Record): a = Integer() b = Integer() r = Example(a=1, b=2) self.assertEqual(r.a, 1) self.assertEqual(r.b, 2) r.b = 5 self.assertEqual(r.b, 5) # Setting non-declared field should fail try: r.c = 3 self.fail('Should have failed') except AttributeError: # Expected pass try: Record(a=1, c=8) self.fail('Should have failed') except AttributeError: # Expected pass except TypeError: # Expected pass def _expectTypeError(self, func): try: func() self.fail('Should have failed') except TypeError: # Expected pass def test_field_type_check(self): class Example(Record): a = Integer() b = String(required=False) self._expectTypeError(lambda: Example(a=1, b=2)) class E2(Record): a = Boolean() E2(a=False) # ok self._expectTypeError(lambda: E2(a=1)) class E3(Record): a = Float() E3(a=1.0) # Ok E3(a=1) # Ok Type promotion: int -> float class E4(Record): a = Null() E4(a=None) # Ok self._expectTypeError(lambda: E4(a=1)) class E5(Record): a = Long() E5(a=1234) # Ok self._expectTypeError(lambda: E5(a=1.12)) class E6(Record): a = String() E6(a="hello") # Ok self._expectTypeError(lambda: E5(a=1.12)) class E6(Record): a = Bytes() E6(a="hello".encode('utf-8')) # Ok self._expectTypeError(lambda: E5(a=1.12)) class E7(Record): a = Double() E7(a=1.0) # Ok E7(a=1) # Ok Type promotion: int -> double class Color(Enum): red = 1 green = 2 blue = 3 class OtherEnum(Enum): red = 1 green = 2 blue = 3 class E8(Record): a = Color e = E8(a=Color.red) # Ok self.assertEqual(e.a, Color.red) e = E8(a='red') # Ok self.assertEqual(e.a, Color.red) e = E8(a=1) # Ok self.assertEqual(e.a, Color.red) self._expectTypeError(lambda: E8(a='redx')) self._expectTypeError(lambda: E8(a=OtherEnum.red)) self._expectTypeError(lambda: E8(a=5)) class E9(Record): a = Array(String()) E9(a=['a', 'b', 'c']) # Ok self._expectTypeError(lambda: E9(a=1)) self._expectTypeError(lambda: E9(a=[1, 2, 3])) self._expectTypeError(lambda: E9(a=['1', '2', 3])) class E10(Record): a = Map(Integer()) E10(a={'a': 1, 'b': 2}) # Ok self._expectTypeError(lambda: E10(a=1)) self._expectTypeError(lambda: E10(a={'a': '1', 'b': 2})) self._expectTypeError(lambda: E10(a={1: 1, 'b': 2})) class SubRecord1(Record): s = Integer() class SubRecord2(Record): s = String() class E11(Record): a = SubRecord1 E11(a=SubRecord1(s=1)) # Ok self._expectTypeError(lambda: E11(a=1)) self._expectTypeError(lambda: E11(a=SubRecord2(s='hello'))) def test_field_type_check_defaults(self): try: class Example(Record): a = Integer(default="xyz") self.fail("Class declaration should have failed") except TypeError: pass # Expected def test_serialize_json(self): class Example(Record): a = Integer() b = Integer() self.assertEqual(Example.schema(), { "name": "Example", "type": "record", "fields": [ {"name": "a", "type": ["null", "int"]}, {"name": "b", "type": ["null", "int"]}, ] }) s = JsonSchema(Example) r = Example(a=1, b=2) data = s.encode(r) self.assertEqual(json.loads(data), {'a': 1, 'b': 2}) r2 = s.decode(data) self.assertEqual(r2.__class__.__name__, 'Example') self.assertEqual(r2, r) def test_serialize_avro(self): class Example(Record): a = Integer() b = Integer() self.assertEqual(Example.schema(), { "name": "Example", "type": "record", "fields": [ {"name": "a", "type": ["null", "int"]}, {"name": "b", "type": ["null", "int"]}, ] }) s = AvroSchema(Example) r = Example(a=1, b=2) data = s.encode(r) r2 = s.decode(data) self.assertEqual(r2.__class__.__name__, 'Example') self.assertEqual(r2, r) def test_non_sorted_fields(self): class T1(Record): a = Integer() b = Integer() c = Double() d = String() class T2(Record): b = Integer() a = Integer() d = String() c = Double() self.assertNotEqual(T1.schema()['fields'], T2.schema()['fields']) def test_sorted_fields(self): class T1(Record): _sorted_fields = True a = Integer() b = Integer() class T2(Record): _sorted_fields = True b = Integer() a = Integer() self.assertEqual(T1.schema()['fields'], T2.schema()['fields']) def test_schema_version(self): class Example(Record): a = Integer() b = Integer() client = pulsar.Client(self.serviceUrl) producer = client.create_producer( 'my-avro-python-schema-version-topic', schema=AvroSchema(Example)) consumer = client.subscribe('my-avro-python-schema-version-topic', 'sub-1', schema=AvroSchema(Example)) r = Example(a=1, b=2) producer.send(r) msg = consumer.receive() self.assertIsNotNone(msg.schema_version()) self.assertEqual(b'\x00\x00\x00\x00\x00\x00\x00\x00', msg.schema_version().encode()) self.assertEqual(r, msg.value()) client.close() def test_serialize_wrong_types(self): class Example(Record): a = Integer() b = Integer() class Foo(Record): x = Integer() y = Integer() s = JsonSchema(Example) try: data = s.encode(Foo(x=1, y=2)) self.fail('Should have failed') except TypeError: pass # expected try: data = s.encode('hello') self.fail('Should have failed') except TypeError: pass # expected def test_defaults(self): class Example(Record): a = Integer(default=5) b = Integer() c = String(default='hello') r = Example() self.assertEqual(r.a, 5) self.assertEqual(r.b, None) self.assertEqual(r.c, 'hello') def test_none_value(self): """ The objective of the test is to check that if no value is assigned to the attribute, the validation is returning the expect default value as defined in the Field class """ class Example(Record): a = Null() b = Boolean() c = Integer() d = Long() e = Float() f = Double() g = Bytes() h = String() r = Example() self.assertIsNone(r.a) self.assertFalse(r.b) self.assertIsNone(r.c) self.assertIsNone(r.d) self.assertIsNone(r.e) self.assertIsNone(r.f) self.assertIsNone(r.g) self.assertIsNone(r.h) #### def test_json_schema(self): class Example(Record): a = Integer() b = Integer() # Incompatible variation of the class class BadExample(Record): a = String() b = Integer() client = pulsar.Client(self.serviceUrl) producer = client.create_producer( 'my-json-python-topic', schema=JsonSchema(Example)) # Validate that incompatible schema is rejected try: client.subscribe('my-json-python-topic', 'sub-1', schema=JsonSchema(BadExample)) self.fail('Should have failed') except Exception as e: pass # Expected try: client.subscribe('my-json-python-topic', 'sub-1', schema=StringSchema(BadExample)) self.fail('Should have failed') except Exception as e: pass # Expected try: client.subscribe('my-json-python-topic', 'sub-1', schema=AvroSchema(BadExample)) self.fail('Should have failed') except Exception as e: pass # Expected consumer = client.subscribe('my-json-python-topic', 'sub-1', schema=JsonSchema(Example)) r = Example(a=1, b=2) producer.send(r) msg = consumer.receive() self.assertEqual(r, msg.value()) producer.close() consumer.close() client.close() def test_string_schema(self): client = pulsar.Client(self.serviceUrl) producer = client.create_producer( 'my-string-python-topic', schema=StringSchema()) # Validate that incompatible schema is rejected try: class Example(Record): a = Integer() b = Integer() client.create_producer('my-string-python-topic', schema=JsonSchema(Example)) self.fail('Should have failed') except Exception as e: pass # Expected consumer = client.subscribe('my-string-python-topic', 'sub-1', schema=StringSchema()) producer.send("Hello") msg = consumer.receive() self.assertEqual("Hello", msg.value()) self.assertEqual(b"Hello", msg.data()) client.close() def test_bytes_schema(self): client = pulsar.Client(self.serviceUrl) producer = client.create_producer( 'my-bytes-python-topic', schema=BytesSchema()) # Validate that incompatible schema is rejected try: class Example(Record): a = Integer() b = Integer() client.create_producer('my-bytes-python-topic', schema=JsonSchema(Example)) self.fail('Should have failed') except Exception as e: pass # Expected consumer = client.subscribe('my-bytes-python-topic', 'sub-1', schema=BytesSchema()) producer.send(b"Hello") msg = consumer.receive() self.assertEqual(b"Hello", msg.value()) client.close() def test_avro_schema(self): class Example(Record): a = Integer() b = Integer() # Incompatible variation of the class class BadExample(Record): a = String() b = Integer() client = pulsar.Client(self.serviceUrl) producer = client.create_producer( 'my-avro-python-topic', schema=AvroSchema(Example)) # Validate that incompatible schema is rejected try: client.subscribe('my-avro-python-topic', 'sub-1', schema=AvroSchema(BadExample)) self.fail('Should have failed') except Exception as e: pass # Expected try: client.subscribe('my-avro-python-topic', 'sub-2', schema=JsonSchema(Example)) self.fail('Should have failed') except Exception as e: pass # Expected consumer = client.subscribe('my-avro-python-topic', 'sub-3', schema=AvroSchema(Example)) r = Example(a=1, b=2) producer.send(r) msg = consumer.receive() self.assertEqual(r, msg.value()) producer.close() consumer.close() client.close() def test_json_enum(self): class MyEnum(Enum): A = 1 B = 2 C = 3 class Example(Record): name = String() v = MyEnum w = CustomEnum(MyEnum) x = CustomEnum(MyEnum, required=True, default=MyEnum.A, required_default=True) topic = 'my-json-enum-topic' client = pulsar.Client(self.serviceUrl) producer = client.create_producer( topic=topic, schema=JsonSchema(Example)) consumer = client.subscribe(topic, 'test', schema=JsonSchema(Example)) r = Example(name='test', v=MyEnum.C, w=MyEnum.B) producer.send(r) msg = consumer.receive() self.assertEqual('test', msg.value().name) self.assertEqual(MyEnum.C, MyEnum(msg.value().v)) self.assertEqual(MyEnum.B, MyEnum(msg.value().w)) self.assertEqual(MyEnum.A, MyEnum(msg.value().x)) client.close() def test_avro_enum(self): class MyEnum(Enum): A = 1 B = 2 C = 3 class Example(Record): name = String() v = MyEnum w = CustomEnum(MyEnum) x = CustomEnum(MyEnum, required=True, default=MyEnum.B, required_default=True) topic = 'my-avro-enum-topic' client = pulsar.Client(self.serviceUrl) producer = client.create_producer( topic=topic, schema=AvroSchema(Example)) consumer = client.subscribe(topic, 'test', schema=AvroSchema(Example)) r = Example(name='test', v=MyEnum.C, w=MyEnum.A) producer.send(r) msg = consumer.receive() msg.value() self.assertEqual(MyEnum.C, msg.value().v) self.assertEqual(MyEnum.A, MyEnum(msg.value().w)) self.assertEqual(MyEnum.B, MyEnum(msg.value().x)) client.close() def test_avro_map_array(self): class MapArray(Record): values = Map(Array(Integer())) class MapMap(Record): values = Map(Map(Integer())) class ArrayMap(Record): values = Array(Map(Integer())) class ArrayArray(Record): values = Array(Array(Integer())) topic_prefix = "my-avro-map-array-topic-" data_list = ( (topic_prefix + "0", AvroSchema(MapArray), MapArray(values={"A": [1, 2], "B": [3]})), (topic_prefix + "1", AvroSchema(MapMap), MapMap(values={"A": {"B": 2},})), (topic_prefix + "2", AvroSchema(ArrayMap), ArrayMap(values=[{"A": 1}, {"B": 2}, {"C": 3}])), (topic_prefix + "3", AvroSchema(ArrayArray), ArrayArray(values=[[1, 2, 3], [4]])), ) client = pulsar.Client(self.serviceUrl) for data in data_list: topic = data[0] schema = data[1] record = data[2] producer = client.create_producer(topic, schema=schema) consumer = client.subscribe(topic, 'sub', schema=schema) producer.send(record) msg = consumer.receive() self.assertEqual(msg.value().values, record.values) consumer.acknowledge(msg) consumer.close() producer.close() client.close() def test_avro_required_default(self): class MySubRecord(Record): _sorted_fields = True x = Integer() y = Long() z = String() class Example(Record): a = Integer() b = Boolean(required=True) c = Long() d = Float() e = Double() f = String() g = Bytes() h = Array(String()) i = Map(String()) j = MySubRecord() class ExampleRequiredDefault(Record): _sorted_fields = True a = Integer(required_default=True) b = Boolean(required=True, required_default=True) c = Long(required_default=True) d = Float(required_default=True) e = Double(required_default=True) f = String(required_default=True) g = Bytes(required_default=True) h = Array(String(), required_default=True) i = Map(String(), required_default=True) j = MySubRecord(required_default=True) self.assertEqual(ExampleRequiredDefault.schema(), { "name": "ExampleRequiredDefault", "type": "record", "fields": [ { "name": "a", "type": [ "null", "int" ], "default": None }, { "name": "b", "type": "boolean", "default": False }, { "name": "c", "type": [ "null", "long" ], "default": None }, { "name": "d", "type": [ "null", "float" ], "default": None }, { "name": "e", "type": [ "null", "double" ], "default": None }, { "name": "f", "type": [ "null", "string" ], "default": None }, { "name": "g", "type": [ "null", "bytes" ], "default": None }, { "name": "h", "type": [ "null", { "type": "array", "items": "string" } ], "default": None }, { "name": "i", "type": [ "null", { "type": "map", "values": "string" } ], "default": None }, { "name": "j", "type": [ "null", { "name": "MySubRecord", "type": "record", "fields": [ { "name": "x", "type": [ "null", "int" ] }, { "name": "y", "type": [ "null", "long" ], }, { "name": "z", "type": [ "null", "string" ] } ] } ], "default": None } ] }) client = pulsar.Client(self.serviceUrl) producer = client.create_producer( 'my-avro-python-default-topic', schema=AvroSchema(Example)) producer_default = client.create_producer( 'my-avro-python-default-topic', schema=AvroSchema(ExampleRequiredDefault)) producer.close() producer_default.close() client.close() def test_default_value(self): class MyRecord(Record): A = Integer() B = String() C = Boolean(default=True, required=True) D = Double(default=6.4) topic = "my-default-value-topic" client = pulsar.Client(self.serviceUrl) producer = client.create_producer( topic=topic, schema=JsonSchema(MyRecord)) consumer = client.subscribe(topic, 'test', schema=JsonSchema(MyRecord)) r = MyRecord(A=5, B="text") producer.send(r) msg = consumer.receive() self.assertEqual(msg.value().A, 5) self.assertEqual(msg.value().B, u'text') self.assertEqual(msg.value().C, True) self.assertEqual(msg.value().D, 6.4) producer.close() consumer.close() client.close() def test_serialize_schema_complex(self): class Color(Enum): red = 1 green = 2 blue = 3 class NestedObj1(Record): _sorted_fields = True na1 = String() nb1 = Double() class NestedObj2(Record): _sorted_fields = True na2 = Integer() nb2 = Boolean() nc2 = NestedObj1() class NestedObj3(Record): _sorted_fields = True color = CustomEnum(Color) na3 = Integer() class NestedObj4(Record): _avro_namespace = 'xxx4' _sorted_fields = True na4 = String() nb4 = Integer() class ComplexRecord(Record): _avro_namespace = 'xxx.xxx' _sorted_fields = True a = Integer() b = Integer() color = Color color2 = Color color3 = CustomEnum(Color, required=True, default=Color.red, required_default=True) nested = NestedObj2() nested2 = NestedObj2() mapNested = Map(NestedObj3()) mapNested2 = Map(NestedObj3()) arrayNested = Array(NestedObj4()) arrayNested2 = Array(NestedObj4()) print('complex schema: ', ComplexRecord.schema()) self.assertEqual(ComplexRecord.schema(), { "name": "ComplexRecord", "namespace": "xxx.xxx", "type": "record", "fields": [ {"name": "a", "type": ["null", "int"]}, {'name': 'arrayNested', 'type': ['null', {'type': 'array', 'items': {'name': 'NestedObj4', 'namespace': 'xxx4', 'type': 'record', 'fields': [ {'name': 'na4', 'type': ['null', 'string']}, {'name': 'nb4', 'type': ['null', 'int']} ]}} ]}, {'name': 'arrayNested2', 'type': ['null', {'type': 'array', 'items': 'xxx4.NestedObj4'}]}, {"name": "b", "type": ["null", "int"]}, {'name': 'color', 'type': ['null', {'type': 'enum', 'name': 'Color', 'symbols': [ 'red', 'green', 'blue']}]}, {'name': 'color2', 'type': ['null', 'Color']}, {'name': 'color3', 'default': 'red', 'type': 'Color'}, {'name': 'mapNested', 'type': ['null', {'type': 'map', 'values': {'name': 'NestedObj3', 'type': 'record', 'fields': [ {'name': 'color', 'type': ['null', 'Color']}, {'name': 'na3', 'type': ['null', 'int']} ]}} ]}, {'name': 'mapNested2', 'type': ['null', {'type': 'map', 'values': 'NestedObj3'}]}, {"name": "nested", "type": ['null', {'name': 'NestedObj2', 'type': 'record', 'fields': [ {'name': 'na2', 'type': ['null', 'int']}, {'name': 'nb2', 'type': ['null', 'boolean']}, {'name': 'nc2', 'type': ['null', {'name': 'NestedObj1', 'type': 'record', 'fields': [ {'name': 'na1', 'type': ['null', 'string']}, {'name': 'nb1', 'type': ['null', 'double']} ]}]} ]}]}, {"name": "nested2", "type": ['null', 'NestedObj2']} ] }) def encode_and_decode(schema_type): data_schema = AvroSchema(ComplexRecord) if schema_type == 'json': data_schema = JsonSchema(ComplexRecord) nested_obj1 = NestedObj1(na1='na1 value', nb1=20.5) nested_obj2 = NestedObj2(na2=22, nb2=True, nc2=nested_obj1) r = ComplexRecord(a=1, b=2, color=Color.red, color2=Color.blue, nested=nested_obj2, nested2=nested_obj2, mapNested={ 'a': NestedObj3(na3=1, color=Color.green), 'b': NestedObj3(na3=2), 'c': NestedObj3(na3=3, color=Color.red) }, mapNested2={ 'd': NestedObj3(na3=4, color=Color.red), 'e': NestedObj3(na3=5, color=Color.blue), 'f': NestedObj3(na3=6) }, arrayNested=[ NestedObj4(na4='value na4 1', nb4=100), NestedObj4(na4='value na4 2', nb4=200) ], arrayNested2=[ NestedObj4(na4='value na4 3', nb4=300), NestedObj4(na4='value na4 4', nb4=400) ]) data_encode = data_schema.encode(r) data_decode = data_schema.decode(data_encode) self.assertEqual(data_decode.__class__.__name__, 'ComplexRecord') self.assertEqual(data_decode, r) self.assertEqual(r.color3, Color.red) self.assertEqual(r.mapNested['a'].color, Color.green) self.assertEqual(r.mapNested['b'].color, None) print('Encode and decode complex schema finish. schema_type: ', schema_type) encode_and_decode('avro') encode_and_decode('json') def test_sub_record_set_to_none(self): class NestedObj1(Record): na1 = String() nb1 = Double() class NestedObj2(Record): na2 = Integer() nb2 = Boolean() nc2 = NestedObj1() data_schema = AvroSchema(NestedObj2) r = NestedObj2(na2=1, nb2=True) data_encode = data_schema.encode(r) data_decode = data_schema.decode(data_encode) self.assertEqual(data_decode.__class__.__name__, 'NestedObj2') self.assertEqual(data_decode, r) self.assertEqual(data_decode.na2, 1) self.assertTrue(data_decode.nb2) def test_produce_and_consume_complex_schema_data(self): class Color(Enum): red = 1 green = 2 blue = 3 class NestedObj1(Record): na1 = String() nb1 = Double() class NestedObj2(Record): na2 = Integer() nb2 = Boolean() nc2 = NestedObj1() class NestedObj3(Record): na3 = Integer() color = CustomEnum(Color, required=True, required_default=True, default=Color.blue) class NestedObj4(Record): na4 = String() nb4 = Integer() class ComplexRecord(Record): a = Integer() b = Integer() color = CustomEnum(Color) nested = NestedObj2() mapNested = Map(NestedObj3()) arrayNested = Array(NestedObj4()) client = pulsar.Client(self.serviceUrl) def produce_consume_test(schema_type): topic = "my-complex-schema-topic-" + schema_type data_schema = AvroSchema(ComplexRecord) if schema_type == 'json': data_schema= JsonSchema(ComplexRecord) producer = client.create_producer( topic=topic, schema=data_schema) consumer = client.subscribe(topic, 'test', schema=data_schema) nested_obj1 = NestedObj1(na1='na1 value', nb1=20.5) nested_obj2 = NestedObj2(na2=22, nb2=True, nc2=nested_obj1) r = ComplexRecord(a=1, b=2, nested=nested_obj2, mapNested={ 'a': NestedObj3(na3=1, color=Color.red), 'b': NestedObj3(na3=2, color=Color.green), 'c': NestedObj3(na3=3) }, arrayNested=[ NestedObj4(na4='value na4 1', nb4=100), NestedObj4(na4='value na4 2', nb4=200) ]) producer.send(r) msg = consumer.receive() value = msg.value() self.assertEqual(value.__class__.__name__, 'ComplexRecord') self.assertEqual(value, r) print('Produce and consume complex schema data finish. schema_type', schema_type) produce_consume_test('avro') produce_consume_test('json') client.close() def custom_schema_test(self): def encode_and_decode(schema_definition): avro_schema = AvroSchema(None, schema_definition=schema_definition) company = { "name": "company-name", "address": 'xxx road xxx street', "employees": [ {"name": "user1", "age": 25}, {"name": "user2", "age": 30}, {"name": "user3", "age": 35}, ], "labels": { "industry": "software", "scale": ">100", "funds": "1000000.0" }, "companyType": "companyType1" } data = avro_schema.encode(company) company_decode = avro_schema.decode(data) self.assertEqual(company, company_decode) schema_definition = { 'doc': 'this is doc', 'namespace': 'example.avro', 'type': 'record', 'name': 'Company', 'fields': [ {'name': 'name', 'type': ['null', 'string']}, {'name': 'address', 'type': ['null', 'string']}, {'name': 'employees', 'type': ['null', {'type': 'array', 'items': { 'type': 'record', 'name': 'Employee', 'fields': [ {'name': 'name', 'type': ['null', 'string']}, {'name': 'age', 'type': ['null', 'int']} ] }}]}, {'name': 'labels', 'type': ['null', {'type': 'map', 'values': 'string'}]}, {'name': 'companyType', 'type': ['null', {'type': 'enum', 'name': 'CompanyType', 'symbols': ['companyType1', 'companyType2', 'companyType3']}]} ] } encode_and_decode(schema_definition) # Users could load schema from file by `fastavro.schema` # Or use `avro.schema` like this `avro.schema.parse(open("examples/company.avsc", "rb").read()).to_json()` encode_and_decode(load_schema("examples/company.avsc")) def custom_schema_produce_and_consume_test(self): client = pulsar.Client(self.serviceUrl) def produce_and_consume(topic, schema_definition): print('custom schema produce and consume test topic - ', topic) example_avro_schema = AvroSchema(None, schema_definition=schema_definition) producer = client.create_producer( topic=topic, schema=example_avro_schema) consumer = client.subscribe(topic, 'test', schema=example_avro_schema) for i in range(0, 10): company = { "name": "company-name" + str(i), "address": 'xxx road xxx street ' + str(i), "employees": [ {"name": "user" + str(i), "age": 20 + i}, {"name": "user" + str(i), "age": 30 + i}, {"name": "user" + str(i), "age": 35 + i}, ], "labels": { "industry": "software" + str(i), "scale": ">100", "funds": "1000000.0" }, "companyType": "companyType" + str((i % 3) + 1) } producer.send(company) for i in range(0, 10): msg = consumer.receive() company = { "name": "company-name" + str(i), "address": 'xxx road xxx street ' + str(i), "employees": [ {"name": "user" + str(i), "age": 20 + i}, {"name": "user" + str(i), "age": 30 + i}, {"name": "user" + str(i), "age": 35 + i}, ], "labels": { "industry": "software" + str(i), "scale": ">100", "funds": "1000000.0" } } self.assertEqual(msg.value(), company) consumer.acknowledge(msg) consumer.close() producer.close() schema_definition = { 'doc': 'this is doc', 'namespace': 'example.avro', 'type': 'record', 'name': 'Company', 'fields': [ {'name': 'name', 'type': ['null', 'string']}, {'name': 'address', 'type': ['null', 'string']}, {'name': 'employees', 'type': ['null', {'type': 'array', 'items': { 'type': 'record', 'name': 'Employee', 'fields': [ {'name': 'name', 'type': ['null', 'string']}, {'name': 'age', 'type': ['null', 'int']} ] }}]}, {'name': 'labels', 'type': ['null', {'type': 'map', 'values': 'string'}]} ] } produce_and_consume('custom-schema-test-1', schema_definition=schema_definition) produce_and_consume('custom-schema-test-2', schema_definition=load_schema("examples/company.avsc")) client.close() def test_json_schema_encode_remove_reserved_key(self): class SchemaB(Record): field = String(required=True) class SchemaA(Record): field = SchemaB() a = SchemaA(field=SchemaB(field="something")) b = JsonSchema(SchemaA).encode(a) # reserved field should not be in the encoded json self.assertTrue(b'_default' not in b) self.assertTrue(b'_required' not in b) self.assertTrue(b'_required_default' not in b) def test_schema_array_wrong_type(self): class SomeSchema(Record): some_field = Array(Integer(), required=False, default=[]) # descriptive error message with self.assertRaises(TypeError) as e: SomeSchema(some_field=["not", "integer"]) self.assertEqual(str(e.exception), "Array field some_field items should all be of type int") def test_schema_evolve(self): class User1(Record): name = String() age = Integer() class User2(Record): _sorted_fields = True name = String() age = Integer(required=True) response = requests.put('http://localhost:8080/admin/v2/namespaces/' 'public/default/schemaCompatibilityStrategy', data='"FORWARD"'.encode(), headers={'Content-Type': 'application/json'}) self.assertEqual(response.status_code, 204) topic = 'schema-test-schema-evolve-2' client = pulsar.Client(self.serviceUrl) producer1 = client.create_producer(topic, schema=AvroSchema(User1)) consumer = client.subscribe(topic, 'sub', schema=AvroSchema(User1)) reader = client.create_reader(topic, schema=AvroSchema(User1), start_message_id=pulsar.MessageId.earliest) producer2 = client.create_producer(topic, schema=AvroSchema(User2)) num_messages = 10 * 2 for i in range(int(num_messages / 2)): producer1.send(User1(age=i+100, name=f'User1 {i}')) producer2.send(User2(age=i+200, name=f'User2 {i}')) def verify_messages(msgs: List[pulsar.Message]): for i, msg in enumerate(msgs): value = msg.value() index = math.floor(i / 2) if i % 2 == 0: self.assertEqual(value.age, index + 100) self.assertEqual(value.name, f'User1 {index}') else: self.assertEqual(value.age, index + 200) self.assertEqual(value.name, f'User2 {index}') msgs1 = [] msgs2 = [] for i in range(num_messages): msgs1.append(consumer.receive()) msgs2.append(reader.read_next(1000)) verify_messages(msgs1) verify_messages(msgs2) client.close() def test_schema_type_promotion(self): client = pulsar.Client(self.serviceUrl) schemas = [("avro", AvroSchema(ExampleRecord)), ("json", JsonSchema(ExampleRecord))] for schema_name, schema in schemas: topic = f'test_schema_type_promotion_{schema_name}' consumer = client.subscribe( topic=topic, subscription_name=f'my-sub-{schema_name}', schema=schema ) producer = client.create_producer( topic=topic, schema=schema ) sendValue = ExampleRecord(str_field=b'test', int_field=1, float_field=3, bytes_field='str') producer.send(sendValue) msg = consumer.receive() msg_value = msg.value() self.assertEqual(msg_value.str_field, sendValue.str_field) self.assertEqual(msg_value.int_field, sendValue.int_field) self.assertEqual(msg_value.float_field, sendValue.float_field) self.assertEqual(msg_value.bytes_field, sendValue.bytes_field) consumer.acknowledge(msg) client.close() class ProtobufNativeSchemaTest(TestCase): """Unit tests for ProtobufNativeSchema (no Pulsar broker required).""" def test_schema_type(self): """Schema type must be PROTOBUF_NATIVE.""" import _pulsar schema = ProtobufNativeSchema(TestMessage) self.assertEqual(schema.schema_info().schema_type(), _pulsar.SchemaType.PROTOBUF_NATIVE) def test_schema_definition_keys(self): """Schema definition JSON must contain the three required keys.""" schema = ProtobufNativeSchema(TestMessage) schema_def = json.loads(schema.schema_info().schema()) self.assertIn('fileDescriptorSet', schema_def) self.assertIn('rootMessageTypeName', schema_def) self.assertIn('rootFileDescriptorName', schema_def) def test_schema_definition_values(self): """rootMessageTypeName and rootFileDescriptorName must match the descriptor.""" schema = ProtobufNativeSchema(TestMessage) schema_def = json.loads(schema.schema_info().schema()) self.assertEqual(schema_def['rootMessageTypeName'], 'test.TestMessage') self.assertEqual(schema_def['rootFileDescriptorName'], 'test_schema.proto') def test_file_descriptor_set_is_valid_base64_proto(self): """fileDescriptorSet must be valid base64-encoded FileDescriptorSet bytes.""" from google.protobuf import descriptor_pb2 schema = ProtobufNativeSchema(TestMessage) schema_def = json.loads(schema.schema_info().schema()) raw = base64.b64decode(schema_def['fileDescriptorSet']) fds = descriptor_pb2.FileDescriptorSet.FromString(raw) file_names = [f.name for f in fds.file] self.assertIn('test_schema.proto', file_names) def test_encode_decode_roundtrip(self): """encode then decode must reproduce the original message.""" schema = ProtobufNativeSchema(TestMessage) original = TestMessage(name='hello', value=42) encoded = schema.encode(original) decoded = schema.decode(encoded) self.assertEqual(decoded.name, 'hello') self.assertEqual(decoded.value, 42) def test_encode_produces_protobuf_binary(self): """Encoded bytes must be valid protobuf binary (parseable by the class directly).""" schema = ProtobufNativeSchema(TestMessage) msg = TestMessage(name='pulsar', value=100) encoded = schema.encode(msg) # Verify with protobuf's own parser reparsed = TestMessage.FromString(encoded) self.assertEqual(reparsed, msg) def test_encode_decode_nested_message(self): """encode/decode round-trip works for messages containing nested message fields.""" schema = ProtobufNativeSchema(TestMessageWithNested) original = TestMessageWithNested( str_field='test', int_field=7, double_field=3.14, nested=TestInner(inner_str='inner', inner_int=999), ) decoded = schema.decode(schema.encode(original)) self.assertEqual(decoded.str_field, 'test') self.assertEqual(decoded.int_field, 7) self.assertAlmostEqual(decoded.double_field, 3.14) self.assertEqual(decoded.nested.inner_str, 'inner') self.assertEqual(decoded.nested.inner_int, 999) def test_wrong_type_raises(self): """Encoding an object of the wrong type must raise TypeError.""" schema = ProtobufNativeSchema(TestMessage) with self.assertRaises(TypeError): schema.encode("not a protobuf message") def test_non_message_class_raises(self): """Constructing with a non-Message class must raise TypeError.""" with self.assertRaises(TypeError): ProtobufNativeSchema(str) def test_schema_name(self): """Schema name must be 'PROTOBUF_NATIVE'.""" schema = ProtobufNativeSchema(TestMessage) self.assertEqual(schema.schema_info().name(), 'PROTOBUF_NATIVE') if __name__ == '__main__': main() pulsar-client-python-3.12.0-candidate-0/tests/table_view_test.py000066400000000000000000000106711520101105200246310ustar00rootroot00000000000000#!/usr/bin/env python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # from typing import Callable from unittest import TestCase, main import time from pulsar import Client from pulsar.schema.schema import StringSchema class TableViewTest(TestCase): def setUp(self): self._client: Client = Client('pulsar://localhost:6650') def tearDown(self): self._client.close() def test_get(self): topic = f'table_view_test_get-{time.time()}' table_view = self._client.create_table_view(topic) self.assertEqual(len(table_view), 0) producer = self._client.create_producer(topic) producer.send(b'value-0', partition_key='key-0') producer.send(b'\xba\xd0\xba\xd0', partition_key='key-1') # an invalid UTF-8 bytes self._wait_for_assertion(lambda: self.assertEqual(len(table_view), 2)) self.assertEqual(table_view.get('key-0'), b'value-0') self.assertEqual(table_view.get('key-1'), b'\xba\xd0\xba\xd0') producer.send(b'value-1', partition_key='key-0') self._wait_for_assertion(lambda: self.assertEqual(table_view.get('key-0'), b'value-1')) producer.close() table_view.close() def test_for_each(self): topic = f'table_view_test_for_each-{time.time()}' table_view = self._client.create_table_view(topic) producer = self._client.create_producer(topic) producer.send(b'value-0', partition_key='key-0') producer.send(b'value-1', partition_key='key-1') self._wait_for_assertion(lambda: self.assertEqual(len(table_view), 2)) d = dict() table_view.for_each(lambda key, value: d.__setitem__(key, value)) self.assertEqual(d, { 'key-0': b'value-0', 'key-1': b'value-1' }) def listener(key: str, value: str): if len(value) == 0: d.pop(key) else: d[key] = value d.clear() table_view.for_each_and_listen(listener) self.assertEqual(d, { 'key-0': b'value-0', 'key-1': b'value-1' }) producer.send(b'value-0-new', partition_key='key-0') producer.send(b'', partition_key='key-1') producer.send(b'value-2', partition_key='key-2') def assert_latest_values(): self.assertEqual(d, { 'key-0': b'value-0-new', 'key-2': b'value-2' }) self._wait_for_assertion(assert_latest_values) def test_schema(self): topic = f'table_view_test_schema-{time.time()}' table_view = self._client.create_table_view(topic, schema=StringSchema()) producer = self._client.create_producer(topic, schema=StringSchema()) producer.send('value', partition_key='key') self._wait_for_assertion(lambda: self.assertEqual(table_view.get('key'), 'value')) self.assertEqual(table_view.get('missed-key'), None) entries = dict() table_view.for_each(lambda key, value: entries.__setitem__(key, value)) self.assertEqual(entries, {'key': 'value'}) entries.clear() table_view.for_each_and_listen(lambda key, value: entries.__setitem__(key, value)) self.assertEqual(entries, {'key': 'value'}) producer.send('new-value', partition_key='key') self._wait_for_assertion(lambda: self.assertEqual(table_view.get('key'), 'new-value')) def _wait_for_assertion(self, assertion: Callable, timeout=5) -> None: start_time = time.time() while time.time() - start_time < timeout: try: assertion() return except AssertionError: time.sleep(0.1) assertion() if __name__ == "__main__": main() pulsar-client-python-3.12.0-candidate-0/tests/test-conf/000077500000000000000000000000001520101105200227745ustar00rootroot00000000000000pulsar-client-python-3.12.0-candidate-0/tests/test-conf/.htpasswd000066400000000000000000000000541520101105200246310ustar00rootroot00000000000000admin:$apr1$FG4AO6aX$KGYPuMoLUou3i6vUkPUUf. pulsar-client-python-3.12.0-candidate-0/tests/test-conf/broker-cert.pem000066400000000000000000000152551520101105200257260ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 4098 (0x1002) Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org Validity Not Before: Feb 17 17:00:44 2021 GMT Not After : Feb 12 17:00:44 2041 GMT Subject: C=US, ST=California, O=Apache Software Foundation, OU=Pulsar, CN=localhost/emailAddress=dev@pulsar.apache.org Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:9b:2a:6f:24:02:23:f7:ff:e6:75:61:ca:07:a8: c0:ab:e9:8d:eb:51:2e:64:f7:9e:9b:d4:b4:be:3a: fa:f4:6e:c6:92:8f:38:4d:08:cd:89:15:3e:2c:c4: 99:6d:cb:58:80:fc:e0:4d:d6:7d:f6:82:ab:0d:94: f2:e2:45:c9:d3:15:95:57:0a:6c:86:dc:78:64:3b: 34:4b:01:7c:5d:de:4f:d4:21:1a:5d:27:a0:a5:70: 7a:2e:02:50:e1:19:b4:b9:05:df:99:0d:8b:cc:62: dc:10:73:fa:72:8b:38:7f:d3:56:54:61:50:bb:92: ff:09:71:09:c7:bd:04:43:3c:8c:9c:8b:32:d1:05: 04:8a:c6:89:d8:78:56:4d:da:2f:f4:ec:34:37:26: b5:87:e4:3f:26:c9:41:60:ba:31:10:19:be:f8:0c: a4:0a:85:19:59:e2:00:5d:b7:c0:bd:d1:2e:fc:a6: 34:8b:85:2a:cc:05:f6:fb:e4:00:e6:74:95:ff:02: 6f:43:7f:39:a7:c2:83:8e:5b:38:40:c9:42:c8:bc: 26:72:36:35:64:c2:54:22:11:87:e8:65:8f:3d:e9: 41:a7:6d:19:88:9a:20:9b:9a:52:e7:d2:cb:b3:e0: 2e:8f:c1:56:54:bc:6d:14:30:73:c5:d7:8e:d0:5a: 5e:cd Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Basic Constraints: CA:FALSE Netscape Cert Type: SSL Server Netscape Comment: OpenSSL Generated Server Certificate X509v3 Subject Key Identifier: 49:3C:B2:98:30:CE:7F:79:7A:C6:8B:57:CA:24:9F:12:82:1E:5D:EF X509v3 Authority Key Identifier: keyid:D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71 DirName:/C=US/ST=California/L=Palo Alto/O=Apache Software Foundation/OU=Pulsar/CN=Pulsar CA/emailAddress=dev@pulsar.apache.org serial:52:7B:B4:00:96:60:B4:26:85:BE:01:82:B8:B8:E2:8C:72:EF:5B:90 X509v3 Key Usage: critical Digital Signature, Key Encipherment X509v3 Extended Key Usage: TLS Web Server Authentication Signature Algorithm: sha256WithRSAEncryption 0f:bd:af:39:0c:2c:dc:8f:7e:06:0d:27:df:35:c7:8d:5a:03: 68:97:f6:dc:d6:d3:39:0e:b4:76:48:7d:e1:1c:a9:4b:83:fa: 52:00:ab:28:93:2d:06:76:0c:14:35:3c:f1:8e:3b:af:c8:d0: 27:1f:58:d4:71:22:5f:05:a6:9e:73:c6:a5:5e:2a:e6:fb:eb: fc:73:52:87:ca:8a:2a:f9:1e:5f:e2:b9:bd:01:27:9f:7c:61: a6:97:ad:a0:ab:4e:fb:cc:fa:c8:77:6a:65:1b:ae:60:5e:fb: 97:14:8c:40:d7:96:c6:2c:64:59:c0:52:52:7c:2d:98:4b:f4: 72:da:83:f7:c6:4f:32:42:ce:df:02:dd:5f:eb:58:42:f9:62: a1:9a:05:ef:13:48:27:af:a3:7f:23:eb:e0:dc:1d:8f:96:2a: 88:47:f7:e4:75:6f:a9:15:f6:44:f1:6d:39:3a:2c:df:a7:82: cc:7e:aa:9c:1c:c0:a7:7d:68:31:4a:4e:21:b8:9f:17:90:4b: f1:68:23:ef:a7:53:fc:a9:a8:35:6b:8f:4c:5e:d4:ea:b0:8a: 27:9a:86:89:ce:f2:5d:03:35:80:fc:45:e8:87:66:0f:32:b5: 2a:f5:1b:79:0e:09:8b:90:40:20:fb:e3:27:8a:c9:92:c1:53: 97:10:5a:8c:50:ef:02:46:7e:ec:68:c8:1e:26:66:0e:1d:d6: 6c:82:e7:38:14:e8:cb:45:77:29:5f:2c:1a:9d:d7:54:21:8a: cf:0f:b7:0c:ae:fe:d6:fb:fb:c3:07:3e:33:df:59:25:1c:73: d4:87:73:14:b4:76:16:8a:3f:82:05:7b:42:0a:55:0c:79:24: 3c:58:31:3f:e0:3e:9f:4e:d0:0e:fd:77:b7:13:2c:d3:d0:46: cc:80:09:0f:50:56:8b:6e:6e:91:b2:5b:c8:2f:4d:86:dc:72: 00:de:08:0d:5e:3e:96:1f:12:7d:3b:0d:4d:71:d5:c8:a8:06: ba:00:23:ec:10:4c:a4:c3:6f:bc:f0:d7:b1:cf:57:3f:3b:79: db:80:87:35:c7:4e:7f:bb:38:30:0a:9f:fe:5a:86:f5:97:ce: 24:38:79:fd:a0:dc:0b:82:11:a1:ea:0c:e9:16:65:e0:c0:54: 80:ad:6e:55:18:ac:27:35:3a:b0:20:70:62:8e:5d:a2:33:53: 8c:ce:f9:ee:a1:27:cb:db:e5:9a:5e:e6:f7:80:93:84:63:04: 26:58:ab:23:bb:94:80:d0:a0:55:a2:8a:ed:bc:0f:c3:41:d2: 26:a5:b9:8d:8a:45:e8:a1:fc:e8:ee:7a:64:93:ed:d6:ef:a2: 51:d7:c9:0a:31:39:35:4a -----BEGIN CERTIFICATE----- MIIGPDCCBCSgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwgaYxCzAJBgNVBAYTAlVT MRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQHDAlQYWxvIEFsdG8xIzAhBgNV BAoMGkFwYWNoZSBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ8wDQYDVQQLDAZQdWxzYXIx EjAQBgNVBAMMCVB1bHNhciBDQTEkMCIGCSqGSIb3DQEJARYVZGV2QHB1bHNhci5h cGFjaGUub3JnMB4XDTIxMDIxNzE3MDA0NFoXDTQxMDIxMjE3MDA0NFowgZIxCzAJ BgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMSMwIQYDVQQKDBpBcGFjaGUg U29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYDVQQDDAls b2NhbGhvc3QxJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9yZzCC ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJsqbyQCI/f/5nVhygeowKvp jetRLmT3npvUtL46+vRuxpKPOE0IzYkVPizEmW3LWID84E3WffaCqw2U8uJFydMV lVcKbIbceGQ7NEsBfF3eT9QhGl0noKVwei4CUOEZtLkF35kNi8xi3BBz+nKLOH/T VlRhULuS/wlxCce9BEM8jJyLMtEFBIrGidh4Vk3aL/TsNDcmtYfkPybJQWC6MRAZ vvgMpAqFGVniAF23wL3RLvymNIuFKswF9vvkAOZ0lf8Cb0N/OafCg45bOEDJQsi8 JnI2NWTCVCIRh+hljz3pQadtGYiaIJuaUufSy7PgLo/BVlS8bRQwc8XXjtBaXs0C AwEAAaOCAYQwggGAMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQDAgZAMDMGCWCG SAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2VydGlmaWNhdGUw HQYDVR0OBBYEFEk8spgwzn95esaLV8oknxKCHl3vMIHmBgNVHSMEgd4wgduAFNKy PbGkfEhLNuGn3tj8upK6p8RxoYGspIGpMIGmMQswCQYDVQQGEwJVUzETMBEGA1UE CAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJUGFsbyBBbHRvMSMwIQYDVQQKDBpBcGFj aGUgU29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYDVQQD DAlQdWxzYXIgQ0ExJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9y Z4IUUnu0AJZgtCaFvgGCuLjijHLvW5AwDgYDVR0PAQH/BAQDAgWgMBMGA1UdJQQM MAoGCCsGAQUFBwMBMA0GCSqGSIb3DQEBCwUAA4ICAQAPva85DCzcj34GDSffNceN WgNol/bc1tM5DrR2SH3hHKlLg/pSAKsoky0GdgwUNTzxjjuvyNAnH1jUcSJfBaae c8alXirm++v8c1KHyooq+R5f4rm9ASeffGGml62gq077zPrId2plG65gXvuXFIxA 15bGLGRZwFJSfC2YS/Ry2oP3xk8yQs7fAt1f61hC+WKhmgXvE0gnr6N/I+vg3B2P liqIR/fkdW+pFfZE8W05Oizfp4LMfqqcHMCnfWgxSk4huJ8XkEvxaCPvp1P8qag1 a49MXtTqsIonmoaJzvJdAzWA/EXoh2YPMrUq9Rt5DgmLkEAg++MnismSwVOXEFqM UO8CRn7saMgeJmYOHdZsguc4FOjLRXcpXywanddUIYrPD7cMrv7W+/vDBz4z31kl HHPUh3MUtHYWij+CBXtCClUMeSQ8WDE/4D6fTtAO/Xe3EyzT0EbMgAkPUFaLbm6R slvIL02G3HIA3ggNXj6WHxJ9Ow1NcdXIqAa6ACPsEEykw2+88Nexz1c/O3nbgIc1 x05/uzgwCp/+Wob1l84kOHn9oNwLghGh6gzpFmXgwFSArW5VGKwnNTqwIHBijl2i M1OMzvnuoSfL2+WaXub3gJOEYwQmWKsju5SA0KBVoortvA/DQdImpbmNikXoofzo 7npkk+3W76JR18kKMTk1Sg== -----END CERTIFICATE----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/broker-key.pem000066400000000000000000000032171520101105200255540ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEpQIBAAKCAQEAmypvJAIj9//mdWHKB6jAq+mN61EuZPeem9S0vjr69G7Gko84 TQjNiRU+LMSZbctYgPzgTdZ99oKrDZTy4kXJ0xWVVwpshtx4ZDs0SwF8Xd5P1CEa XSegpXB6LgJQ4Rm0uQXfmQ2LzGLcEHP6cos4f9NWVGFQu5L/CXEJx70EQzyMnIsy 0QUEisaJ2HhWTdov9Ow0Nya1h+Q/JslBYLoxEBm++AykCoUZWeIAXbfAvdEu/KY0 i4UqzAX2++QA5nSV/wJvQ385p8KDjls4QMlCyLwmcjY1ZMJUIhGH6GWPPelBp20Z iJogm5pS59LLs+Auj8FWVLxtFDBzxdeO0FpezQIDAQABAoIBAG9pk63mP49l1kM4 eQjw2Y9WvslVXBuxVNiNbU4eKW1zUO+RGJrvlC027JLWg1g7pwvPBvu85GspPcsd xRxFgfonyDhcSrq2+Vb2z8B/i54W73jgX/69YnMIBSKeFRbcD1C+7+MEv/l8jojd zdmLL4FQ7O7fhUl57dgIqz4Y8UOYyyBsPpz3pzJLFEb5rE/ajqmFzyl+dO+8140B niQ0+7+tAK0njX8OC0WN844GkO24WPCfWhUFrYGkfLq498eRUCWM2YP2tAJ+Uxnh v3K9icDwOX6PJXYlbvNEUCE+t60NoDYHcMpfzUdFEhBYpKadfKE/RFFcu0vAZ+aR y24oAuECgYEAyPLYXWIs88pPHQhSf2DAMRref5eeV+XA6Dy/P+z8z0bA7I6X9dl6 AK6rRKGJl9HI7c/Gky6P10fymopYopNkClXm7SBTLKx0vfjil0U6Mx5ZsfDspE3q 0o9MJKVgobCxVZlLErU55XzktKwjlv2UvDX7VuxRndqN9qdf+YSMb9kCgYEAxayx sOrJcPZVfy3Ohy5CeStF+E2dtfcKB7M7xZxZqykVy+6J1XjXHmp1L7Wpi0ju57Hi l2ZqKasHDwtlLOnfSTbvC47hsa1ydnoFTjJBObR1wS43oVkyV0AHid4w81ddOWPC H0ZmhvNe7pUxm5crpxsY6hAAraJ4Hej23MOxghUCgYEAip26UvCeQa2U1VogTm3X Jgh641kbiVabs5fz9Yzs966+9m+Gs7jJSB81Vap415mHGUTyniTIZKDk4WX9rmgt 4lNPcNOTjIWKImHFLMQ8WXbeOLkRBGYbThQ7WiwadG8GZR3Rg54vyfZVbawxAL78 ErjKIDP0OQfCVhsvQVgF6EECgYEAlQ2P+xA/Dv+gHkLjDUmTdBxuKToVZqU9merL cklfz9EuD1Tx99ajltq9PFll25IGGw0mB/WAraS5sN1tz/0VkfZrL7LwefKIcc+2 em0og6OQezcnWXGRpPqx9IJnNMY2lFSlhsGmA7I1bf9vpZvKnbmwAqZIbKUqn5sP sg2ZprUCgYEApAVD+9wXfZE/YDHVZX1k6p38ORqjq/04AJkL/LmUW5DL5to1+1KQ Q438HzMtYIq7aZyzWmlF6DmyN5mxKKK3yY79p0rvdV74AoT+ucDzM3ge0Md7liCs 0GwNnDSiPzdau738UoIKc1VbF7dMDL3LzqnfrBUCr7nXRbR3BHHuqws= -----END RSA PRIVATE KEY----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/cacert.pem000066400000000000000000000166521520101105200247520ustar00rootroot00000000000000Certificate: Data: Version: 3 (0x2) Serial Number: 52:7b:b4:00:96:60:b4:26:85:be:01:82:b8:b8:e2:8c:72:ef:5b:90 Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org Validity Not Before: Feb 17 16:43:44 2021 GMT Not After : Feb 12 16:43:44 2041 GMT Subject: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (4096 bit) Modulus: 00:b1:3c:7d:ab:4a:54:72:37:2a:92:94:0a:66:46: af:8c:ed:f4:2e:f3:87:1a:d0:c7:9d:23:35:1b:61: 74:69:ca:f7:f5:3e:95:9c:86:f2:21:34:f8:0b:ed: 45:76:22:ec:75:52:c0:67:db:2f:ba:da:25:3f:e1: 5b:ac:da:15:dd:a5:75:24:b2:12:f0:b0:ce:fd:ab: 44:06:a9:09:f6:b0:8e:8f:83:53:16:69:fa:9c:cc: 00:fa:dd:13:f3:da:fd:f2:bf:88:8e:c4:f8:1a:6f: ab:4d:f8:32:81:80:7e:51:7a:99:2d:94:cd:f3:5d: 1c:58:b2:44:f1:96:12:46:56:bd:60:8f:65:32:b7: d4:4b:7b:f3:23:88:2d:9b:a4:c4:c9:52:ea:9f:66: c1:74:be:4b:91:c6:b9:57:ec:c1:cc:81:bb:03:d5: fa:a0:46:4f:9a:a7:3e:3c:27:26:2b:97:eb:69:53: 04:75:50:97:d6:0d:90:b1:37:9f:64:df:70:4d:d9: b3:e3:b7:cc:76:50:d9:3c:9b:4c:ac:e9:26:2e:cf: ac:47:42:14:b7:60:00:0a:de:42:47:66:0c:c7:7a: b9:4d:f4:fb:c2:6a:45:78:ec:b0:b4:ce:b3:1f:50: 25:96:13:0c:55:0a:e0:d6:76:f7:1f:e1:16:e6:41: d6:72:6a:49:17:12:d9:05:8f:dc:56:b6:31:b3:b7: 9c:e3:d8:a9:99:8a:1d:3b:9d:d9:59:44:ee:46:88: 11:5f:ab:fa:38:a9:8b:d2:23:15:8b:af:1a:de:66: ba:7d:51:95:37:94:91:aa:01:01:d7:83:19:4b:5d: 8d:f4:18:39:ef:e3:32:d0:62:c8:12:50:4e:91:c2: ac:58:73:68:bb:92:20:fc:14:e5:1a:86:bd:40:4c: 94:e0:7d:0d:9c:08:57:ae:00:44:38:94:a3:3d:64: 99:43:f8:e3:12:90:14:0f:5d:63:e2:c6:07:ea:d0: 4c:8e:cf:e0:ae:34:be:86:4f:fc:58:e2:ea:f5:23: 82:37:96:02:57:1b:b4:29:ca:fd:68:a0:48:79:e8: 31:97:9a:5a:0e:2b:b4:b0:84:bb:57:4e:5f:4f:a7: 43:45:97:d7:de:05:fc:2f:6c:3e:f5:53:26:56:a3: a5:da:52:69:57:8e:a0:4b:27:50:f9:ad:6e:76:a6: 29:cc:06:94:dd:d0:ac:c6:18:22:a0:e2:bb:ed:d5: e4:97:f7:ac:23:df:75:30:41:97:07:3f:d3:12:8e: c5:a4:ef:ce:40:e8:3b:57:24:19:33:1b:ee:8a:0e: dd:0c:70:f2:1a:87:35:d9:71:d8:18:a7:9c:47:db: 93:51:c3 Exponent: 65537 (0x10001) X509v3 extensions: X509v3 Subject Key Identifier: D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71 X509v3 Authority Key Identifier: keyid:D2:B2:3D:B1:A4:7C:48:4B:36:E1:A7:DE:D8:FC:BA:92:BA:A7:C4:71 X509v3 Basic Constraints: critical CA:TRUE X509v3 Key Usage: critical Digital Signature, Certificate Sign, CRL Sign Signature Algorithm: sha256WithRSAEncryption 14:3d:7c:15:86:de:aa:5a:30:5d:d4:f2:bc:5f:10:d2:af:fe: 91:d7:ee:f3:b8:5f:ce:e4:c9:b2:01:c3:16:da:66:8e:7e:b1: c1:e3:30:ff:1d:73:d0:9c:20:3d:54:32:57:ae:07:80:4a:24: 6e:7e:32:a3:e7:23:4d:5c:31:54:8b:c1:1b:c5:bc:20:5d:43: 62:93:e0:2e:a7:01:77:39:cf:fd:ec:4c:57:09:4f:2b:ad:ac: b6:c0:be:5a:a3:ea:12:ac:5a:7f:60:23:81:bb:9a:fa:5f:7a: 67:a9:31:c3:34:af:db:ff:32:22:83:40:c2:7d:2f:39:5e:8a: 29:44:73:5f:6e:b4:f4:a2:ae:60:1f:8e:ef:91:9a:49:bb:a6: 90:2b:e0:44:95:24:8b:37:90:18:2d:41:32:8a:8e:07:8d:ea: 75:62:b8:9c:ec:73:6f:12:54:23:6d:40:00:74:c7:d3:fb:b7: 95:06:7d:cc:6d:8e:2c:d0:8b:11:06:8a:b7:43:1a:d7:e9:98: f4:c6:ef:ad:2a:75:08:fb:07:8f:20:36:7a:86:1a:cf:f7:d6: 96:ad:ed:71:59:d1:81:56:18:8d:98:c2:c0:44:e5:29:7a:7c: c0:e3:d7:fb:b8:f5:b2:50:53:8a:cf:38:ff:99:aa:bb:28:51: 60:e8:05:91:e1:ee:86:90:90:9b:87:60:63:38:cf:54:a5:82: 74:0f:40:b5:d2:6a:c5:a9:98:22:59:4e:fb:a5:81:e2:7b:0e: 3f:71:f3:24:17:1e:c5:89:fc:ae:ed:f3:69:65:02:b8:1e:98: bc:37:c6:25:36:f8:ca:99:60:8e:13:3b:33:ec:91:b3:eb:04: 6d:41:97:3e:35:c0:97:ed:66:12:25:44:23:f3:2e:fa:9c:2e: c2:ba:dd:f3:63:d7:5b:b2:72:03:4d:3b:fb:5e:29:d6:5c:02: 32:93:47:d1:4c:77:4a:58:c5:aa:81:ab:67:84:80:81:14:28: e1:db:11:16:6d:31:50:7a:47:b2:a8:2d:15:a1:c4:63:1b:ce: d5:e1:d7:57:dc:1a:71:e0:55:9f:6d:fb:be:e6:99:e8:89:be: 2c:e0:19:5e:cd:02:79:52:ee:93:56:9f:dc:d7:de:31:9b:2a: c8:91:48:a0:c7:44:7d:72:32:27:c3:2b:d8:e8:6b:94:67:b5: 1d:9d:99:25:23:d9:24:b5:ed:4b:f2:18:2d:88:f5:d4:36:bb: 53:8c:a8:b1:7f:05:13:d7:8d:89:9d:55:33:90:bc:60:99:cf: 05:ba:bd:cb:c5:61:f9:c5:1a:f7:46:9c:40:90:dd:83:aa:7a: 1f:ab:5c:10:8d:26:27:1e -----BEGIN CERTIFICATE----- MIIGPzCCBCegAwIBAgIUUnu0AJZgtCaFvgGCuLjijHLvW5AwDQYJKoZIhvcNAQEL BQAwgaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRIwEAYDVQQH DAlQYWxvIEFsdG8xIzAhBgNVBAoMGkFwYWNoZSBTb2Z0d2FyZSBGb3VuZGF0aW9u MQ8wDQYDVQQLDAZQdWxzYXIxEjAQBgNVBAMMCVB1bHNhciBDQTEkMCIGCSqGSIb3 DQEJARYVZGV2QHB1bHNhci5hcGFjaGUub3JnMB4XDTIxMDIxNzE2NDM0NFoXDTQx MDIxMjE2NDM0NFowgaYxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlh MRIwEAYDVQQHDAlQYWxvIEFsdG8xIzAhBgNVBAoMGkFwYWNoZSBTb2Z0d2FyZSBG b3VuZGF0aW9uMQ8wDQYDVQQLDAZQdWxzYXIxEjAQBgNVBAMMCVB1bHNhciBDQTEk MCIGCSqGSIb3DQEJARYVZGV2QHB1bHNhci5hcGFjaGUub3JnMIICIjANBgkqhkiG 9w0BAQEFAAOCAg8AMIICCgKCAgEAsTx9q0pUcjcqkpQKZkavjO30LvOHGtDHnSM1 G2F0acr39T6VnIbyITT4C+1FdiLsdVLAZ9svutolP+FbrNoV3aV1JLIS8LDO/atE BqkJ9rCOj4NTFmn6nMwA+t0T89r98r+IjsT4Gm+rTfgygYB+UXqZLZTN810cWLJE 8ZYSRla9YI9lMrfUS3vzI4gtm6TEyVLqn2bBdL5Lkca5V+zBzIG7A9X6oEZPmqc+ PCcmK5fraVMEdVCX1g2QsTefZN9wTdmz47fMdlDZPJtMrOkmLs+sR0IUt2AACt5C R2YMx3q5TfT7wmpFeOywtM6zH1AllhMMVQrg1nb3H+EW5kHWcmpJFxLZBY/cVrYx s7ec49ipmYodO53ZWUTuRogRX6v6OKmL0iMVi68a3ma6fVGVN5SRqgEB14MZS12N 9Bg57+My0GLIElBOkcKsWHNou5Ig/BTlGoa9QEyU4H0NnAhXrgBEOJSjPWSZQ/jj EpAUD11j4sYH6tBMjs/grjS+hk/8WOLq9SOCN5YCVxu0Kcr9aKBIeegxl5paDiu0 sIS7V05fT6dDRZfX3gX8L2w+9VMmVqOl2lJpV46gSydQ+a1udqYpzAaU3dCsxhgi oOK77dXkl/esI991MEGXBz/TEo7FpO/OQOg7VyQZMxvuig7dDHDyGoc12XHYGKec R9uTUcMCAwEAAaNjMGEwHQYDVR0OBBYEFNKyPbGkfEhLNuGn3tj8upK6p8RxMB8G A1UdIwQYMBaAFNKyPbGkfEhLNuGn3tj8upK6p8RxMA8GA1UdEwEB/wQFMAMBAf8w DgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQAUPXwVht6qWjBd1PK8 XxDSr/6R1+7zuF/O5MmyAcMW2maOfrHB4zD/HXPQnCA9VDJXrgeASiRufjKj5yNN XDFUi8EbxbwgXUNik+AupwF3Oc/97ExXCU8rray2wL5ao+oSrFp/YCOBu5r6X3pn qTHDNK/b/zIig0DCfS85XoopRHNfbrT0oq5gH47vkZpJu6aQK+BElSSLN5AYLUEy io4Hjep1Yric7HNvElQjbUAAdMfT+7eVBn3MbY4s0IsRBoq3QxrX6Zj0xu+tKnUI +wePIDZ6hhrP99aWre1xWdGBVhiNmMLAROUpenzA49f7uPWyUFOKzzj/maq7KFFg 6AWR4e6GkJCbh2BjOM9UpYJ0D0C10mrFqZgiWU77pYHiew4/cfMkFx7Fifyu7fNp ZQK4Hpi8N8YlNvjKmWCOEzsz7JGz6wRtQZc+NcCX7WYSJUQj8y76nC7Cut3zY9db snIDTTv7XinWXAIyk0fRTHdKWMWqgatnhICBFCjh2xEWbTFQekeyqC0VocRjG87V 4ddX3Bpx4FWfbfu+5pnoib4s4BlezQJ5Uu6TVp/c194xmyrIkUigx0R9cjInwyvY 6GuUZ7UdnZklI9kkte1L8hgtiPXUNrtTjKixfwUT142JnVUzkLxgmc8Fur3LxWH5 xRr3RpxAkN2Dqnofq1wQjSYnHg== -----END CERTIFICATE----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/client-cert.pem000066400000000000000000000124271520101105200257160ustar00rootroot00000000000000Certificate: Data: Version: 1 (0x0) Serial Number: 4097 (0x1001) Signature Algorithm: sha256WithRSAEncryption Issuer: C=US, ST=California, L=Palo Alto, O=Apache Software Foundation, OU=Pulsar, CN=Pulsar CA/emailAddress=dev@pulsar.apache.org Validity Not Before: Feb 17 16:56:55 2021 GMT Not After : Feb 12 16:56:55 2041 GMT Subject: C=US, ST=California, O=Apache Software Foundation, OU=Pulsar, CN=admin/emailAddress=dev@pulsar.apache.org Subject Public Key Info: Public Key Algorithm: rsaEncryption Public-Key: (2048 bit) Modulus: 00:ab:61:f5:12:b1:e1:ae:19:01:3e:59:4a:c6:ca: 00:0c:96:e8:76:3a:83:20:d9:af:3a:e1:11:20:12: e0:e4:d0:70:8f:4b:7b:af:e1:89:ef:9b:c5:a9:c2: ed:ae:24:8d:bb:42:6e:ec:59:11:3f:f5:63:59:61: 18:9f:70:b6:76:88:e2:ca:79:15:cc:fb:9c:5e:5c: bb:a1:d7:f0:d8:11:d4:17:34:1e:81:7e:0b:0e:05: be:5d:fa:d6:46:af:e1:95:d8:a0:5d:c5:2f:d9:a9: 8f:69:64:49:95:f7:42:16:6a:84:2b:2e:af:91:73: 3d:b6:d4:44:56:9a:61:43:49:15:22:ae:90:5d:04: 29:90:4e:b2:41:34:73:3e:a2:48:05:1c:bc:8e:1b: 0b:c1:d5:df:56:32:40:e9:91:a2:7b:de:31:2b:67: f1:8e:d6:c5:c0:87:57:70:29:f9:af:db:57:a0:2e: 8c:30:0a:a7:47:39:33:4c:d7:2d:32:aa:48:29:bd: c4:48:c5:58:52:07:c4:99:b1:cc:66:da:ac:28:4d: c1:bc:1f:44:3f:a3:63:61:bd:ff:48:61:76:04:b2: 7d:1c:6e:9c:ee:82:bb:f7:60:1c:7a:a0:98:be:2d: 70:43:2f:64:bf:d2:0f:20:25:f7:c7:7d:70:05:b8: 2e:bf Exponent: 65537 (0x10001) Signature Algorithm: sha256WithRSAEncryption 1c:31:b8:0f:a1:03:28:a0:da:31:ec:34:ce:e0:fd:01:99:9d: 9b:ad:f8:03:5d:20:85:18:de:ca:b5:ea:61:c9:3b:65:42:9c: e5:21:73:d2:06:41:4b:a9:3a:fb:7f:ff:45:f3:5a:4a:ab:5a: 86:cd:57:6a:5f:13:c0:ae:7e:ad:5c:6e:c3:c4:e7:b7:d3:14: bf:86:fe:f2:d1:70:0e:fc:98:50:a7:fe:53:62:5a:2d:f5:63: 2c:ee:4a:7c:dd:32:3e:d1:52:3a:1f:15:38:4b:2a:4a:ee:27: a9:d8:92:a8:33:92:83:c9:3a:09:5a:01:66:0e:68:da:8f:82: c0:18:cc:78:ea:c5:db:09:7c:2f:61:c3:51:f8:58:7a:27:d7: 92:c0:ff:f8:29:d7:a0:e9:54:17:8d:48:a8:ff:5e:92:ee:81: 6c:37:90:1c:93:28:8c:d2:f5:b1:20:96:d3:1d:0f:c0:7f:db: 0c:6d:65:7f:3a:55:e5:c9:9a:ad:09:91:a5:57:cb:fc:bf:df: 69:bd:6b:87:94:5b:d0:cf:3b:8b:48:41:3d:56:b6:1d:3f:e7: f6:b6:58:f7:54:2a:dd:da:60:68:db:9b:70:04:8b:19:c3:44: bf:1d:b4:28:b9:f8:ea:ad:d3:1a:6e:64:72:b1:61:6a:f3:e1: d4:68:56:7b:0e:ad:4c:53:1e:d2:2e:1c:bc:b7:82:59:af:65: d2:fd:ef:89:7c:34:8f:51:a1:4e:9d:7e:dc:c7:97:68:ea:aa: e5:67:ed:be:dc:38:74:0e:c3:6f:fd:08:62:54:d8:1f:15:d1: 25:fc:21:f6:8c:f9:2f:65:5e:07:b9:e9:56:ba:48:14:5c:0d: 18:ba:f8:83:54:5b:b6:27:0c:36:2c:20:29:9c:c2:68:c5:3a: 0f:a5:d6:5f:7c:aa:f9:a6:2a:2b:69:c5:b1:39:e7:1c:02:31: 5b:f5:82:de:c9:4e:8d:33:dc:94:02:44:0a:44:95:75:7b:a1: e7:ee:92:fc:35:93:73:8c:22:c1:32:ea:39:17:ca:d0:87:fc: 4d:8e:04:f8:59:66:d3:14:3f:59:ad:76:14:20:16:7b:77:4f: 94:58:f8:85:5c:ba:b3:69:ed:7f:75:54:9a:1a:88:21:5d:04: 57:87:85:e2:d4:0e:1b:61:7f:5d:36:dc:72:a1:9d:0b:c8:ce: 19:69:49:fa:1b:bb:3f:3d:1b:4d:81:42:95:4e:d8:0b:04:d1: 08:6d:15:b3:ae:52:41:12:ff:e1:90:c4:7d:52:88:55:8b:87: 83:06:48:8b:fc:3a:a7:47:0e:6c:a8:4c:9e:b0:aa:da:50:f5: 97:97:98:3e:9d:18:ef:43 -----BEGIN CERTIFICATE----- MIIEqzCCApMCAhABMA0GCSqGSIb3DQEBCwUAMIGmMQswCQYDVQQGEwJVUzETMBEG A1UECAwKQ2FsaWZvcm5pYTESMBAGA1UEBwwJUGFsbyBBbHRvMSMwIQYDVQQKDBpB cGFjaGUgU29mdHdhcmUgRm91bmRhdGlvbjEPMA0GA1UECwwGUHVsc2FyMRIwEAYD VQQDDAlQdWxzYXIgQ0ExJDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hl Lm9yZzAeFw0yMTAyMTcxNjU2NTVaFw00MTAyMTIxNjU2NTVaMIGOMQswCQYDVQQG EwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEjMCEGA1UECgwaQXBhY2hlIFNvZnR3 YXJlIEZvdW5kYXRpb24xDzANBgNVBAsMBlB1bHNhcjEOMAwGA1UEAwwFYWRtaW4x JDAiBgkqhkiG9w0BCQEWFWRldkBwdWxzYXIuYXBhY2hlLm9yZzCCASIwDQYJKoZI hvcNAQEBBQADggEPADCCAQoCggEBAKth9RKx4a4ZAT5ZSsbKAAyW6HY6gyDZrzrh ESAS4OTQcI9Le6/hie+bxanC7a4kjbtCbuxZET/1Y1lhGJ9wtnaI4sp5Fcz7nF5c u6HX8NgR1Bc0HoF+Cw4Fvl361kav4ZXYoF3FL9mpj2lkSZX3QhZqhCsur5FzPbbU RFaaYUNJFSKukF0EKZBOskE0cz6iSAUcvI4bC8HV31YyQOmRonveMStn8Y7WxcCH V3Ap+a/bV6AujDAKp0c5M0zXLTKqSCm9xEjFWFIHxJmxzGbarChNwbwfRD+jY2G9 /0hhdgSyfRxunO6Cu/dgHHqgmL4tcEMvZL/SDyAl98d9cAW4Lr8CAwEAATANBgkq hkiG9w0BAQsFAAOCAgEAHDG4D6EDKKDaMew0zuD9AZmdm634A10ghRjeyrXqYck7 ZUKc5SFz0gZBS6k6+3//RfNaSqtahs1Xal8TwK5+rVxuw8Tnt9MUv4b+8tFwDvyY UKf+U2JaLfVjLO5KfN0yPtFSOh8VOEsqSu4nqdiSqDOSg8k6CVoBZg5o2o+CwBjM eOrF2wl8L2HDUfhYeifXksD/+CnXoOlUF41IqP9eku6BbDeQHJMojNL1sSCW0x0P wH/bDG1lfzpV5cmarQmRpVfL/L/fab1rh5Rb0M87i0hBPVa2HT/n9rZY91Qq3dpg aNubcASLGcNEvx20KLn46q3TGm5kcrFhavPh1GhWew6tTFMe0i4cvLeCWa9l0v3v iXw0j1GhTp1+3MeXaOqq5Wftvtw4dA7Db/0IYlTYHxXRJfwh9oz5L2VeB7npVrpI FFwNGLr4g1RbticMNiwgKZzCaMU6D6XWX3yq+aYqK2nFsTnnHAIxW/WC3slOjTPc lAJECkSVdXuh5+6S/DWTc4wiwTLqORfK0If8TY4E+Flm0xQ/Wa12FCAWe3dPlFj4 hVy6s2ntf3VUmhqIIV0EV4eF4tQOG2F/XTbccqGdC8jOGWlJ+hu7Pz0bTYFClU7Y CwTRCG0Vs65SQRL/4ZDEfVKIVYuHgwZIi/w6p0cObKhMnrCq2lD1l5eYPp0Y70M= -----END CERTIFICATE----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/client-key.pem000066400000000000000000000032131520101105200255420ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEoQIBAAKCAQEAq2H1ErHhrhkBPllKxsoADJbodjqDINmvOuERIBLg5NBwj0t7 r+GJ75vFqcLtriSNu0Ju7FkRP/VjWWEYn3C2dojiynkVzPucXly7odfw2BHUFzQe gX4LDgW+XfrWRq/hldigXcUv2amPaWRJlfdCFmqEKy6vkXM9ttREVpphQ0kVIq6Q XQQpkE6yQTRzPqJIBRy8jhsLwdXfVjJA6ZGie94xK2fxjtbFwIdXcCn5r9tXoC6M MAqnRzkzTNctMqpIKb3ESMVYUgfEmbHMZtqsKE3BvB9EP6NjYb3/SGF2BLJ9HG6c 7oK792AceqCYvi1wQy9kv9IPICX3x31wBbguvwIDAQABAoIBAQCcwbSPrPRncaeZ h8LFoO36le16dnqKCZIloMcxNxNNNvo9lyVC8mBgMXLSm+Eab4TTyyf6Nl14ytJc ZltHOqkqMnp+B9LQ8zNLfDaDCijY+TWtI5bjio5B/S7qdwyXCzii/slv+3SQ+m6a T4ifCtH//t11QfaEa4v/NphrPjnIeAgB681bk8nKdRop84ar+51lgbHoAza+wv+8 e+aK3Od8r4yD19ZoPiMg0o4t2cEi8kupVgjsuZVtcvF9Q6QLYV17BFYEHqYjcr18 N1EJ96f2FLO6cwEM+cG4n8gHjfDGRcDlhT9Cum1kDpg4J88auVUXnrDyi5Dcv1Pz 6EC+ZmXBAoGBAOHUSUDMkbEePKDaM3Z+4jLqZWc3UZhxQLnqg5l7phdQ6iSogQX9 1LpZCJ+lOMTHBCnaTCoQpuSHgYgraVkD4KG6nzC423oDesd/xNvlfW3TRsmwZWbL khdcdBSoVy3Kbv1v8kxw0NlcR68qo1XYfmFCAITcFHdxDz/jGStydlR9AoGBAMJH gyPenL595X8t47R93rkGOIx5cVf5YrDIZCByp4K44Tf9OqZHbky7jSPSSbur10mI pypRq5EcZ/cudU4w4gGaMauczt5Dgvlqd3T+GTZY3jO8bxi66gvzYTbigAxaJWcY Uafiv5W9ldRKsY3pyCL8ubg38Ed2cSaS2wGd/SDrAn8NO2MPaO0gc6UZx688QjL+ yL0oTxV42Snxusv7MkOJGjSd8UGeGEFeqdjXgdbRsNeNnDzaOh+NRGNSlziU/qUq 1MR/FlXF0G5hQhtGxyuSQ87iAnPukf79X21tyG9TP4lBUE3iLLoQAlgw606muQiu qi9dmYeZeAZst+HBqfNFAoGADg6qmH/VC5uEbY1eeoLZCL5AfTmUT+9FitEVHZvu LvE9qpVyFvH4Mykm7z6aAzBN5Y4zukYqiddqVmJQLpYu5DrJ+UbhWQe9hFqFxjtU i7Amc8vgpgNwR+kWUahV547mQe1qiyFHB4iuPKwi6MfPqWhr775sbl9NlKLvodBS rn0CgYBDrLH6ehNV/RnJIVZYQD6YcocYdYFy4u76mCYKmEP57XmstZHZXQgiRwbK Oy2Yg/qieKtSMjstgHFK6ZNYIR37l9J9Lh9aeal61+wW2dsGEy29Rhg01FpvKReq wCHz3tneUyaOhq9m0gKMOpWYcO+FBX1/2K5Gwj8FgEpu9r2b3w== -----END RSA PRIVATE KEY----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/client-ssl.conf000066400000000000000000000021511520101105200257170ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # Pulsar Client configuration webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ tlsAllowInsecureConnection=false tlsTrustCertsFilePath=test-conf/cacert.pem authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:test-conf/client-cert.pem,tlsKeyFile:test-conf/client-key.pem pulsar-client-python-3.12.0-candidate-0/tests/test-conf/client.conf000066400000000000000000000017311520101105200251230ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # # Pulsar Client configuration webServiceUrl=http://localhost:8765/ brokerServiceUrl=pulsar://localhost:8885/ #authPlugin= #authParams= #useTls= #tlsAllowInsecureConnection #tlsTrustCertsFilePath pulsar-client-python-3.12.0-candidate-0/tests/test-conf/cpp_credentials_file.json000066400000000000000000000002131520101105200300210ustar00rootroot00000000000000{ "client_id":"Xd23RHsUnvUlP7wchjNYOaIfazgeHd9x", "client_secret":"rT7ps7WY8uhdVuBTKWZkttwLdQotmdEliaM5rLfmgNibvqziZ-g07ZH52N_poGAb" } pulsar-client-python-3.12.0-candidate-0/tests/test-conf/private-key.client-ecdsa.pem000066400000000000000000000011631520101105200302720ustar00rootroot00000000000000-----BEGIN EC PARAMETERS----- MIGXAgEBMBwGByqGSM49AQECEQD////9////////////////MDsEEP////3///// //////////wEEOh1ecEQefQ92CSZPCzuXtMDFQAADg1NaW5naHVhUXUMwDpEc9A2 eQQhBBYf91KLiZstDChgfKUsW4bPWsg5W6/rE8AtopLd7XqDAhEA/////gAAAAB1 ow0bkDihFQIBAQ== -----END EC PARAMETERS----- -----BEGIN EC PRIVATE KEY----- MIHYAgEBBBDeu9hc8kOvL3pl+LYSjLq9oIGaMIGXAgEBMBwGByqGSM49AQECEQD/ ///9////////////////MDsEEP////3///////////////wEEOh1ecEQefQ92CSZ PCzuXtMDFQAADg1NaW5naHVhUXUMwDpEc9A2eQQhBBYf91KLiZstDChgfKUsW4bP Wsg5W6/rE8AtopLd7XqDAhEA/////gAAAAB1ow0bkDihFQIBAaEkAyIABOsqPpE8 cY80pxkog5xw3i2AQ0yfV3MqMusxlOQnigBp -----END EC PRIVATE KEY----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/private-key.client-mismatch-rsa.pem000066400000000000000000000031631520101105200316050ustar00rootroot00000000000000-----BEGIN EC PARAMETERS----- MIIBwgIBATBNBgcqhkjOPQEBAkIB//////////////////////////////////// //////////////////////////////////////////////////8wgZ4EQgH///// //////////////////////////////////////////////////////////////// /////////////////ARBUZU+uWGOHJofkpohoLaFQO6i2nJbmbMV87i0iZGO8Qnh Vhk5Uex+k3sWUsC9O7G/BzVz34g9LDTx70Uf1GtQPwADFQDQnogAKRy4U5bMZxc5 MoSqoNpkugSBhQQAxoWOBrcEBOnNnj7LZiOVtEKcZIE5BT+1Ifgor2BrTT26oUte d+/nWSj+HcEnov+o3jNIs8GFakKb+X5+McLlvWYBGDkpaniaO8AEXIpftCx9G9mY 9URJV5tEaBevvRcnPmYsl+5ymV70JkDFULkBP60HYTU8cIaicsJAiL6Udp/RZlAC QgH///////////////////////////////////////////pRhoeDvy+Wa3/MAUj3 CaXQO7XJuImcR667b7cekThkCQIBAQ== -----END EC PARAMETERS----- -----BEGIN EC PRIVATE KEY----- MIICnQIBAQRCAeNLEp1HefZ1nMl5vvgFMsJCd5ieCWqPT7TXbQkn27A8WkyAGTYC GtolyPokOgSjbJh+ofBt/MgvE/nMrqzmkZVtoIIBxjCCAcICAQEwTQYHKoZIzj0B AQJCAf////////////////////////////////////////////////////////// ////////////////////////////MIGeBEIB//////////////////////////// //////////////////////////////////////////////////////////wEQVGV PrlhjhyaH5KaIaC2hUDuotpyW5mzFfO4tImRjvEJ4VYZOVHsfpN7FlLAvTuxvwc1 c9+IPSw08e9FH9RrUD8AAxUA0J6IACkcuFOWzGcXOTKEqqDaZLoEgYUEAMaFjga3 BATpzZ4+y2YjlbRCnGSBOQU/tSH4KK9ga009uqFLXnfv51ko/h3BJ6L/qN4zSLPB hWpCm/l+fjHC5b1mARg5KWp4mjvABFyKX7QsfRvZmPVESVebRGgXr70XJz5mLJfu cple9CZAxVC5AT+tB2E1PHCGonLCQIi+lHaf0WZQAkIB//////////////////// ///////////////////////6UYaHg78vlmt/zAFI9wml0Du1ybiJnEeuu2+3HpE4 ZAkCAQGhgYkDgYYABAFhUHeaHfIWre/pPmv2a2l891co79dFpg6ixPRg+Y5qe0C7 src//LT/ZR5rgj8ne+YcaIlwyQRl5OYEd25n799IcgHIBTGyaLB6Td5mW/oWT/Fz soufOnUJ7O/kDHjIQ15sczk3rDhe8/mB9zPjKlKTuAl5jBEt6E3yiB44Dtng02xD uQ== -----END EC PRIVATE KEY----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/private-key.client-rsa.pem000066400000000000000000000032131520101105200277760ustar00rootroot00000000000000-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEAtKWwgqdnTYrOCv+j1MkTWfSH0wCsHZZca9wAW3qP4uuhlBvn b10JcFf5ZjzP9BSXK+tHmI8uoN368vEv6yhURHM4yuXqzCxzuAwkQSo39rzX8PGC 7qdjCN7LDJ3MnqiBIrUsSaEP1wrNsB1kI+o9ER1e5O/uEPAotP933hHQ0J2hMEek HqL7sBlJ98h6NmsicEaUkardk0TOXrlkjC+cMd8ZbGScPqI9M38bmn3OLxFTn1vt hpvnXLvCmG4M+6xtYtD+npcVPZw1i1R90fMs7ppZnRbv8Hc/DFdOKVQIgam6CDdn NKgW7c7IBMrP0AEm37HTu0LSOjP2OHXlvvlQGQIDAQABAoIBAAaJFAi2C7u3cNrf AstY9vVDLoLIvHFZlkBktjKZDYmVIsRb+hSCViwVUrWLL67R6+Iv4eg4DeTOAx00 8pncXKgZTw2wIb1/QjR/Y/RjlaC8lkdmRWli7udMQCZVsyhuSjW6Pj7vr8YE4woj FhNijxEGcf9wWrmMJrzdnTWQiXByo+eTvUQ9BPgPGrRjsMZmTkLyAVJff2DfxO5b IWFDYDJcyYAMCIMQu7vys/I50ou6ilb1CO6QM6Z7KpPeOoVFPwtzbh8cf9xM8UNS j6J/JmdWhgI34GS3NA68xTQ6PV7zjnhCc+iccm3JKyzGXwaApAZ+Eoce/9j4WKmu 5B4ziR0CgYEA3l/9OHbl1zmyV+rRxWOIj/i2rTvHzwBnbnPJyuemL5VMFdpGodQ3 vwHvyQmcECRVRxmXojQ4QuPPHs3qp6wEEFPCWxChLSTxlUc85SOFHWU2O99jV7zI 7+JOpDK/Mstsx9nHgXduJF+glTFtA3LH8Oqylzu2aFPsprwKuZf94Q8CgYEAz/Zx akEG+PEMtP5YS28cX5XfjsIX/V26Fs6/sH16QjUIEddE5T4fCuokxCjSiwUcWhml pHEJ5S5xp3VYRfISW3jRW3qstIH1tpZipB6+S0zTuJmLJbA3IiWEg2rtMt7X1uJv A/bYOqe0hOPTuXuZdtVZ0nMTKk7GG8O6VkBI7FcCgYEAkDfCmscJgs7JahlBWHmX zH9pwem+SPKjIc/4NB6N+dgikx2Pp05hpP/VihUwYIufvs/LNogVYNQrtHepUnrN 2+TmbHbZgNSv1Ldxt82UfB7y0FutKu6lhmXHyNecho3Fi8sih0V0aiSWmYuHfrAH GaiskEZKo1iiZvQXJIx9O2MCgYATBf0r9hTYMtyxtc6H3/sdd01C9thQ8gDy0yjP 0Tqc0dMSJroDqmIWkoKYew9/bhFA4LW5TCnWkCAPbHmNtG4fdfbYwmkH/hdnA2y0 jKdlpfp8GXeUFAGHGx17FA3sqFvgKUh0eWEgRHUL7vdQMVFBgJS93o7zQM94fLgP 6cOB8wKBgFcGV4GjI2Ww9cillaC554MvoSjf8B/+04kXzDOh8iYIIzO9EUil1jjK Jvxp4hnLzTKWbux3MEWqurLkYas6GpKBjw+iNOCar6YdqWGVqM3RUx7PTUaZwkKx UdP63IfY7iZCIT/QbyHQvIUe2MaiVnH+ulxdkK6Y5e7gxcbckIH4 -----END RSA PRIVATE KEY----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/public-key.client-ecdsa.pem000066400000000000000000000005151520101105200300760ustar00rootroot00000000000000-----BEGIN PUBLIC KEY----- MIHKMIGjBgcqhkjOPQIBMIGXAgEBMBwGByqGSM49AQECEQD////9//////////// ////MDsEEP////3///////////////wEEOh1ecEQefQ92CSZPCzuXtMDFQAADg1N aW5naHVhUXUMwDpEc9A2eQQhBBYf91KLiZstDChgfKUsW4bPWsg5W6/rE8AtopLd 7XqDAhEA/////gAAAAB1ow0bkDihFQIBAQMiAATrKj6RPHGPNKcZKIOccN4tgENM n1dzKjLrMZTkJ4oAaQ== -----END PUBLIC KEY----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/public-key.client-mismatch-rsa.pem000066400000000000000000000007031520101105200314060ustar00rootroot00000000000000-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtKWwgqdnTYrOCv+j1MkT WfSH0wCsHZZca9wAW3qP4uuhlBvnb10JcFf5ZjzP9BSXK+tHmI8uoN368vEv6yhU RHM4yuXqzCxzuAwkQSo39rzX8PGC7qdjCN7LDJ3MnqiBIrUsSaEP1wrNsB1kI+o9 ER1e5O/uEPAotP933hHQ0J2hMEekHqL7sBlJ98h6NmsicEaUkardk0TOXrlkjC+c Md8ZbGScPqI9M38bmn3OLxFTn1vthpvnXLvCmG4M+6xtYtD+npcVPZw1i1R90fMs 7ppZnRbv8Hc/DFdOKVQIgam6CDdnNKgW7c7IBMrP0AEm37HTu0LSOjP2OHXlvvlQ GQIDAQAB -----END PUBLIC KEY----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/public-key.client-rsa.pem000066400000000000000000000007031520101105200276030ustar00rootroot00000000000000-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtKWwgqdnTYrOCv+j1MkT WfSH0wCsHZZca9wAW3qP4uuhlBvnb10JcFf5ZjzP9BSXK+tHmI8uoN368vEv6yhU RHM4yuXqzCxzuAwkQSo39rzX8PGC7qdjCN7LDJ3MnqiBIrUsSaEP1wrNsB1kI+o9 ER1e5O/uEPAotP933hHQ0J2hMEekHqL7sBlJ98h6NmsicEaUkardk0TOXrlkjC+c Md8ZbGScPqI9M38bmn3OLxFTn1vthpvnXLvCmG4M+6xtYtD+npcVPZw1i1R90fMs 7ppZnRbv8Hc/DFdOKVQIgam6CDdnNKgW7c7IBMrP0AEm37HTu0LSOjP2OHXlvvlQ GQIDAQAB -----END PUBLIC KEY----- pulsar-client-python-3.12.0-candidate-0/tests/test-conf/standalone-ssl.conf000066400000000000000000000271121520101105200265750ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # ### --- General broker settings --- ### # Zookeeper quorum connection string zookeeperServers= # Configuration Store connection string configurationStoreServers= brokerServicePort=6650 brokerServicePortTls=6651 # Port to use to server HTTP request webServicePort=8080 webServicePortTls=8443 # Hostname or IP address the service binds on, default is 0.0.0.0. bindAddress=0.0.0.0 # Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getCanonicalHostName() is used. advertisedAddress=localhost # Name of the cluster to which this broker belongs to clusterName=standalone # Zookeeper session timeout in milliseconds zooKeeperSessionTimeoutMillis=30000 # Time to wait for broker graceful shutdown. After this time elapses, the process will be killed brokerShutdownTimeoutMs=3000 # Enable backlog quota check. Enforces action on topic when the quota is reached backlogQuotaCheckEnabled=true # How often to check for topics that have reached the quota backlogQuotaCheckIntervalInSeconds=60 # Default per-topic backlog quota limit backlogQuotaDefaultLimitGB=10 # Enable the deletion of inactive topics brokerDeleteInactiveTopicsEnabled=true # How often to check for inactive topics brokerDeleteInactiveTopicsFrequencySeconds=60 # How frequently to proactively check and purge expired messages messageExpiryCheckIntervalInMinutes=5 # Enable check for minimum allowed client library version clientLibraryVersionCheckEnabled=false # Allow client libraries with no version information clientLibraryVersionCheckAllowUnversioned=true # Path for the file used to determine the rotation status for the broker when responding # to service discovery health checks statusFilePath=/usr/local/apache/htdocs # Max number of unacknowledged messages allowed to receive messages by a consumer on a shared subscription. Broker will stop sending # messages to consumer once, this limit reaches until consumer starts acknowledging messages back # Using a value of 0, is disabling unackeMessage limit check and consumer can receive messages without any restriction maxUnackedMessagesPerConsumer=50000 subscriptionRedeliveryTrackerEnabled=true ### --- Authentication --- ### # Enable TLS tlsEnabled=true tlsCertificateFilePath=test-conf/broker-cert.pem tlsKeyFilePath=test-conf/broker-key.pem tlsTrustCertsFilePath=test-conf/cacert.pem tlsAllowInsecureConnection=false anonymousUserRole=anonymous # Enable authentication authenticationEnabled=true # Authentication provider name list, which is comma separated list of class names authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls,org.apache.pulsar.broker.authentication.AuthenticationProviderToken,org.apache.pulsar.broker.authentication.AuthenticationProviderBasic # Enforce authorization authorizationEnabled=true tokenSecretKey=file:///pulsar/data/tokens/secret.key # Role names that are treated as "super-user", meaning they will be able to do all admin # operations and publish/consume from all topics superUserRoles=localhost,superUser,admin # Authentication settings of the broker itself. Used when the broker connects to other brokers, # either in same or other clusters brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= # Enable batch index ACK acknowledgmentAtBatchIndexLevelEnabled=true ### --- BookKeeper Client --- ### # Authentication plugin to use when connecting to bookies bookkeeperClientAuthenticationPlugin= # BookKeeper auth plugin implementation specifics parameters name and values bookkeeperClientAuthenticationParametersName= bookkeeperClientAuthenticationParameters= # Timeout for BK add / read operations bookkeeperClientTimeoutInSeconds=30 # Speculative reads are initiated if a read request doesn't complete within a certain time # Using a value of 0, is disabling the speculative reads bookkeeperClientSpeculativeReadTimeoutInMillis=0 # Enable bookies health check. Bookies that have more than the configured number of failure within # the interval will be quarantined for some time. During this period, new ledgers won't be created # on these bookies bookkeeperClientHealthCheckEnabled=true bookkeeperClientHealthCheckIntervalSeconds=60 bookkeeperClientHealthCheckErrorThresholdPerInterval=5 bookkeeperClientHealthCheckQuarantineTimeInSeconds=1800 # Enable rack-aware bookie selection policy. BK will chose bookies from different racks when # forming a new bookie ensemble bookkeeperClientRackawarePolicyEnabled=true # Enable region-aware bookie selection policy. BK will chose bookies from # different regions and racks when forming a new bookie ensemble # If enabled, the value of bookkeeperClientRackawarePolicyEnabled is ignored bookkeeperClientRegionawarePolicyEnabled=false # Minimum number of racks per write quorum. BK rack-aware bookie selection policy will try to # get bookies from at least 'bookkeeperClientMinNumRacksPerWriteQuorum' racks for a write quorum. bookkeeperClientMinNumRacksPerWriteQuorum=1 # Enforces rack-aware bookie selection policy to pick bookies from 'bookkeeperClientMinNumRacksPerWriteQuorum' # racks for a writeQuorum. # If BK can't find bookie then it would throw BKNotEnoughBookiesException instead of picking random one. bookkeeperClientEnforceMinNumRacksPerWriteQuorum=false # Enable/disable reordering read sequence on reading entries. bookkeeperClientReorderReadSequenceEnabled=false # Enable bookie isolation by specifying a list of bookie groups to choose from. Any bookie # outside the specified groups will not be used by the broker bookkeeperClientIsolationGroups= ### --- Managed Ledger --- ### # Number of bookies to use when creating a ledger managedLedgerDefaultEnsembleSize=1 # Number of copies to store for each message managedLedgerDefaultWriteQuorum=1 # Number of guaranteed copies (acks to wait before write is complete) managedLedgerDefaultAckQuorum=1 # Amount of memory to use for caching data payload in managed ledger. This memory # is allocated from JVM direct memory and it's shared across all the topics # running in the same broker managedLedgerCacheSizeMB=1024 # Threshold to which bring down the cache level when eviction is triggered managedLedgerCacheEvictionWatermark=0.9 # Rate limit the amount of writes generated by consumer acking the messages managedLedgerDefaultMarkDeleteRateLimit=0.1 # Max number of entries to append to a ledger before triggering a rollover # A ledger rollover is triggered after the min rollover time has passed # and one of the following conditions is true: # * The max rollover time has been reached # * The max entries have been written to the ledger # * The max ledger size has been written to the ledger managedLedgerMaxEntriesPerLedger=50000 # Minimum time between ledger rollover for a topic managedLedgerMinLedgerRolloverTimeMinutes=10 # Maximum time before forcing a ledger rollover for a topic managedLedgerMaxLedgerRolloverTimeMinutes=240 # Max number of entries to append to a cursor ledger managedLedgerCursorMaxEntriesPerLedger=50000 # Max time before triggering a rollover on a cursor ledger managedLedgerCursorRolloverTimeInSeconds=14400 ### --- Load balancer --- ### # Enable load balancer loadBalancerEnabled=false # Strategy to assign a new bundle loadBalancerPlacementStrategy=weightedRandomSelection # Percentage of change to trigger load report update loadBalancerReportUpdateThresholdPercentage=10 # maximum interval to update load report loadBalancerReportUpdateMaxIntervalMinutes=15 # Frequency of report to collect loadBalancerHostUsageCheckIntervalMinutes=1 # Load shedding interval. Broker periodically checks whether some traffic should be offload from # some over-loaded broker to other under-loaded brokers loadBalancerSheddingIntervalMinutes=30 # Prevent the same topics to be shed and moved to other broker more than once within this timeframe loadBalancerSheddingGracePeriodMinutes=30 # Usage threshold to determine a broker as under-loaded loadBalancerBrokerUnderloadedThresholdPercentage=1 # Usage threshold to determine a broker as over-loaded loadBalancerBrokerOverloadedThresholdPercentage=85 # Interval to update namespace bundle resource quota loadBalancerResourceQuotaUpdateIntervalMinutes=15 # Usage threshold to determine a broker is having just right level of load loadBalancerBrokerComfortLoadLevelPercentage=65 # enable/disable namespace bundle auto split loadBalancerAutoBundleSplitEnabled=false # interval to detect & split hot namespace bundle loadBalancerNamespaceBundleSplitIntervalMinutes=15 # maximum topics in a bundle, otherwise bundle split will be triggered loadBalancerNamespaceBundleMaxTopics=1000 # maximum sessions (producers + consumers) in a bundle, otherwise bundle split will be triggered loadBalancerNamespaceBundleMaxSessions=1000 # maximum msgRate (in + out) in a bundle, otherwise bundle split will be triggered loadBalancerNamespaceBundleMaxMsgRate=1000 # maximum bandwidth (in + out) in a bundle, otherwise bundle split will be triggered loadBalancerNamespaceBundleMaxBandwidthMbytes=100 # maximum number of bundles in a namespace loadBalancerNamespaceMaximumBundles=128 ### --- Replication --- ### # Enable replication metrics replicationMetricsEnabled=true # Max number of connections to open for each broker in a remote cluster # More connections host-to-host lead to better throughput over high-latency # links. replicationConnectionsPerBroker=16 # Replicator producer queue size replicationProducerQueueSize=1000 # Default message retention time. 0 means retention is disabled. -1 means data is not removed by time quota defaultRetentionTimeInMinutes=0 # Default retention size. 0 means retention is disabled. -1 means data is not removed by size quota defaultRetentionSizeInMB=0 # How often to check whether the connections are still alive keepAliveIntervalSeconds=30 # Enable topic auto creation if new producer or consumer connected (disable auto creation with value false) allowAutoTopicCreation=true # The type of topic that is allowed to be automatically created.(partitioned/non-partitioned) allowAutoTopicCreationType=non-partitioned # The number of partitioned topics that is allowed to be automatically created if allowAutoTopicCreationType is partitioned. defaultNumPartitions=1 ### --- Deprecated config variables --- ### # Deprecated. Use configurationStoreServers globalZookeeperServers= # Deprecated. Use brokerDeleteInactiveTopicsFrequencySeconds brokerServicePurgeInactiveFrequencyInSeconds=60 # Given a specific limit of the max message size maxMessageSize=1024000 # Disable consistent hashing to fix flaky `KeySharedConsumerTest#testMultiTopics`. subscriptionKeySharedUseConsistentHashing=false # It's true by default since 2.11. After https://github.com/apache/pulsar/pull/21445, we must configure # brokerClientAuthenticationPlugin and brokerClientAuthenticationParameters correctly when enabling topic # level policies. Otherwise, no topic could be loaded. topicLevelPoliciesEnabled=false pulsar-client-python-3.12.0-candidate-0/tests/test-conf/standalone.conf000066400000000000000000000250621520101105200260000ustar00rootroot00000000000000# # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # ### --- General broker settings --- ### # Zookeeper quorum connection string zookeeperServers= # Configuration Store connection string configurationStoreServers= brokerServicePort=8885 # Port to use to server HTTP request webServicePort=8765 # Hostname or IP address the service binds on, default is 0.0.0.0. bindAddress=0.0.0.0 # Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getCanonicalHostName() is used. advertisedAddress=localhost # Name of the cluster to which this broker belongs to clusterName=standalone # Zookeeper session timeout in milliseconds zooKeeperSessionTimeoutMillis=30000 # Time to wait for broker graceful shutdown. After this time elapses, the process will be killed brokerShutdownTimeoutMs=3000 # Enable backlog quota check. Enforces action on topic when the quota is reached backlogQuotaCheckEnabled=true # How often to check for topics that have reached the quota backlogQuotaCheckIntervalInSeconds=60 # Default per-topic backlog quota limit backlogQuotaDefaultLimitGB=10 # Enable the deletion of inactive topics brokerDeleteInactiveTopicsEnabled=true # How often to check for inactive topics brokerDeleteInactiveTopicsFrequencySeconds=60 # How frequently to proactively check and purge expired messages messageExpiryCheckIntervalInMinutes=5 # Enable check for minimum allowed client library version clientLibraryVersionCheckEnabled=false # Allow client libraries with no version information clientLibraryVersionCheckAllowUnversioned=true # Path for the file used to determine the rotation status for the broker when responding # to service discovery health checks statusFilePath=/usr/local/apache/htdocs # Max number of unacknowledged messages allowed to receive messages by a consumer on a shared subscription. Broker will stop sending # messages to consumer once, this limit reaches until consumer starts acknowledging messages back # Using a value of 0, is disabling unackeMessage limit check and consumer can receive messages without any restriction maxUnackedMessagesPerConsumer=50000 subscriptionRedeliveryTrackerEnabled=true ### --- Authentication --- ### # Enable authentication authenticationEnabled=false # Authentication provider name list, which is comma separated list of class names authenticationProviders=false # Enforce authorization authorizationEnabled=false # Role names that are treated as "super-user", meaning they will be able to do all admin # operations and publish/consume from all topics superUserRoles= # Authentication settings of the broker itself. Used when the broker connects to other brokers, # either in same or other clusters brokerClientAuthenticationPlugin= brokerClientAuthenticationParameters= # Enable batch index ACK acknowledgmentAtBatchIndexLevelEnabled=true ### --- BookKeeper Client --- ### # Authentication plugin to use when connecting to bookies bookkeeperClientAuthenticationPlugin= # BookKeeper auth plugin implementation specifics parameters name and values bookkeeperClientAuthenticationParametersName= bookkeeperClientAuthenticationParameters= # Timeout for BK add / read operations bookkeeperClientTimeoutInSeconds=30 # Speculative reads are initiated if a read request doesn't complete within a certain time # Using a value of 0, is disabling the speculative reads bookkeeperClientSpeculativeReadTimeoutInMillis=0 # Enable bookies health check. Bookies that have more than the configured number of failure within # the interval will be quarantined for some time. During this period, new ledgers won't be created # on these bookies bookkeeperClientHealthCheckEnabled=true bookkeeperClientHealthCheckIntervalSeconds=60 bookkeeperClientHealthCheckErrorThresholdPerInterval=5 bookkeeperClientHealthCheckQuarantineTimeInSeconds=1800 # Enable rack-aware bookie selection policy. BK will chose bookies from different racks when # forming a new bookie ensemble bookkeeperClientRackawarePolicyEnabled=true # Enable region-aware bookie selection policy. BK will chose bookies from # different regions and racks when forming a new bookie ensemble # If enabled, the value of bookkeeperClientRackawarePolicyEnabled is ignored bookkeeperClientRegionawarePolicyEnabled=false # Minimum number of racks per write quorum. BK rack-aware bookie selection policy will try to # get bookies from at least 'bookkeeperClientMinNumRacksPerWriteQuorum' racks for a write quorum. bookkeeperClientMinNumRacksPerWriteQuorum=1 # Enforces rack-aware bookie selection policy to pick bookies from 'bookkeeperClientMinNumRacksPerWriteQuorum' # racks for a writeQuorum. # If BK can't find bookie then it would throw BKNotEnoughBookiesException instead of picking random one. bookkeeperClientEnforceMinNumRacksPerWriteQuorum=false # Enable/disable reordering read sequence on reading entries. bookkeeperClientReorderReadSequenceEnabled=false # Enable bookie isolation by specifying a list of bookie groups to choose from. Any bookie # outside the specified groups will not be used by the broker bookkeeperClientIsolationGroups= ### --- Managed Ledger --- ### # Number of bookies to use when creating a ledger managedLedgerDefaultEnsembleSize=1 # Number of copies to store for each message managedLedgerDefaultWriteQuorum=1 # Number of guaranteed copies (acks to wait before write is complete) managedLedgerDefaultAckQuorum=1 # Amount of memory to use for caching data payload in managed ledger. This memory # is allocated from JVM direct memory and it's shared across all the topics # running in the same broker managedLedgerCacheSizeMB=1024 # Threshold to which bring down the cache level when eviction is triggered managedLedgerCacheEvictionWatermark=0.9 # Rate limit the amount of writes generated by consumer acking the messages managedLedgerDefaultMarkDeleteRateLimit=0.1 # Max number of entries to append to a ledger before triggering a rollover # A ledger rollover is triggered after the min rollover time has passed # and one of the following conditions is true: # * The max rollover time has been reached # * The max entries have been written to the ledger # * The max ledger size has been written to the ledger managedLedgerMaxEntriesPerLedger=50000 # Minimum time between ledger rollover for a topic managedLedgerMinLedgerRolloverTimeMinutes=10 # Maximum time before forcing a ledger rollover for a topic managedLedgerMaxLedgerRolloverTimeMinutes=240 # Max number of entries to append to a cursor ledger managedLedgerCursorMaxEntriesPerLedger=50000 # Max time before triggering a rollover on a cursor ledger managedLedgerCursorRolloverTimeInSeconds=14400 ### --- Load balancer --- ### # Enable load balancer loadBalancerEnabled=false # Strategy to assign a new bundle loadBalancerPlacementStrategy=weightedRandomSelection # Percentage of change to trigger load report update loadBalancerReportUpdateThresholdPercentage=10 # maximum interval to update load report loadBalancerReportUpdateMaxIntervalMinutes=15 # Frequency of report to collect loadBalancerHostUsageCheckIntervalMinutes=1 # Load shedding interval. Broker periodically checks whether some traffic should be offload from # some over-loaded broker to other under-loaded brokers loadBalancerSheddingIntervalMinutes=30 # Prevent the same topics to be shed and moved to other broker more than once within this timeframe loadBalancerSheddingGracePeriodMinutes=30 # Usage threshold to determine a broker as under-loaded loadBalancerBrokerUnderloadedThresholdPercentage=1 # Usage threshold to determine a broker as over-loaded loadBalancerBrokerOverloadedThresholdPercentage=85 # Interval to update namespace bundle resource quota loadBalancerResourceQuotaUpdateIntervalMinutes=15 # Usage threshold to determine a broker is having just right level of load loadBalancerBrokerComfortLoadLevelPercentage=65 # enable/disable namespace bundle auto split loadBalancerAutoBundleSplitEnabled=false # interval to detect & split hot namespace bundle loadBalancerNamespaceBundleSplitIntervalMinutes=15 # maximum topics in a bundle, otherwise bundle split will be triggered loadBalancerNamespaceBundleMaxTopics=1000 # maximum sessions (producers + consumers) in a bundle, otherwise bundle split will be triggered loadBalancerNamespaceBundleMaxSessions=1000 # maximum msgRate (in + out) in a bundle, otherwise bundle split will be triggered loadBalancerNamespaceBundleMaxMsgRate=1000 # maximum bandwidth (in + out) in a bundle, otherwise bundle split will be triggered loadBalancerNamespaceBundleMaxBandwidthMbytes=100 # maximum number of bundles in a namespace loadBalancerNamespaceMaximumBundles=128 ### --- Replication --- ### # Enable replication metrics replicationMetricsEnabled=true # Max number of connections to open for each broker in a remote cluster # More connections host-to-host lead to better throughput over high-latency # links. replicationConnectionsPerBroker=16 # Replicator producer queue size replicationProducerQueueSize=1000 # Default message retention time. 0 means retention is disabled. -1 means data is not removed by time quota defaultRetentionTimeInMinutes=0 # Default retention size. 0 means retention is disabled. -1 means data is not removed by size quota defaultRetentionSizeInMB=0 # How often to check whether the connections are still alive keepAliveIntervalSeconds=30 # Enable topic auto creation if new producer or consumer connected (disable auto creation with value false) allowAutoTopicCreation=true # The type of topic that is allowed to be automatically created.(partitioned/non-partitioned) allowAutoTopicCreationType=non-partitioned # The number of partitioned topics that is allowed to be automatically created if allowAutoTopicCreationType is partitioned. defaultNumPartitions=1 ### --- Deprecated config variables --- ### # Deprecated. Use configurationStoreServers globalZookeeperServers= # Deprecated. Use brokerDeleteInactiveTopicsFrequencySeconds brokerServicePurgeInactiveFrequencyInSeconds=60