pax_global_header 0000666 0000000 0000000 00000000064 15205164533 0014516 g ustar 00root root 0000000 0000000 52 comment=270bf720d0ad98f57e387d5ed3178c12bd46cd85
apache-datafusion-java-0.1.0/ 0000775 0000000 0000000 00000000000 15205164533 0016007 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/.asf.yaml 0000664 0000000 0000000 00000003437 15205164533 0017531 0 ustar 00root root 0000000 0000000 # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# This file controls the settings of this repository
#
# See more details at
# https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features
notifications:
commits: commits@datafusion.apache.org
issues: github@datafusion.apache.org
pullrequests: github@datafusion.apache.org
discussions: github@datafusion.apache.org
jira_options: link label worklog
github:
description: "Java bindings for Apache DataFusion"
homepage: https://datafusion.apache.org/
labels:
- apache
- arrow
- datafusion
- java
- jni
- jvm
- query-engine
- sql
enabled_merge_buttons:
squash: true
squash_commit_message: PR_TITLE_AND_DESC
merge: false
rebase: false
features:
issues: true
discussions: true
protected_branches:
main: {}
pull_requests:
allow_update_branch: true
allow_auto_merge: true
del_branch_on_merge: true
# publishes the content of the `asf-site` branch to
# https://datafusion.apache.org/java/
publish:
whoami: asf-site
subdir: java
apache-datafusion-java-0.1.0/.github/ 0000775 0000000 0000000 00000000000 15205164533 0017347 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/.github/ISSUE_TEMPLATE/ 0000775 0000000 0000000 00000000000 15205164533 0021532 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml 0000664 0000000 0000000 00000001342 15205164533 0024425 0 ustar 00root root 0000000 0000000 name: Bug report
description: Create a report to help us improve
type: Bug
labels: bug
body:
- type: textarea
attributes:
label: Describe the bug
description: Describe the bug.
placeholder: >
A clear and concise description of what the bug is.
validations:
required: true
- type: textarea
attributes:
label: To Reproduce
placeholder: >
Steps to reproduce the behavior:
- type: textarea
attributes:
label: Expected behavior
placeholder: >
A clear and concise description of what you expected to happen.
- type: textarea
attributes:
label: Additional context
placeholder: >
Add any other context about the problem here.
apache-datafusion-java-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml 0000664 0000000 0000000 00000002057 15205164533 0025464 0 ustar 00root root 0000000 0000000 name: Feature request
description: Suggest an idea for this project
type: Feature
labels: enhancement
body:
- type: textarea
attributes:
label: Is your feature request related to a problem or challenge?
description: Please describe what you are trying to do.
placeholder: >
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
(This section helps DataFusion developers understand the context and *why* for this feature, in addition to the *what*)
- type: textarea
attributes:
label: Describe the solution you'd like
placeholder: >
A clear and concise description of what you want to happen.
- type: textarea
attributes:
label: Describe alternatives you've considered
placeholder: >
A clear and concise description of any alternative solutions or features you've considered.
- type: textarea
attributes:
label: Additional context
placeholder: >
Add any other context or screenshots about the feature request here.
apache-datafusion-java-0.1.0/.github/pull_request_template.md 0000664 0000000 0000000 00000002600 15205164533 0024306 0 ustar 00root root 0000000 0000000 ## Which issue does this PR close?
- Closes #.
## Rationale for this change
## What changes are included in this PR?
## Are these changes tested?
## Are there any user-facing changes?
apache-datafusion-java-0.1.0/.github/workflows/ 0000775 0000000 0000000 00000000000 15205164533 0021404 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/.github/workflows/build.yml 0000664 0000000 0000000 00000003162 15205164533 0023230 0 ustar 00root root 0000000 0000000 # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Build
on:
push:
branches: [main]
pull_request:
branches: [main]
permissions:
contents: read
jobs:
build:
name: make test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: '17'
cache: maven
- name: Set up Rust toolchain
run: |
rustup update stable
rustup default stable
- name: Cache cargo
uses: actions/cache@v4
with:
path: |
~/.cargo/registry
~/.cargo/git
native/target
key: ${{ runner.os }}-cargo-${{ hashFiles('native/Cargo.lock') }}
restore-keys: ${{ runner.os }}-cargo-
- name: Build native and run JVM tests
run: make test
apache-datafusion-java-0.1.0/.github/workflows/docs.yaml 0000664 0000000 0000000 00000005141 15205164533 0023221 0 ustar 00root root 0000000 0000000 # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Deploy DataFusion Java site
on:
push:
branches:
- main
paths:
- .asf.yaml
- .github/workflows/docs.yaml
- docs/**
pull_request:
branches:
- main
paths:
- .asf.yaml
- .github/workflows/docs.yaml
- docs/**
permissions:
contents: write
jobs:
build-docs:
name: Build docs
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-latest
steps:
- name: Checkout docs sources
uses: actions/checkout@v4
- name: Checkout asf-site branch
if: github.event_name == 'push'
uses: actions/checkout@v4
with:
ref: asf-site
path: asf-site
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Install dependencies
run: |
set -x
python3 -m venv venv
source venv/bin/activate
pip install --upgrade pip
pip install -r docs/requirements.txt
- name: Build docs
run: |
set -x
source venv/bin/activate
cd docs
./build.sh
- name: Copy & push the generated HTML
if: github.event_name == 'push'
run: |
set -x
cd asf-site
rsync \
-a \
--delete \
--exclude '/.git/' \
../docs/build/html/ \
./
cp ../.asf.yaml .
touch .nojekyll
git status --porcelain
if [ -n "$(git status --porcelain)" ]; then
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git add --all
git commit -m "Publish built docs triggered by ${{ github.sha }}"
git push || git push --force
fi
apache-datafusion-java-0.1.0/.gitignore 0000664 0000000 0000000 00000000177 15205164533 0020004 0 ustar 00root root 0000000 0000000 target/
*.class
.idea/
.vscode/
*.iml
.DS_Store
tpch-data/
.claude
docs/superpowers
docs/build/
docs/source/_extra/
docs/venv/
apache-datafusion-java-0.1.0/.mvn/ 0000775 0000000 0000000 00000000000 15205164533 0016665 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/.mvn/wrapper/ 0000775 0000000 0000000 00000000000 15205164533 0020345 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/.mvn/wrapper/maven-wrapper.properties 0000664 0000000 0000000 00000000246 15205164533 0025251 0 ustar 00root root 0000000 0000000 wrapperVersion=3.3.4
distributionType=only-script
distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip
apache-datafusion-java-0.1.0/CONTRIBUTING.md 0000664 0000000 0000000 00000002566 15205164533 0020251 0 ustar 00root root 0000000 0000000
# Contributing to Apache DataFusion Java
Bug reports, design discussion, and patches are welcome. This project follows
the Apache DataFusion contribution model.
- File bugs and feature requests on
[GitHub issues](https://github.com/apache/datafusion-java/issues).
- For larger or design-level discussion, the mailing list is
[dev@datafusion.apache.org](mailto:dev@datafusion.apache.org).
- Please open an issue before sending a PR for any significant change so
the approach can be agreed on first.
For build, test, code style, and version-bump workflows, see the
[contributor guide](docs/source/contributor-guide/index.md).
apache-datafusion-java-0.1.0/LICENSE.txt 0000664 0000000 0000000 00000026715 15205164533 0017645 0 ustar 00root root 0000000 0000000
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
--------------------------------------------------------------------------------
This project includes code from Apache Aurora.
* dev/release/{release,changelog,release-candidate} are based on the scripts from
Apache Aurora
Copyright: 2016 The Apache Software Foundation.
Home page: https://aurora.apache.org/
License: http://www.apache.org/licenses/LICENSE-2.0 apache-datafusion-java-0.1.0/Makefile 0000664 0000000 0000000 00000003055 15205164533 0017452 0 ustar 00root root 0000000 0000000 # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
.PHONY: all native native-runtime-metrics jvm test clean tpch-data
all: native jvm
native:
cd native && cargo build
# Build the native crate with the `runtime-metrics` Cargo feature enabled.
# Requires `--cfg tokio_unstable` because tokio-metrics gates its API there.
# Default `make native` does not pull this in; callers who need
# SessionContext.runtimeStats() pick this target explicitly.
native-runtime-metrics:
cd native && RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics
jvm:
./mvnw package -DskipTests
test: native
./mvnw test
clean:
cd native && cargo clean
./mvnw clean
tpch-data:
@command -v tpchgen-cli >/dev/null || \
(echo "Install: cargo install tpchgen-cli" && exit 1)
mkdir -p tpch-data/sf1
tpchgen-cli -s 1 -f parquet -o tpch-data/sf1
apache-datafusion-java-0.1.0/NOTICE.txt 0000664 0000000 0000000 00000000253 15205164533 0017531 0 ustar 00root root 0000000 0000000 Apache DataFusion Java
Copyright 2026 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
apache-datafusion-java-0.1.0/README.md 0000664 0000000 0000000 00000003641 15205164533 0017272 0 ustar 00root root 0000000 0000000 # Apache DataFusion Java
Java bindings for [Apache DataFusion]. Queries run in native Rust and results
return to the JVM as [Apache Arrow] batches via the Arrow C Data Interface.
[Apache DataFusion]: https://datafusion.apache.org/
[Apache Arrow]: https://arrow.apache.org/
> Early development: no releases yet, API will change. Bug reports and
> contributions welcome.
## Quickstart
```java
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.SessionContext;
try (var allocator = new RootAllocator();
var ctx = new SessionContext()) {
ctx.registerParquet("orders", "/path/to/orders.parquet");
try (DataFrame df = ctx.sql(
"SELECT o_orderpriority, COUNT(*) AS n " +
"FROM orders GROUP BY o_orderpriority");
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
var batch = reader.getVectorSchemaRoot();
// ...
}
}
}
```
`SessionContext` and `DataFrame` are `AutoCloseable` and not thread-safe.
## Documentation
The full documentation lives under [`docs/source/`](docs/source/index.md)
and is built with Sphinx (see [`docs/README.md`](docs/README.md) for the
build steps):
- [User guide](docs/source/user-guide/index.md) — installation, the
DataFrame and SQL APIs, Parquet ingestion.
- [Contributor guide](docs/source/contributor-guide/index.md) — build,
test, code style, and how to bump the DataFusion version.
## Requirements
JDK 17+. Building from source: see
[`docs/source/contributor-guide/development.md`](docs/source/contributor-guide/development.md).
## Contributing
Open an issue to discuss non-trivial changes before sending a PR. See the
[contributor guide](docs/source/contributor-guide/index.md).
## License
Apache License 2.0. See [LICENSE.txt](LICENSE.txt) and [NOTICE.txt](NOTICE.txt).
apache-datafusion-java-0.1.0/core/ 0000775 0000000 0000000 00000000000 15205164533 0016737 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/pom.xml 0000664 0000000 0000000 00000030203 15205164533 0020252 0 ustar 00root root 0000000 0000000
4.0.0org.apache.datafusiondatafusion-java-parent0.1.0datafusion-javajarApache DataFusion Javadebugorg.junit.jupiterjunit-jupitertestorg.apache.arrowarrow-vectororg.apache.arrowarrow-c-dataorg.apache.arrowarrow-memory-nettyruntimecom.google.protobufprotobuf-javaorg.apache.avroavrotestkr.motd.mavenos-maven-plugin1.7.1org.apache.maven.pluginsmaven-surefire-plugin--add-opens=java.base/java.nio=ALL-UNNAMEDorg.apache.maven.pluginsmaven-antrun-plugincopy-native-libprocess-classesruncom.googlecode.maven-download-plugindownload-maven-plugin1.9.0fetch-datafusion-common-protogenerate-sourceswgethttps://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto-common/proto/datafusion_common.proto${project.build.directory}/proto/datafusion/proto-common/protodatafusion_common.protod6f3368372ea277cc23e26f196994b81616d38599357bb374cbd7eb1760e649a789e4c133d86a395ac701049a500348da2ec039d3f978ac5d8112c2876dded1ffetch-datafusion-protogenerate-sourceswgethttps://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto/proto/datafusion.proto${project.build.directory}/proto/datafusion/proto/protodatafusion.protoc3d162b8e2a418e03f74caceaccfd934af89bb95a12ede13d4cc1701d24c734d74b1e96372142b173db05938dab7f965ad60d476363308c441677a63ea5fbcf7org.xolstice.maven.pluginsprotobuf-maven-plugin0.6.1com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}${project.build.directory}/protocompile-upstreamcompilecompile-localcompile${maven.multiModuleProjectDirectory}/protofalseorg.apache.maven.pluginsmaven-javadoc-plugin17truefalseApache DataFusion Java ${project.version}Apache DataFusion Java ${project.version}
https://docs.oracle.com/en/java/javase/17/docs/api/
datafusion_common:org.apache.datafusion.protobufnative-linux-amd64unixlinuxamd64linuxamd64libdatafusion_jni.sonative-linux-aarch64unixlinuxaarch64linuxaarch64libdatafusion_jni.sonative-mac-x86_64macx86_64darwinx86_64libdatafusion_jni.dylibnative-mac-amd64macamd64darwinx86_64libdatafusion_jni.dylibnative-mac-aarch64macaarch64darwinaarch64libdatafusion_jni.dylibnative-windows-amd64windowsamd64windowsamd64datafusion_jni.dll
apache-datafusion-java-0.1.0/core/src/ 0000775 0000000 0000000 00000000000 15205164533 0017526 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/main/ 0000775 0000000 0000000 00000000000 15205164533 0020452 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/main/java/ 0000775 0000000 0000000 00000000000 15205164533 0021373 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/main/java/org/ 0000775 0000000 0000000 00000000000 15205164533 0022162 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/main/java/org/apache/ 0000775 0000000 0000000 00000000000 15205164533 0023403 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ 0000775 0000000 0000000 00000000000 15205164533 0025540 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ArrowReadOptions.java 0000664 0000000 0000000 00000004237 15205164533 0031653 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.protobuf.ArrowReadOptionsProto;
/**
* Configuration knobs for Arrow IPC sources passed to {@link SessionContext#registerArrow(String,
* String, ArrowReadOptions)} and {@link SessionContext#readArrow(String, ArrowReadOptions)}.
*
*
Mirrors the subset of DataFusion's {@code ArrowReadOptions} that maps onto the Java surface
* today: {@code fileExtension} (default {@code ".arrow"}) and an explicit Arrow {@code schema} that
* bypasses on-read schema inference. {@code tablePartitionCols} is intentionally deferred --
* neither Parquet nor CSV expose Hive-style partitioning on the Java side yet.
*
*
Arrow IPC files carry their own body compression (LZ4_FRAME / ZSTD per-buffer) inside the file
* format itself, so unlike CSV / NDJSON there is no {@code FileCompressionType} setter.
*/
public final class ArrowReadOptions {
private String fileExtension = ".arrow";
private Schema schema;
public ArrowReadOptions fileExtension(String ext) {
this.fileExtension = ext;
return this;
}
public ArrowReadOptions schema(Schema schema) {
this.schema = schema;
return this;
}
byte[] toBytes() {
return ArrowReadOptionsProto.newBuilder().setFileExtension(fileExtension).build().toByteArray();
}
Schema schema() {
return schema;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/AvroReadOptions.java 0000664 0000000 0000000 00000004254 15205164533 0031467 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.protobuf.AvroReadOptionsProto;
/**
* Configuration knobs for Avro sources passed to {@link SessionContext#registerAvro(String, String,
* AvroReadOptions)} and {@link SessionContext#readAvro(String, AvroReadOptions)}.
*
*
Mirrors the subset of DataFusion's {@code AvroReadOptions} that maps onto the Java surface
* today: {@code fileExtension} (default {@code ".avro"}) and an explicit Arrow {@code schema} that
* bypasses on-read schema inference. {@code tablePartitionCols} is intentionally deferred -- no
* other Java reader exposes Hive-style partitioning yet.
*
*
Avro carries its own per-block compression (snappy, deflate, bzip2, xz, zstandard) inside the
* object container itself, negotiated when the file is written, so unlike CSV / NDJSON there is no
* {@code FileCompressionType} setter.
*/
public final class AvroReadOptions {
private String fileExtension = ".avro";
private Schema schema;
public AvroReadOptions fileExtension(String ext) {
this.fileExtension = ext;
return this;
}
public AvroReadOptions schema(Schema schema) {
this.schema = schema;
return this;
}
byte[] toBytes() {
return AvroReadOptionsProto.newBuilder().setFileExtension(fileExtension).build().toByteArray();
}
Schema schema() {
return schema;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/CacheManagerOptions.java 0000664 0000000 0000000 00000014675 15205164533 0032272 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.time.Duration;
import org.apache.datafusion.protobuf.CacheManagerOptionsProto;
import org.apache.datafusion.protobuf.ListFilesCacheOptionsProto;
/**
* Configuration for DataFusion's built-in {@code CacheManager}. Pass an instance to {@link
* SessionContextBuilder#cacheManager(CacheManagerOptions)} to turn on any of the three
* upstream-provided caches at session construction time.
*
*
The three caches are independent; calling one setter does not affect the others. A setter that
* is never called leaves the upstream default in place — so existing callers that don't touch this
* builder see no behavioral change.
*
*
*
{@link Builder#fileMetadataCache(long)} — caches file-embedded metadata (e.g. parquet
* footers / page metadata). Backed by {@code DefaultFilesMetadataCache} with the supplied
* byte cap.
*
{@link Builder#listFilesCache(long, Duration)} — caches results of object-store {@code
* list} operations. Backed by {@code DefaultListFilesCache} with the supplied byte cap and
* optional TTL ({@code null} = infinite).
*
{@link Builder#fileStatisticsCache(boolean)} — caches per-file row counts and column
* statistics. Backed by {@code DefaultFileStatisticsCache}; no bytewise cap exists upstream.
*
*
*
This v1 surface configures the built-in cache implementations. Plugging custom Java
* cache implementations through a JNI upcall path is intentionally out of scope — every cache
* lookup is a hot path during scans, and routing them through Java would defeat the cache.
*/
public final class CacheManagerOptions {
private final Long fileMetadataCacheMaxBytes;
private final ListFilesCacheConfig listFilesCache;
private final Boolean fileStatisticsCacheEnabled;
private CacheManagerOptions(Builder b) {
this.fileMetadataCacheMaxBytes = b.fileMetadataCacheMaxBytes;
this.listFilesCache = b.listFilesCache;
this.fileStatisticsCacheEnabled = b.fileStatisticsCacheEnabled;
}
/** Begin building a {@link CacheManagerOptions} instance. */
public static Builder builder() {
return new Builder();
}
CacheManagerOptionsProto toProto() {
CacheManagerOptionsProto.Builder b = CacheManagerOptionsProto.newBuilder();
if (fileMetadataCacheMaxBytes != null) {
b.setFileMetadataCacheMaxBytes(fileMetadataCacheMaxBytes);
}
if (listFilesCache != null) {
ListFilesCacheOptionsProto.Builder lb = ListFilesCacheOptionsProto.newBuilder();
if (listFilesCache.maxBytes != null) {
lb.setMaxBytes(listFilesCache.maxBytes);
}
if (listFilesCache.ttlMillis != null) {
lb.setTtlMillis(listFilesCache.ttlMillis);
}
b.setListFilesCache(lb.build());
}
if (fileStatisticsCacheEnabled != null) {
b.setFileStatisticsCacheEnabled(fileStatisticsCacheEnabled);
}
return b.build();
}
/** Internal carrier — `null`-vs-set is the source of truth for "user called this setter". */
private static final class ListFilesCacheConfig {
final Long maxBytes;
final Long ttlMillis;
ListFilesCacheConfig(Long maxBytes, Long ttlMillis) {
this.maxBytes = maxBytes;
this.ttlMillis = ttlMillis;
}
}
/** Builder for {@link CacheManagerOptions}. */
public static final class Builder {
private Long fileMetadataCacheMaxBytes;
private ListFilesCacheConfig listFilesCache;
private Boolean fileStatisticsCacheEnabled;
private Builder() {}
/**
* Enable the file-embedded metadata cache (parquet footers, page metadata) with the given byte
* cap. The cap is the budget the upstream {@code DefaultFilesMetadataCache} uses to evict
* entries; {@code 0} is legal and means "construct the cache but with a 0-byte budget"
* (effectively disabled but observable in stats).
*
* @throws IllegalArgumentException if {@code maxBytes} is negative.
*/
public Builder fileMetadataCache(long maxBytes) {
if (maxBytes < 0) {
throw new IllegalArgumentException(
"fileMetadataCache maxBytes must be non-negative, got " + maxBytes);
}
this.fileMetadataCacheMaxBytes = maxBytes;
return this;
}
/**
* Enable the list-files cache with the given byte cap and TTL. Pass {@code null} for {@code
* ttl} to use upstream's "no expiration" semantics (entries are evicted only by capacity
* pressure).
*
* @throws IllegalArgumentException if {@code maxBytes} is negative or {@code ttl} is negative.
*/
public Builder listFilesCache(long maxBytes, Duration ttl) {
if (maxBytes < 0) {
throw new IllegalArgumentException(
"listFilesCache maxBytes must be non-negative, got " + maxBytes);
}
Long ttlMillis = null;
if (ttl != null) {
if (ttl.isNegative()) {
throw new IllegalArgumentException("listFilesCache ttl must be non-negative, got " + ttl);
}
ttlMillis = ttl.toMillis();
}
this.listFilesCache = new ListFilesCacheConfig(maxBytes, ttlMillis);
return this;
}
/**
* Enable the file-statistics cache (per-file row counts / column statistics). When {@code
* enabled} is {@code true}, the upstream {@code DefaultFileStatisticsCache} is installed. When
* {@code false}, the slot is explicitly set to disabled in the wire format — the same end-state
* as never calling this setter, but distinguishable for testing.
*/
public Builder fileStatisticsCache(boolean enabled) {
this.fileStatisticsCacheEnabled = enabled;
return this;
}
public CacheManagerOptions build() {
return new CacheManagerOptions(this);
}
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ColumnarValue.java 0000664 0000000 0000000 00000005711 15205164533 0031164 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.util.Objects;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.types.pojo.ArrowType;
/**
* The value of a scalar UDF argument or result: either a per-row {@link Array} of length {@code
* rowCount}, or a {@link Scalar} (length-1 vector) that the framework broadcasts.
*
*
Mirrors DataFusion's {@code datafusion::logical_expr::ColumnarValue} enum. Use {@link
* #array(FieldVector)} and {@link #scalar(FieldVector)} factories rather than constructing the
* records directly so the length invariants are enforced consistently.
*/
public sealed interface ColumnarValue permits ColumnarValue.Array, ColumnarValue.Scalar {
/** The underlying Arrow vector. For {@link Scalar} this vector has {@code valueCount == 1}. */
FieldVector vector();
/** Convenience: the vector's declared Arrow type. */
default ArrowType dataType() {
return vector().getField().getType();
}
/** Wrap an arbitrary-length vector as an {@link Array}. */
static ColumnarValue array(FieldVector vector) {
return new Array(Objects.requireNonNull(vector, "vector"));
}
/**
* Wrap a length-1 vector as a {@link Scalar}.
*
* @throws IllegalArgumentException if {@code vector.getValueCount() != 1}
*/
static ColumnarValue scalar(FieldVector vector) {
Objects.requireNonNull(vector, "vector");
if (vector.getValueCount() != 1) {
throw new IllegalArgumentException(
"Scalar vector must have valueCount == 1, got " + vector.getValueCount());
}
return new Scalar(vector);
}
/** Per-row Arrow vector of length equal to the batch row count. */
record Array(FieldVector vector) implements ColumnarValue {
public Array {
Objects.requireNonNull(vector, "vector");
}
}
/** Length-1 Arrow vector representing a single value broadcast across all rows. */
record Scalar(FieldVector vector) implements ColumnarValue {
public Scalar {
Objects.requireNonNull(vector, "vector");
if (vector.getValueCount() != 1) {
throw new IllegalArgumentException(
"Scalar vector must have valueCount == 1, got " + vector.getValueCount());
}
}
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ConfigurationException.java 0000664 0000000 0000000 00000002360 15205164533 0033072 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Invalid or unrecognised DataFusion configuration option. Surfaces upstream {@code
* DataFusionError::Configuration}.
*/
public class ConfigurationException extends DataFusionException {
private static final long serialVersionUID = 1L;
public ConfigurationException(String message) {
super(message);
}
public ConfigurationException(String message, Throwable cause) {
super(message, cause);
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/CsvReadOptions.java 0000664 0000000 0000000 00000007717 15205164533 0031322 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import org.apache.arrow.vector.types.pojo.Schema;
/**
* Configuration knobs for CSV sources passed to {@link SessionContext#registerCsv(String, String,
* CsvReadOptions)} and {@link SessionContext#readCsv(String, CsvReadOptions)}.
*
*
Mirrors a subset of DataFusion's {@code CsvReadOptions}. All setters return {@code this} for
* fluent chaining. Defaults match the Rust struct: {@code hasHeader = true}, {@code delimiter =
* ','}, {@code quote = '"'}, {@code fileExtension = ".csv"}, {@code fileCompressionType =
* UNCOMPRESSED}, all other fields {@code null} (meaning the DataFusion default is used, or the
* schema is inferred from the file).
*/
public final class CsvReadOptions {
private boolean hasHeader = true;
private byte delimiter = (byte) ',';
private byte quote = (byte) '"';
private Byte terminator;
private Byte escape;
private Byte comment;
private Boolean newlinesInValues;
private Long schemaInferMaxRecords;
private String fileExtension = ".csv";
private FileCompressionType fileCompressionType = FileCompressionType.UNCOMPRESSED;
private Schema schema;
public CsvReadOptions hasHeader(boolean v) {
this.hasHeader = v;
return this;
}
public CsvReadOptions delimiter(byte b) {
this.delimiter = b;
return this;
}
public CsvReadOptions quote(byte b) {
this.quote = b;
return this;
}
public CsvReadOptions terminator(byte b) {
this.terminator = b;
return this;
}
public CsvReadOptions escape(byte b) {
this.escape = b;
return this;
}
public CsvReadOptions comment(byte b) {
this.comment = b;
return this;
}
public CsvReadOptions newlinesInValues(boolean v) {
this.newlinesInValues = v;
return this;
}
public CsvReadOptions schemaInferMaxRecords(long n) {
if (n < 0) {
throw new IllegalArgumentException("schemaInferMaxRecords must be non-negative, got " + n);
}
this.schemaInferMaxRecords = n;
return this;
}
public CsvReadOptions fileExtension(String ext) {
this.fileExtension = ext;
return this;
}
public CsvReadOptions fileCompressionType(FileCompressionType t) {
this.fileCompressionType = t;
return this;
}
public CsvReadOptions schema(Schema schema) {
this.schema = schema;
return this;
}
byte[] toBytes() {
org.apache.datafusion.protobuf.CsvReadOptionsProto.Builder b =
org.apache.datafusion.protobuf.CsvReadOptionsProto.newBuilder()
.setHasHeader(hasHeader)
.setDelimiter(delimiter & 0xFF)
.setQuote(quote & 0xFF)
.setFileExtension(fileExtension)
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
if (terminator != null) {
b.setTerminator(terminator & 0xFF);
}
if (escape != null) {
b.setEscape(escape & 0xFF);
}
if (comment != null) {
b.setComment(comment & 0xFF);
}
if (newlinesInValues != null) {
b.setNewlinesInValues(newlinesInValues);
}
if (schemaInferMaxRecords != null) {
b.setSchemaInferMaxRecords(schemaInferMaxRecords);
}
return b.build().toByteArray();
}
Schema schema() {
return schema;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/CsvWriteOptions.java 0000664 0000000 0000000 00000011131 15205164533 0031522 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.util.ArrayList;
import java.util.List;
/**
* Configuration knobs for writing CSV, passed to {@link DataFrame#writeCsv(String,
* CsvWriteOptions)}.
*
*
Mirrors a subset of DataFusion's {@code DataFrameWriteOptions} and the writer-side {@code
* CsvOptions}. All setters return {@code this} for fluent chaining. Defaults: every field {@code
* null} or empty (meaning the DataFusion default is used).
*
*
Path semantics: when {@link #singleFileOutput(boolean)} is {@code true}, the path passed to
* {@code writeCsv} is the literal output filename. When left unset (the default) and there are no
* partition columns, the path is treated as a directory that DataFusion populates with one or more
* part-files.
*
*
Compression reuses {@link FileCompressionType} -- both the read and write sides accept the
* same codec set ({@code UNCOMPRESSED}, {@code GZIP}, {@code BZIP2}, {@code XZ}, {@code ZSTD}).
*/
public final class CsvWriteOptions {
private Boolean singleFileOutput;
private final List partitionCols = new ArrayList<>();
private Boolean hasHeader;
private Byte delimiter;
private Byte quote;
private Byte escape;
private String nullValue;
private FileCompressionType fileCompressionType;
/**
* When {@code true}, write to a single file at the supplied path. When left unset (the default)
* and no partition columns are configured, the path is treated as a directory and DataFusion
* writes one or more part-files.
*/
public CsvWriteOptions singleFileOutput(boolean v) {
this.singleFileOutput = v;
return this;
}
/**
* Hive-style partition columns. Each column listed here is removed from the data rows and encoded
* into the directory layout (one subdirectory per distinct value). Mutually exclusive with {@link
* #singleFileOutput(boolean)} -- DataFusion rejects the combination at write time.
*/
public CsvWriteOptions partitionCols(String... cols) {
this.partitionCols.clear();
for (String c : cols) {
this.partitionCols.add(c);
}
return this;
}
/** Whether to write a header row. Defaults to DataFusion's setting (typically {@code true}). */
public CsvWriteOptions hasHeader(boolean v) {
this.hasHeader = v;
return this;
}
/** Field delimiter byte. Defaults to {@code ','}. */
public CsvWriteOptions delimiter(byte b) {
this.delimiter = b;
return this;
}
/** Quote character byte. Defaults to {@code '"'}. */
public CsvWriteOptions quote(byte b) {
this.quote = b;
return this;
}
/** Escape character byte. Defaults to none. */
public CsvWriteOptions escape(byte b) {
this.escape = b;
return this;
}
/** String to write for SQL NULL values. Defaults to the empty string. */
public CsvWriteOptions nullValue(String s) {
this.nullValue = s;
return this;
}
/** Output compression codec. Defaults to uncompressed. */
public CsvWriteOptions fileCompressionType(FileCompressionType t) {
this.fileCompressionType = t;
return this;
}
byte[] toBytes() {
org.apache.datafusion.protobuf.CsvWriteOptionsProto.Builder b =
org.apache.datafusion.protobuf.CsvWriteOptionsProto.newBuilder();
if (singleFileOutput != null) {
b.setSingleFileOutput(singleFileOutput);
}
b.addAllPartitionCols(partitionCols);
if (hasHeader != null) {
b.setHasHeader(hasHeader);
}
if (delimiter != null) {
b.setDelimiter(delimiter & 0xFF);
}
if (quote != null) {
b.setQuote(quote & 0xFF);
}
if (escape != null) {
b.setEscape(escape & 0xFF);
}
if (nullValue != null) {
b.setNullValue(nullValue);
}
if (fileCompressionType != null) {
b.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
}
return b.build().toByteArray();
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/DataFrame.java 0000664 0000000 0000000 00000070122 15205164533 0030231 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.channels.Channels;
import org.apache.arrow.c.ArrowArrayStream;
import org.apache.arrow.c.Data;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.ipc.ReadChannel;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Schema;
/**
* A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
* by {@link SessionContext#sql(String)} or other planning entry points and executed by either
* {@link #collect} (materializes every batch on the native heap before returning) or {@link
* #executeStream} (yields one batch at a time as Java drains the reader).
*
*
Instances are not thread-safe and must be closed. Both {@link #collect} and
* {@link #executeStream} consume the DataFrame: a successfully consumed DataFrame cannot be
* consumed again by either method (or by other executors such as {@link #count}), and {@link
* #close()} on an already-consumed instance is a no-op.
*/
public final class DataFrame implements AutoCloseable {
static {
NativeLibraryLoader.loadLibrary();
}
private long nativeHandle;
DataFrame(long nativeHandle) {
if (nativeHandle == 0) {
throw new IllegalArgumentException("DataFrame native handle is null");
}
this.nativeHandle = nativeHandle;
}
/**
* Execute the plan and return its record batches as an {@link ArrowReader}.
*
*
Consumes this DataFrame: the native plan is released as soon as the stream is established.
* The caller is responsible for closing the returned reader, and the supplied allocator must
* outlive it.
*
*
This method materializes every batch on the native heap before the first batch crosses the
* FFI boundary, which can OOM the Rust side for unbounded or very large result sets. Prefer
* {@link #executeStream(BufferAllocator)} for analytics-scale queries.
*/
public ArrowReader collect(BufferAllocator allocator) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator);
long handle = nativeHandle;
nativeHandle = 0;
try {
collectDataFrame(handle, stream.memoryAddress());
return Data.importArrayStream(allocator, stream);
} catch (Throwable e) {
stream.close();
throw e;
}
}
/**
* Execute the plan and return its record batches as a streaming {@link ArrowReader}. Each call to
* {@link ArrowReader#loadNextBatch} drives one async {@code stream.next()} on the native side, so
* memory pressure stays bounded by the executor pipeline plus one in-flight batch instead of the
* full result set.
*
*
Consumes this DataFrame with the same lifecycle rules as {@link #collect(BufferAllocator)}:
* the native plan is released as soon as the stream is established, the caller closes the
* returned reader, and the supplied allocator must outlive it.
*
*
For result sets that fit comfortably in native memory and are read in their entirety, {@link
* #collect(BufferAllocator)} remains a reasonable choice. For TB-scale or unbounded result sets,
* use this method.
*/
public ArrowReader executeStream(BufferAllocator allocator) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator);
long handle = nativeHandle;
nativeHandle = 0;
try {
executeStreamDataFrame(handle, stream.memoryAddress());
return Data.importArrayStream(allocator, stream);
} catch (Throwable e) {
stream.close();
throw e;
}
}
/**
* Return the Arrow {@link Schema} of this DataFrame's output. Non-consuming: the receiver remains
* usable and must still be closed independently. Schema inspection does not execute the plan.
*
*
The schema is transferred via Arrow IPC; no {@link BufferAllocator} is required because a
* schema carries no buffer data.
*/
public Schema schema() {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
byte[] ipcBytes = schemaIpc(nativeHandle);
try {
return MessageSerializer.deserializeSchema(
new ReadChannel(Channels.newChannel(new ByteArrayInputStream(ipcBytes))));
} catch (IOException e) {
throw new RuntimeException("Failed to deserialize IPC schema", e);
}
}
/**
* Return a new DataFrame whose rows describe the plan that would execute this DataFrame.
* Non-consuming: the receiver remains usable and must still be closed independently.
*
*
With {@code verbose=false} and {@code analyze=false} (the cheap, lazy variant), the result
* contains the logical plan only. {@code verbose=true} adds optimised-plan and physical-plan
* rows; {@code analyze=true} runs the plan and attaches per-operator metrics. Render via {@link
* #show()} or {@link #collect(BufferAllocator)}.
*/
public DataFrame explain(boolean verbose, boolean analyze) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return new DataFrame(explainPlan(nativeHandle, verbose, analyze));
}
/**
* Materialise this DataFrame into an in-memory table and return a new DataFrame that scans it.
* Non-consuming: the receiver remains usable and must still be closed independently.
*
*
Executes the plan eagerly: the entire result set is held in native memory until the returned
* DataFrame is closed. Suitable for intermediate results that will be reused across multiple
* downstream queries.
*
* @throws RuntimeException if execution fails.
*/
public DataFrame cache() {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return new DataFrame(cachePlan(nativeHandle));
}
/**
* Compute summary statistics (count, null_count, mean, std, min, max, median) over this
* DataFrame's columns and return them as a new DataFrame. Non-consuming: the receiver remains
* usable and must still be closed independently.
*
*
Executes the plan: DataFusion runs seven aggregate sub-plans against this DataFrame to build
* the summary table. Numeric columns receive every statistic; non-numeric columns receive {@code
* count} / {@code null_count} / {@code min} / {@code max} where applicable.
*
* @throws RuntimeException if execution fails.
*/
public DataFrame describe() {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return new DataFrame(describePlan(nativeHandle));
}
/** Execute the plan and return the number of rows. */
public long count() {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return countRows(nativeHandle);
}
/** Execute the plan and print formatted batches to native stdout. */
public void show() {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
showDataFrame(nativeHandle);
}
/** Execute the plan and print the first {@code limit} rows to native stdout. */
public void show(int limit) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
showDataFrameWithLimit(nativeHandle, limit);
}
/**
* Project the listed columns into a new DataFrame. The receiver remains usable and must still be
* closed independently.
*/
public DataFrame select(String... columnNames) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return new DataFrame(selectColumns(nativeHandle, columnNames));
}
/**
* Apply a SQL predicate to produce a filtered DataFrame. The predicate is parsed against this
* DataFrame's own schema. The receiver remains usable and must still be closed independently.
*/
public DataFrame filter(String predicate) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return new DataFrame(filterRows(nativeHandle, predicate));
}
/**
* Take the first {@code fetch} rows. Equivalent to {@link #limit(int, int)} with {@code skip =
* 0}. The receiver remains usable and must still be closed independently.
*/
public DataFrame limit(int fetch) {
return limit(0, fetch);
}
/**
* Skip {@code skip} rows, then take the next {@code fetch} rows. Both arguments must be
* non-negative. The receiver remains usable and must still be closed independently.
*/
public DataFrame limit(int skip, int fetch) {
if (skip < 0) {
throw new IllegalArgumentException("skip must be non-negative, was " + skip);
}
if (fetch < 0) {
throw new IllegalArgumentException("fetch must be non-negative, was " + fetch);
}
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return new DataFrame(limitRows(nativeHandle, skip, fetch));
}
/**
* Deduplicate rows across all columns. The receiver remains usable and must still be closed
* independently.
*/
public DataFrame distinct() {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return new DataFrame(distinctRows(nativeHandle));
}
/**
* Drop the named columns. The inverse of {@link #select(String...)}. The receiver remains usable
* and must still be closed independently.
*/
public DataFrame dropColumns(String... columnNames) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return new DataFrame(dropColumns(nativeHandle, columnNames));
}
/** Rename a column. The receiver remains usable and must still be closed independently. */
public DataFrame withColumnRenamed(String oldName, String newName) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
return new DataFrame(renameColumn(nativeHandle, oldName, newName));
}
/**
* Add a column to this DataFrame computed from a SQL expression. If a column with the given name
* already exists, it is replaced in place; otherwise the new column is appended. The expression
* is parsed against this DataFrame's own schema, matching the convention used by {@link
* #filter(String)}. The receiver remains usable and must still be closed independently.
*
* @throws IllegalArgumentException if {@code name} or {@code expr} is {@code null}.
*/
public DataFrame withColumn(String name, String expr) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
if (name == null) {
throw new IllegalArgumentException("withColumn name must be non-null");
}
if (expr == null) {
throw new IllegalArgumentException("withColumn expr must be non-null");
}
return new DataFrame(withColumnExpr(nativeHandle, name, expr));
}
/**
* Expand list or struct columns into rows or fields, with default {@link UnnestOptions} (i.e.
* {@code preserveNulls = true}). The receiver remains usable and must still be closed
* independently.
*/
public DataFrame unnestColumns(String... columns) {
return unnestColumns(new UnnestOptions(), columns);
}
/**
* Expand list or struct columns into rows or fields with the supplied {@link UnnestOptions}. The
* receiver remains usable and must still be closed independently.
*
* @throws IllegalArgumentException if {@code options} or {@code columns} is {@code null}.
*/
public DataFrame unnestColumns(UnnestOptions options, String... columns) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
if (options == null) {
throw new IllegalArgumentException("unnestColumns options must be non-null");
}
if (columns == null) {
throw new IllegalArgumentException("unnestColumns columns must be non-null");
}
return new DataFrame(unnestColumns(nativeHandle, columns, options.preserveNulls()));
}
/**
* Order the rows by the supplied sort keys. Each {@link SortExpr} names a column and a direction
* ({@link SortExpr#asc(String)} / {@link SortExpr#desc(String)}); call {@link
* SortExpr#nullsFirst(boolean)} to override null placement.
*
*
An empty {@code exprs} array is a no-op (matches DataFusion's {@code sort(vec![])}). The
* receiver remains usable and must still be closed independently.
*
* @throws IllegalArgumentException if {@code exprs} or any element is {@code null}.
* @throws RuntimeException if a sort column does not exist in this DataFrame's schema.
*/
public DataFrame sort(SortExpr... exprs) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
if (exprs == null) {
throw new IllegalArgumentException("sort exprs must be non-null");
}
String[] columns = new String[exprs.length];
boolean[] ascending = new boolean[exprs.length];
boolean[] nullsFirst = new boolean[exprs.length];
for (int i = 0; i < exprs.length; i++) {
SortExpr e = exprs[i];
if (e == null) {
throw new IllegalArgumentException("sort exprs[" + i + "] must be non-null");
}
columns[i] = e.column();
ascending[i] = e.ascending();
nullsFirst[i] = e.nullsFirst();
}
return new DataFrame(sortRows(nativeHandle, columns, ascending, nullsFirst));
}
/**
* Repartition this DataFrame using a round-robin scheme across {@code numPartitions} output
* partitions. The receiver remains usable and must still be closed independently.
*
* @throws IllegalArgumentException if {@code numPartitions <= 0}.
* @throws RuntimeException if the underlying repartition plan rejects the request.
*/
public DataFrame repartitionRoundRobin(int numPartitions) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
if (numPartitions <= 0) {
throw new IllegalArgumentException("numPartitions must be positive, was " + numPartitions);
}
return new DataFrame(repartitionRoundRobinRows(nativeHandle, numPartitions));
}
/**
* Repartition this DataFrame by hashing the named columns into {@code numPartitions} output
* partitions. v1 supports column-name keys only; expression keys are deferred until the Java
* binding gains an {@code Expr} builder. The receiver remains usable and must still be closed
* independently.
*
* @throws IllegalArgumentException if {@code numPartitions <= 0}, {@code columns} is {@code null}
* or empty, or any element of {@code columns} is {@code null}.
* @throws RuntimeException if a partition column does not exist in this DataFrame's schema.
*/
public DataFrame repartitionHash(int numPartitions, String... columns) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
if (numPartitions <= 0) {
throw new IllegalArgumentException("numPartitions must be positive, was " + numPartitions);
}
if (columns == null) {
throw new IllegalArgumentException("repartitionHash columns must be non-null");
}
if (columns.length == 0) {
throw new IllegalArgumentException("repartitionHash requires at least one column");
}
for (int i = 0; i < columns.length; i++) {
if (columns[i] == null) {
throw new IllegalArgumentException("repartitionHash columns[" + i + "] must be non-null");
}
}
return new DataFrame(repartitionHashRows(nativeHandle, numPartitions, columns));
}
/**
* Equi-join this DataFrame with {@code right} on the named columns, using the given {@link
* JoinType}. The receiver and {@code right} both remain usable and must still be closed
* independently.
*
*
Equivalent to SQL {@code left JOIN right ON l.leftCols[0] = r.rightCols[0] AND ...}.
* {@code leftCols} and {@code rightCols} must have the same length.
*
* @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
* rightCols.length}.
* @throws IllegalStateException if either DataFrame is closed or already collected.
* @throws RuntimeException if join planning fails (column collision in the combined schema,
* unknown column names, etc.).
*/
public DataFrame join(DataFrame right, JoinType type, String[] leftCols, String[] rightCols) {
checkJoinArgs(right, type, leftCols, rightCols);
return new DataFrame(
joinDataFrame(nativeHandle, right.nativeHandle, type.code(), leftCols, rightCols, null));
}
/**
* Equi-join this DataFrame with {@code right}, restricting the result with a residual SQL filter
* parsed against the combined schema (left columns followed by right columns; columns
* may be qualified with the relation alias when ambiguous). The receiver and {@code right} both
* remain usable and must still be closed independently.
*
*
For outer joins, the filter is applied only to matched rows; unmatched rows are passed
* through with nulls on the unmatched side, matching DataFusion's semantics.
*
* @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
* rightCols.length}.
* @throws IllegalStateException if either DataFrame is closed or already collected.
* @throws RuntimeException if join planning or filter parsing fails.
*/
public DataFrame join(
DataFrame right, JoinType type, String[] leftCols, String[] rightCols, String filter) {
checkJoinArgs(right, type, leftCols, rightCols);
if (filter == null) {
throw new IllegalArgumentException("join filter must be non-null");
}
return new DataFrame(
joinDataFrame(nativeHandle, right.nativeHandle, type.code(), leftCols, rightCols, filter));
}
/**
* Join this DataFrame with {@code right} using arbitrary SQL predicates parsed against the
* combined schema. Each predicate is parsed independently and the join evaluates their
* conjunction. Predicates may reference columns from either side and may be qualified with the
* relation alias when ambiguous (e.g. {@code "left.x = right.x"}). The receiver and {@code right}
* both remain usable and must still be closed independently.
*
*
DataFusion's optimiser identifies and rewrites equality predicates into hash-join keys
* automatically, so {@code joinOn(right, INNER, "l.id = r.id")} plans equivalently to {@link
* #join(DataFrame, JoinType, String[], String[])} with a single key. Use {@code joinOn} when the
* predicate is not a simple equality, e.g. inequality joins or range conditions.
*
* @throws IllegalArgumentException if {@code right} or {@code type} is {@code null}, or {@code
* predicates} is {@code null} or empty, or any predicate is {@code null}.
* @throws IllegalStateException if either DataFrame is closed or already collected.
* @throws RuntimeException if predicate parsing or join planning fails.
*/
public DataFrame joinOn(DataFrame right, JoinType type, String... predicates) {
if (right == null) {
throw new IllegalArgumentException("joinOn right must be non-null");
}
if (type == null) {
throw new IllegalArgumentException("joinOn type must be non-null");
}
if (predicates == null || predicates.length == 0) {
throw new IllegalArgumentException("joinOn predicates must be non-null and non-empty");
}
for (String p : predicates) {
if (p == null) {
throw new IllegalArgumentException("joinOn predicates must not contain null");
}
}
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
if (right.nativeHandle == 0) {
throw new IllegalStateException("right DataFrame is closed or already collected");
}
return new DataFrame(
joinOnDataFrame(nativeHandle, right.nativeHandle, type.code(), predicates));
}
private void checkJoinArgs(
DataFrame right, JoinType type, String[] leftCols, String[] rightCols) {
if (right == null) {
throw new IllegalArgumentException("join right must be non-null");
}
if (type == null) {
throw new IllegalArgumentException("join type must be non-null");
}
if (leftCols == null) {
throw new IllegalArgumentException("join leftCols must be non-null");
}
if (rightCols == null) {
throw new IllegalArgumentException("join rightCols must be non-null");
}
if (leftCols.length != rightCols.length) {
throw new IllegalArgumentException(
"join leftCols and rightCols must have the same length, got "
+ leftCols.length
+ " and "
+ rightCols.length);
}
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
if (right.nativeHandle == 0) {
throw new IllegalStateException("right DataFrame is closed or already collected");
}
}
/**
* Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
* unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
* remains usable and must still be closed independently.
*
* @throws RuntimeException if the write fails.
*/
public void writeParquet(String path) {
writeParquet(path, new ParquetWriteOptions());
}
/**
* Materialize this DataFrame as Parquet at {@code path} with the supplied {@link
* ParquetWriteOptions}. The receiver remains usable and must still be closed independently.
*
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
* etc.).
*/
public void writeParquet(String path, ParquetWriteOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
writeParquetWithOptions(
nativeHandle,
path,
options.compression(),
options.singleFileOutput() != null,
options.singleFileOutput() != null && options.singleFileOutput());
}
/**
* Materialize this DataFrame as CSV at {@code path}. The path is treated as a directory unless
* overridden via {@link CsvWriteOptions#singleFileOutput(boolean)}. The receiver remains usable
* and must still be closed independently.
*
* @throws RuntimeException if the write fails.
*/
public void writeCsv(String path) {
writeCsv(path, new CsvWriteOptions());
}
/**
* Materialize this DataFrame as CSV at {@code path} with the supplied {@link CsvWriteOptions}.
* The receiver remains usable and must still be closed independently.
*
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
* etc.).
*/
public void writeCsv(String path, CsvWriteOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
if (path == null) {
throw new IllegalArgumentException("writeCsv path must be non-null");
}
if (options == null) {
throw new IllegalArgumentException("writeCsv options must be non-null");
}
writeCsvWithOptions(nativeHandle, path, options.toBytes());
}
/**
* Materialize this DataFrame as newline-delimited JSON at {@code path}. The path is treated as a
* directory unless overridden via {@link JsonWriteOptions#singleFileOutput(boolean)}. The
* receiver remains usable and must still be closed independently.
*
* @throws RuntimeException if the write fails.
*/
public void writeJson(String path) {
writeJson(path, new JsonWriteOptions());
}
/**
* Materialize this DataFrame as newline-delimited JSON at {@code path} with the supplied {@link
* JsonWriteOptions}. The receiver remains usable and must still be closed independently.
*
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
* etc.).
*/
public void writeJson(String path, JsonWriteOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("DataFrame is closed or already collected");
}
if (path == null) {
throw new IllegalArgumentException("writeJson path must be non-null");
}
if (options == null) {
throw new IllegalArgumentException("writeJson options must be non-null");
}
writeJsonWithOptions(nativeHandle, path, options.toBytes());
}
@Override
public void close() {
if (nativeHandle != 0) {
closeDataFrame(nativeHandle);
nativeHandle = 0;
}
}
private static native void collectDataFrame(long handle, long ffiStreamAddr);
private static native void executeStreamDataFrame(long handle, long ffiStreamAddr);
private static native void closeDataFrame(long handle);
private static native long countRows(long handle);
private static native byte[] schemaIpc(long handle);
private static native long explainPlan(long handle, boolean verbose, boolean analyze);
private static native long cachePlan(long handle);
private static native long describePlan(long handle);
private static native void showDataFrame(long handle);
private static native void showDataFrameWithLimit(long handle, int limit);
private static native long selectColumns(long handle, String[] columnNames);
private static native long filterRows(long handle, String predicate);
private static native long limitRows(long handle, int skip, int fetch);
private static native long distinctRows(long handle);
private static native long dropColumns(long handle, String[] columnNames);
private static native long renameColumn(long handle, String oldName, String newName);
private static native long withColumnExpr(long handle, String name, String expr);
private static native long unnestColumns(long handle, String[] columns, boolean preserveNulls);
private static native long sortRows(
long handle, String[] columns, boolean[] ascending, boolean[] nullsFirst);
private static native long repartitionRoundRobinRows(long handle, int numPartitions);
private static native long repartitionHashRows(long handle, int numPartitions, String[] columns);
private static native long joinDataFrame(
long leftHandle,
long rightHandle,
byte joinType,
String[] leftCols,
String[] rightCols,
String filter);
private static native long joinOnDataFrame(
long leftHandle, long rightHandle, byte joinType, String[] predicates);
private static native void writeParquetWithOptions(
long handle,
String path,
String compression,
boolean singleFileOutputSet,
boolean singleFileOutputValue);
private static native void writeCsvWithOptions(long handle, String path, byte[] optionsBytes);
private static native void writeJsonWithOptions(long handle, String path, byte[] optionsBytes);
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/DataFusionException.java 0000664 0000000 0000000 00000003462 15205164533 0032324 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Base unchecked exception for every error surfaced from the native DataFusion side.
*
*
Concrete subclasses correspond to caller-relevant error categories (planning, execution, IO,
* resources, configuration, not-implemented). Variants of {@code DataFusionError} on the Rust side
* that don't fit a clean caller-facing category surface as the parent class itself.
*
*
All subclasses extend {@link RuntimeException} so existing callers that {@code catch
* (RuntimeException)} keep working unchanged. Callers that want to discriminate can {@code catch
* (PlanException)} / {@code catch (ResourcesExhaustedException)} etc., or {@code catch
* (DataFusionException)} for "anything from DataFusion".
*/
public class DataFusionException extends RuntimeException {
private static final long serialVersionUID = 1L;
public DataFusionException(String message) {
super(message);
}
public DataFusionException(String message, Throwable cause) {
super(message, cause);
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ExecutionException.java 0000664 0000000 0000000 00000003116 15205164533 0032226 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Runtime execution failure: a UDF threw, a join task panicked, an external (non-DataFusion) error
* propagated up, or an FFI-level failure surfaced. Surfaces upstream {@code
* DataFusionError::Execution}, {@code ExecutionJoin}, {@code External}, and {@code Ffi}.
*
*
Note: this is {@code org.apache.datafusion.ExecutionException}, distinct from {@code
* java.util.concurrent.ExecutionException}. A file that needs both should fully-qualify one or
* import them with care.
*/
public class ExecutionException extends DataFusionException {
private static final long serialVersionUID = 1L;
public ExecutionException(String message) {
super(message);
}
public ExecutionException(String message, Throwable cause) {
super(message, cause);
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/FileCompressionType.java 0000664 0000000 0000000 00000002264 15205164533 0032352 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Compression of a file-format source. Shared by {@link CsvReadOptions} and {@link
* NdJsonReadOptions} (and any future format that exposes the same set of compressions). Variant
* names match DataFusion's Rust {@code FileCompressionType} so they round-trip across JNI.
*/
public enum FileCompressionType {
UNCOMPRESSED,
GZIP,
BZIP2,
XZ,
ZSTD
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/FileCompressionTypes.java 0000664 0000000 0000000 00000003663 15205164533 0032541 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Internal helpers for translating between the user-facing {@link FileCompressionType} enum and the
* protobuf-generated {@code org.apache.datafusion.protobuf.FileCompressionType} enum used on the
* wire. Same variant set, different Java types.
*/
final class FileCompressionTypes {
private FileCompressionTypes() {}
static org.apache.datafusion.protobuf.FileCompressionType toProto(FileCompressionType t) {
switch (t) {
case UNCOMPRESSED:
return org.apache.datafusion.protobuf.FileCompressionType
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
case GZIP:
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
case BZIP2:
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
case XZ:
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
case ZSTD:
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
default:
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
}
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/IoException.java 0000664 0000000 0000000 00000003260 15205164533 0030632 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* IO-shaped failure: a local filesystem read failed, an object store request failed, or a parquet /
* arrow / avro decoder reported a malformed file. Surfaces upstream {@code
* DataFusionError::IoError}, {@code ObjectStore}, {@code ArrowError}, {@code ParquetError}, and
* {@code AvroError}.
*
*
Note: this is {@code org.apache.datafusion.IoException} (lowercase {@code o}), distinct from
* {@code java.io.IOException}. The {@code IoException} spelling matches the orthography of the
* underlying {@code DataFusionError::IoError} variant; a file that needs both should fully-qualify
* one.
*/
public class IoException extends DataFusionException {
private static final long serialVersionUID = 1L;
public IoException(String message) {
super(message);
}
public IoException(String message, Throwable cause) {
super(message, cause);
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/JoinType.java 0000664 0000000 0000000 00000004404 15205164533 0030146 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Join algorithm requested for {@link DataFrame#join} / {@link DataFrame#joinOn}. Mirrors
* DataFusion's {@code JoinType} enum one-to-one.
*
*
*
{@link #INNER} — rows where the join condition matches in both sides.
*
{@link #LEFT} / {@link #RIGHT} — outer joins; unmatched rows on the named side are kept and
* padded with nulls on the other side.
*
{@link #FULL} — full outer join; unmatched rows from either side are kept with nulls.
*
{@link #LEFT_SEMI} / {@link #RIGHT_SEMI} — returns rows from the named side that have at
* least one match on the other; only the named side's columns appear in the output.
*
{@link #LEFT_ANTI} / {@link #RIGHT_ANTI} — returns rows from the named side that have no
* match on the other; only the named side's columns appear in the output.
*
{@link #LEFT_MARK} / {@link #RIGHT_MARK} — returns one row per row of the named side, with
* an additional boolean {@code mark} column indicating whether the join condition matched.
*
*/
public enum JoinType {
INNER((byte) 0),
LEFT((byte) 1),
RIGHT((byte) 2),
FULL((byte) 3),
LEFT_SEMI((byte) 4),
RIGHT_SEMI((byte) 5),
LEFT_ANTI((byte) 6),
RIGHT_ANTI((byte) 7),
LEFT_MARK((byte) 8),
RIGHT_MARK((byte) 9);
private final byte code;
JoinType(byte code) {
this.code = code;
}
/** Stable byte code for FFI. */
public byte code() {
return code;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/JsonWriteOptions.java 0000664 0000000 0000000 00000007052 15205164533 0031707 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.util.ArrayList;
import java.util.List;
/**
* Configuration knobs for writing JSON, passed to {@link DataFrame#writeJson(String,
* JsonWriteOptions)}.
*
*
Mirrors a subset of DataFusion's {@code DataFrameWriteOptions} and the writer-side {@code
* JsonOptions}. All setters return {@code this} for fluent chaining. Defaults: every field {@code
* null} or empty (meaning the DataFusion default is used).
*
*
Path semantics: when {@link #singleFileOutput(boolean)} is {@code true}, the path passed to
* {@code writeJson} is the literal output filename. When left unset (the default) and there are no
* partition columns, the path is treated as a directory that DataFusion populates with one or more
* part-files.
*
*
The output is always newline-delimited JSON (NDJSON). DataFusion's JSON writer does not emit
* the bracketed array form, so there is no toggle for it here.
*
*
Compression reuses {@link FileCompressionType} -- the same codec set ({@code UNCOMPRESSED},
* {@code GZIP}, {@code BZIP2}, {@code XZ}, {@code ZSTD}) the read side and the CSV writer accept.
*/
public final class JsonWriteOptions {
private Boolean singleFileOutput;
private final List partitionCols = new ArrayList<>();
private FileCompressionType fileCompressionType;
/**
* When {@code true}, write to a single file at the supplied path. When left unset (the default)
* and no partition columns are configured, the path is treated as a directory and DataFusion
* writes one or more part-files.
*/
public JsonWriteOptions singleFileOutput(boolean v) {
this.singleFileOutput = v;
return this;
}
/**
* Hive-style partition columns. Each column listed here is removed from the data rows and encoded
* into the directory layout (one subdirectory per distinct value). Mutually exclusive with {@link
* #singleFileOutput(boolean)} -- DataFusion rejects the combination at write time.
*/
public JsonWriteOptions partitionCols(String... cols) {
this.partitionCols.clear();
for (String c : cols) {
this.partitionCols.add(c);
}
return this;
}
/** Output compression codec. Defaults to uncompressed. */
public JsonWriteOptions fileCompressionType(FileCompressionType t) {
this.fileCompressionType = t;
return this;
}
byte[] toBytes() {
org.apache.datafusion.protobuf.JsonWriteOptionsProto.Builder b =
org.apache.datafusion.protobuf.JsonWriteOptionsProto.newBuilder();
if (singleFileOutput != null) {
b.setSingleFileOutput(singleFileOutput);
}
b.addAllPartitionCols(partitionCols);
if (fileCompressionType != null) {
b.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
}
return b.build().toByteArray();
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/MemoryUsage.java 0000664 0000000 0000000 00000004227 15205164533 0030645 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Snapshot of session-wide memory usage from {@link SessionContext#memoryUsage()}. Returns the
* total bytes currently reserved against the session's {@code MemoryPool} and the peak observed
* since the session was created.
*
*
Multi-tenant attribution: place each tenant in its own {@link SessionContext}. Sessions in
* DataFusion are cheap; one per tenant is the standard pattern. Within a single session, the
* snapshot is the sum across all in-flight queries' operator reservations -- see {@link
* SessionContext#memoryUsage()} for the precise definition of what is and is not counted.
*
*
Instances are immutable.
*/
public final class MemoryUsage {
private final long currentBytes;
private final long peakBytes;
public MemoryUsage(long currentBytes, long peakBytes) {
this.currentBytes = currentBytes;
this.peakBytes = peakBytes;
}
/** Bytes currently reserved against this session's {@code MemoryPool}. */
public long currentBytes() {
return currentBytes;
}
/**
* Maximum value of {@link #currentBytes()} observed since the session was created. Monotonic:
* never decreases for a given session.
*/
public long peakBytes() {
return peakBytes;
}
@Override
public String toString() {
return "MemoryUsage{currentBytes=" + currentBytes + ", peakBytes=" + peakBytes + "}";
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/NativeLibraryLoader.java 0000664 0000000 0000000 00000010055 15205164533 0032306 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
/**
* Loads the {@code datafusion_jni} native library on demand.
*
*
The loader first tries {@link System#loadLibrary(String)} so that operators can override the
* bundled library by placing a build on {@code java.library.path} (for example via {@code
* -Djava.library.path=...} or {@code LD_LIBRARY_PATH}). If that fails the loader extracts the
* platform-specific library from the JAR resource tree and loads it via {@link
* System#load(String)}.
*
*
The bundled libraries live at the conventional path {@code
* org/apache/datafusion/<os>/<arch>/<libfile>}. Each JVM extracts into its own
* private temporary directory created with {@link Files#createTempDirectory(String,
* java.nio.file.attribute.FileAttribute[])}, which is mode 0700 on POSIX and user-private under the
* default Windows {@code TMP}. Keeping the directory private prevents another user on the host from
* planting a same-named library for the JVM to pick up.
*/
public final class NativeLibraryLoader {
private static final String TMP_DIR_PREFIX = "datafusion-java-";
private static volatile boolean loaded;
private NativeLibraryLoader() {}
public static synchronized void loadLibrary() {
if (loaded) {
return;
}
if (tryLoadFromLibraryPath()) {
loaded = true;
return;
}
loadFromClasspath();
loaded = true;
}
private static boolean tryLoadFromLibraryPath() {
try {
System.loadLibrary(Platform.LIBRARY_NAME);
return true;
} catch (UnsatisfiedLinkError ignored) {
return false;
}
}
private static void loadFromClasspath() {
Platform platform = Platform.current();
String resource = platform.resourcePath();
try (InputStream check = NativeLibraryLoader.class.getResourceAsStream(resource)) {
if (check == null) {
throw new UnsatisfiedLinkError(
"No bundled datafusion_jni library for "
+ platform
+ " (expected classpath:"
+ resource
+ ")."
+ " Build the native crate and add it to java.library.path,"
+ " or depend on a JAR built for this platform.");
}
} catch (IOException e) {
throw linkError("Failed to probe " + resource, e);
}
try {
Path extracted = extractToPrivateTempDir(resource, platform.libFileName());
System.load(extracted.toAbsolutePath().toString());
} catch (IOException e) {
throw linkError("Failed to extract " + resource, e);
}
}
private static Path extractToPrivateTempDir(String resource, String fileName) throws IOException {
Path tmpDir = Files.createTempDirectory(TMP_DIR_PREFIX);
tmpDir.toFile().deleteOnExit();
Path target = tmpDir.resolve(fileName);
target.toFile().deleteOnExit();
try (InputStream in = NativeLibraryLoader.class.getResourceAsStream(resource)) {
Files.copy(in, target);
}
return target;
}
private static UnsatisfiedLinkError linkError(String message, Throwable cause) {
UnsatisfiedLinkError err = new UnsatisfiedLinkError(message + ": " + cause.getMessage());
err.initCause(cause);
return err;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/NdJsonReadOptions.java 0000664 0000000 0000000 00000005230 15205164533 0031746 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import org.apache.arrow.vector.types.pojo.Schema;
/**
* Configuration knobs for newline-delimited JSON sources passed to {@link
* SessionContext#registerJson(String, String, NdJsonReadOptions)} and {@link
* SessionContext#readJson(String, NdJsonReadOptions)}.
*
*
Mirrors a subset of DataFusion's {@code NdJsonReadOptions}. All setters return {@code this}
* for fluent chaining. Defaults match the Rust struct: {@code fileExtension = ".json"}, {@code
* fileCompressionType = UNCOMPRESSED}; {@code schemaInferMaxRecords} unset (the DataFusion default
* is used).
*/
public final class NdJsonReadOptions {
private String fileExtension = ".json";
private FileCompressionType fileCompressionType = FileCompressionType.UNCOMPRESSED;
private Long schemaInferMaxRecords;
private Schema schema;
public NdJsonReadOptions fileExtension(String ext) {
this.fileExtension = ext;
return this;
}
public NdJsonReadOptions fileCompressionType(FileCompressionType t) {
this.fileCompressionType = t;
return this;
}
public NdJsonReadOptions schemaInferMaxRecords(long n) {
if (n < 0) {
throw new IllegalArgumentException("schemaInferMaxRecords must be non-negative, got " + n);
}
this.schemaInferMaxRecords = n;
return this;
}
public NdJsonReadOptions schema(Schema schema) {
this.schema = schema;
return this;
}
byte[] toBytes() {
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.Builder b =
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.newBuilder()
.setFileExtension(fileExtension)
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
if (schemaInferMaxRecords != null) {
b.setSchemaInferMaxRecords(schemaInferMaxRecords);
}
return b.build().toByteArray();
}
Schema schema() {
return schema;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/NotImplementedException.java 0000664 0000000 0000000 00000002406 15205164533 0033210 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* The requested feature is recognised by DataFusion but not implemented yet. Surfaces upstream
* {@code DataFusionError::NotImplemented}.
*/
public class NotImplementedException extends DataFusionException {
private static final long serialVersionUID = 1L;
public NotImplementedException(String message) {
super(message);
}
public NotImplementedException(String message, Throwable cause) {
super(message, cause);
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ObjectStoreOptions.java 0000664 0000000 0000000 00000026743 15205164533 0032216 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import org.apache.datafusion.protobuf.GcsOptions;
import org.apache.datafusion.protobuf.HttpOptions;
import org.apache.datafusion.protobuf.ObjectStoreRegistration;
import org.apache.datafusion.protobuf.S3Options;
/**
* Description of an {@code object_store::ObjectStore} backend to register on a {@link
* SessionContext}'s {@code RuntimeEnv}. Pass instances to {@link
* SessionContextBuilder#registerObjectStore(ObjectStoreOptions)}.
*
*
Build instances via the per-backend factories: {@link #s3()}, {@link #gcs()}, {@link
* #http(String)}. Each factory returns a builder whose {@code build()} produces an immutable {@code
* ObjectStoreOptions} you can register.
*
*
For the cloud backends (S3 / GCS), the standard SDK environment variables are read as the
* default floor (e.g. {@code AWS_ACCESS_KEY_ID}, {@code AWS_DEFAULT_REGION}, ECS /
* web-identity vars; {@code GOOGLE_APPLICATION_CREDENTIALS}). Explicit setters on the per-backend
* builder always override those env values, so a JVM running on a credentialed host (IAM role, ECS
* task, GKE workload identity) can register an empty-credentials store and let the SDK's default
* chain do its job.
*
*
The URL DataFusion uses to look up the store is either an explicit {@code url(...)} on the
* builder or, if unset, derived from the typed fields:
*
*
*
S3 → {@code s3://}
*
GCS → {@code gs://}
*
HTTP → no scheme-default; {@link #http(String)} requires the URL up front. The URL must be
* a host root ({@code https://example.com/} or {@code https://example.com}); paths cause
* lookup-vs-base-URL collisions in DataFusion's registry. Register one HTTP store per
* scheme+host and let SQL paths carry the rest.
*
*/
public abstract sealed class ObjectStoreOptions
permits ObjectStoreOptions.S3, ObjectStoreOptions.Gcs, ObjectStoreOptions.Http {
private final String url;
private ObjectStoreOptions(String url) {
this.url = url;
}
/** Begin building an {@link S3} (also covers MinIO, R2, any S3-compatible) registration. */
public static S3.Builder s3() {
return new S3.Builder();
}
/** Begin building a {@link Gcs} (Google Cloud Storage) registration. */
public static Gcs.Builder gcs() {
return new Gcs.Builder();
}
/**
* Begin building an {@link Http} (listing-capable HTTP/WebDAV) registration. The listing-root URL
* is required because no sensible default exists.
*
* @throws IllegalArgumentException if {@code listingUrl} is {@code null}.
*/
public static Http.Builder http(String listingUrl) {
if (listingUrl == null) {
throw new IllegalArgumentException("http listing URL must be non-null");
}
return new Http.Builder(listingUrl);
}
abstract ObjectStoreRegistration toRegistration();
/** Amazon S3 (and any S3-compatible endpoint such as MinIO, Cloudflare R2, Wasabi). */
public static final class S3 extends ObjectStoreOptions {
private final String bucket;
private final String region;
private final String endpoint;
private final String accessKeyId;
private final String secretAccessKey;
private final String sessionToken;
private final Boolean allowHttp;
private final Boolean skipSignature;
private final Boolean imdsv1Fallback;
private S3(Builder b) {
super(b.url);
this.bucket = b.bucket;
this.region = b.region;
this.endpoint = b.endpoint;
this.accessKeyId = b.accessKeyId;
this.secretAccessKey = b.secretAccessKey;
this.sessionToken = b.sessionToken;
this.allowHttp = b.allowHttp;
this.skipSignature = b.skipSignature;
this.imdsv1Fallback = b.imdsv1Fallback;
}
@Override
ObjectStoreRegistration toRegistration() {
S3Options.Builder s3 = S3Options.newBuilder().setBucket(bucket);
if (region != null) s3.setRegion(region);
if (endpoint != null) s3.setEndpoint(endpoint);
if (accessKeyId != null) s3.setAccessKeyId(accessKeyId);
if (secretAccessKey != null) s3.setSecretAccessKey(secretAccessKey);
if (sessionToken != null) s3.setSessionToken(sessionToken);
if (allowHttp != null) s3.setAllowHttp(allowHttp);
if (skipSignature != null) s3.setSkipSignature(skipSignature);
if (imdsv1Fallback != null) s3.setImdsv1Fallback(imdsv1Fallback);
ObjectStoreRegistration.Builder reg = ObjectStoreRegistration.newBuilder().setS3(s3);
if (super.url != null) reg.setUrl(super.url);
return reg.build();
}
public static final class Builder {
private String url;
private String bucket;
private String region;
private String endpoint;
private String accessKeyId;
private String secretAccessKey;
private String sessionToken;
private Boolean allowHttp;
private Boolean skipSignature;
private Boolean imdsv1Fallback;
private Builder() {}
/**
* Override the registration URL. Default is {@code s3://}; use this to register under
* a non-default scheme such as {@code s3a://}.
*/
public Builder url(String url) {
this.url = url;
return this;
}
/** Bucket name. Required. */
public Builder bucket(String bucket) {
this.bucket = bucket;
return this;
}
public Builder region(String region) {
this.region = region;
return this;
}
/** Override the endpoint URL (e.g. {@code https://minio.internal:9000}). */
public Builder endpoint(String endpoint) {
this.endpoint = endpoint;
return this;
}
public Builder accessKeyId(String accessKeyId) {
this.accessKeyId = accessKeyId;
return this;
}
public Builder secretAccessKey(String secretAccessKey) {
this.secretAccessKey = secretAccessKey;
return this;
}
public Builder sessionToken(String sessionToken) {
this.sessionToken = sessionToken;
return this;
}
/** Allow plain {@code http://} endpoints (e.g. local MinIO). */
public Builder allowHttp(boolean allowHttp) {
this.allowHttp = allowHttp;
return this;
}
/** Skip request signing (anonymous / public-bucket reads). */
public Builder skipSignature(boolean skipSignature) {
this.skipSignature = skipSignature;
return this;
}
/** Allow falling back to IMDSv1 for credential discovery on EC2. */
public Builder imdsv1Fallback(boolean imdsv1Fallback) {
this.imdsv1Fallback = imdsv1Fallback;
return this;
}
public S3 build() {
if (bucket == null || bucket.isEmpty()) {
throw new IllegalArgumentException("S3 ObjectStoreOptions: bucket is required");
}
return new S3(this);
}
}
}
/** Google Cloud Storage. */
public static final class Gcs extends ObjectStoreOptions {
private final String bucket;
private final String serviceAccountKey;
private final String serviceAccountPath;
private final String applicationCredentials;
private Gcs(Builder b) {
super(b.url);
this.bucket = b.bucket;
this.serviceAccountKey = b.serviceAccountKey;
this.serviceAccountPath = b.serviceAccountPath;
this.applicationCredentials = b.applicationCredentials;
}
@Override
ObjectStoreRegistration toRegistration() {
GcsOptions.Builder gcs = GcsOptions.newBuilder().setBucket(bucket);
if (serviceAccountKey != null) gcs.setServiceAccountKey(serviceAccountKey);
if (serviceAccountPath != null) gcs.setServiceAccountPath(serviceAccountPath);
if (applicationCredentials != null) gcs.setApplicationCredentials(applicationCredentials);
ObjectStoreRegistration.Builder reg = ObjectStoreRegistration.newBuilder().setGcs(gcs);
if (super.url != null) reg.setUrl(super.url);
return reg.build();
}
public static final class Builder {
private String url;
private String bucket;
private String serviceAccountKey;
private String serviceAccountPath;
private String applicationCredentials;
private Builder() {}
/** Override the registration URL. Default is {@code gs://}. */
public Builder url(String url) {
this.url = url;
return this;
}
/** Bucket name. Required. */
public Builder bucket(String bucket) {
this.bucket = bucket;
return this;
}
/** Inline service-account JSON. Mutually exclusive with {@link #serviceAccountPath}. */
public Builder serviceAccountKey(String serviceAccountKey) {
this.serviceAccountKey = serviceAccountKey;
return this;
}
/**
* Filesystem path to service-account JSON. Mutually exclusive with {@link
* #serviceAccountKey}.
*/
public Builder serviceAccountPath(String serviceAccountPath) {
this.serviceAccountPath = serviceAccountPath;
return this;
}
/**
* Filesystem path to the application-default-credentials JSON. Maps to {@code
* with_application_credentials} on the upstream Rust builder, which takes a path, not inline
* content.
*/
public Builder applicationCredentials(String applicationCredentialsPath) {
this.applicationCredentials = applicationCredentialsPath;
return this;
}
public Gcs build() {
if (bucket == null || bucket.isEmpty()) {
throw new IllegalArgumentException("GCS ObjectStoreOptions: bucket is required");
}
if (serviceAccountKey != null && serviceAccountPath != null) {
throw new IllegalArgumentException(
"GCS ObjectStoreOptions: serviceAccountKey and serviceAccountPath are mutually"
+ " exclusive");
}
return new Gcs(this);
}
}
}
/** Listing-capable HTTP / WebDAV store. */
public static final class Http extends ObjectStoreOptions {
private final Boolean allowHttp;
private Http(Builder b) {
super(b.url);
this.allowHttp = b.allowHttp;
}
@Override
ObjectStoreRegistration toRegistration() {
HttpOptions.Builder http = HttpOptions.newBuilder();
if (allowHttp != null) http.setAllowHttp(allowHttp);
ObjectStoreRegistration.Builder reg = ObjectStoreRegistration.newBuilder().setHttp(http);
reg.setUrl(super.url);
return reg.build();
}
public static final class Builder {
private final String url;
private Boolean allowHttp;
private Builder(String url) {
this.url = url;
}
public Builder allowHttp(boolean allowHttp) {
this.allowHttp = allowHttp;
return this;
}
public Http build() {
return new Http(this);
}
}
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ParquetReadOptions.java 0000664 0000000 0000000 00000005174 15205164533 0032203 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import org.apache.arrow.vector.types.pojo.Schema;
/**
* Configuration knobs for parquet sources passed to {@link SessionContext#registerParquet(String,
* String, ParquetReadOptions)} and {@link SessionContext#readParquet(String, ParquetReadOptions)}.
*
*
Mirrors a subset of DataFusion's {@code ParquetReadOptions}. All setters return {@code this}
* for fluent chaining. Defaults: {@code fileExtension = ".parquet"}; all other fields {@code null}
* (meaning the SessionConfig default is used, or the schema is inferred from the file).
*/
public final class ParquetReadOptions {
private String fileExtension = ".parquet";
private Boolean parquetPruning;
private Boolean skipMetadata;
private Long metadataSizeHint;
private Schema schema;
public ParquetReadOptions fileExtension(String ext) {
this.fileExtension = ext;
return this;
}
public ParquetReadOptions parquetPruning(boolean v) {
this.parquetPruning = v;
return this;
}
public ParquetReadOptions skipMetadata(boolean v) {
this.skipMetadata = v;
return this;
}
public ParquetReadOptions metadataSizeHint(long bytes) {
this.metadataSizeHint = bytes;
return this;
}
public ParquetReadOptions schema(Schema schema) {
this.schema = schema;
return this;
}
byte[] toBytes() {
org.apache.datafusion.protobuf.ParquetReadOptionsProto.Builder b =
org.apache.datafusion.protobuf.ParquetReadOptionsProto.newBuilder()
.setFileExtension(fileExtension);
if (parquetPruning != null) {
b.setParquetPruning(parquetPruning);
}
if (skipMetadata != null) {
b.setSkipMetadata(skipMetadata);
}
if (metadataSizeHint != null) {
b.setMetadataSizeHint(metadataSizeHint);
}
return b.build().toByteArray();
}
Schema schema() {
return schema;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ParquetWriteOptions.java 0000664 0000000 0000000 00000004510 15205164533 0032413 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Configuration knobs for writing parquet, passed to {@link DataFrame#writeParquet(String,
* ParquetWriteOptions)}.
*
*
Mirrors a subset of DataFusion's {@code DataFrameWriteOptions} and {@code
* TableParquetOptions}. All setters return {@code this} for fluent chaining. Defaults: all fields
* {@code null} (meaning the DataFusion default is used).
*
*
Path semantics: when {@link #singleFileOutput(boolean)} is {@code true}, the path passed to
* {@code writeParquet} is the literal output filename. When left unset (the default), the path is a
* directory that DataFusion populates with one or more part-files.
*/
public final class ParquetWriteOptions {
private String compression;
private Boolean singleFileOutput;
/**
* Compression codec spec, passed verbatim to DataFusion. Examples: {@code "snappy"}, {@code
* "zstd(3)"}, {@code "gzip(6)"}, {@code "lz4"}, {@code "uncompressed"}. Invalid values surface as
* {@link RuntimeException} at write time.
*/
public ParquetWriteOptions compression(String spec) {
this.compression = spec;
return this;
}
/**
* When {@code true}, write to a single file at the supplied path. When left unset (the default),
* the path is treated as a directory and DataFusion writes one or more part-files.
*/
public ParquetWriteOptions singleFileOutput(boolean v) {
this.singleFileOutput = v;
return this;
}
String compression() {
return compression;
}
Boolean singleFileOutput() {
return singleFileOutput;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/PlanException.java 0000664 0000000 0000000 00000002430 15205164533 0031153 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* SQL parsing, logical planning, or schema-resolution failure. Surfaces upstream {@code
* DataFusionError::Plan}, {@code DataFusionError::SQL}, and {@code DataFusionError::SchemaError}.
*/
public class PlanException extends DataFusionException {
private static final long serialVersionUID = 1L;
public PlanException(String message) {
super(message);
}
public PlanException(String message, Throwable cause) {
super(message, cause);
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/Platform.java 0000664 0000000 0000000 00000010111 15205164533 0030161 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.util.Locale;
/**
* Identifies a target OS/arch pair and the path at which the bundled native library is published
* inside the JAR.
*
*
Resource layout follows the convention from the {@code datafusion-java} packaging design:
*
*
Package-private; consumed only by {@link NativeLibraryLoader}.
*/
final class Platform {
static final String LIBRARY_NAME = "datafusion_jni";
static final String RESOURCE_PREFIX = "org/apache/datafusion";
enum Os {
LINUX("linux", "lib", "so"),
DARWIN("darwin", "lib", "dylib"),
WINDOWS("windows", "", "dll");
final String dirName;
final String libPrefix;
final String libSuffix;
Os(String dirName, String libPrefix, String libSuffix) {
this.dirName = dirName;
this.libPrefix = libPrefix;
this.libSuffix = libSuffix;
}
}
final Os os;
final String arch;
private Platform(Os os, String arch) {
this.os = os;
this.arch = arch;
}
static Platform current() {
return of(System.getProperty("os.name"), System.getProperty("os.arch"));
}
static Platform of(String osName, String osArch) {
Os os = detectOs(osName);
String arch = detectArch(os, osArch);
return new Platform(os, arch);
}
static Os detectOs(String osName) {
if (osName == null) {
throw new UnsupportedOperationException("os.name is not set");
}
String n = osName.toLowerCase(Locale.ROOT);
if (n.startsWith("linux")) {
return Os.LINUX;
}
if (n.startsWith("mac") || n.contains("darwin")) {
return Os.DARWIN;
}
if (n.startsWith("windows")) {
return Os.WINDOWS;
}
throw new UnsupportedOperationException("Unsupported OS for datafusion_jni: " + osName);
}
/**
* Returns the architecture segment used in the resource path for {@code os}.
*
*
Linux and Windows use {@code amd64} (Java's preferred name for x86_64), while macOS uses
* {@code x86_64}; all use {@code aarch64} for ARM64.
*/
static String detectArch(Os os, String osArch) {
if (osArch == null) {
throw new UnsupportedOperationException("os.arch is not set");
}
String n = osArch.toLowerCase(Locale.ROOT);
boolean isX64 = n.equals("amd64") || n.equals("x86_64") || n.equals("x64");
boolean isArm64 = n.equals("aarch64") || n.equals("arm64");
if (isX64) {
return os == Os.DARWIN ? "x86_64" : "amd64";
}
if (isArm64) {
return "aarch64";
}
throw new UnsupportedOperationException(
"Unsupported CPU architecture for datafusion_jni: " + osArch);
}
String libFileName() {
return os.libPrefix + LIBRARY_NAME + "." + os.libSuffix;
}
/**
* Absolute classpath resource path (with leading slash) of the bundled native library for this
* platform.
*/
String resourcePath() {
return "/" + RESOURCE_PREFIX + "/" + os.dirName + "/" + arch + "/" + libFileName();
}
@Override
public String toString() {
return os.dirName + "/" + arch;
}
}
ResourcesExhaustedException.java 0000664 0000000 0000000 00000002741 15205164533 0034034 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* The DataFusion runtime exhausted a configured resource budget — typically the memory pool, but
* applies to any guard upstream surfaces as {@code DataFusionError::ResourcesExhausted}.
*
*
Distinguishing this from {@link ExecutionException} lets callers retry transient
* resource-pressure failures without retrying genuine query bugs.
*/
public class ResourcesExhaustedException extends DataFusionException {
private static final long serialVersionUID = 1L;
public ResourcesExhaustedException(String message) {
super(message);
}
public ResourcesExhaustedException(String message, Throwable cause) {
super(message, cause);
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/RuntimeStats.java 0000664 0000000 0000000 00000011620 15205164533 0031045 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Snapshot of the underlying Tokio runtime from {@link SessionContext#runtimeStats()}. Mirrors the
* subset of {@code tokio_metrics::RuntimeMetrics} fields that node-level dashboards typically
* surface (worker count, busy time, queue depth, etc.).
*
*
Instances are immutable. The snapshot is process-wide rather than per-session because the JNI
* library drives a single shared multi-threaded Tokio runtime; gating on the {@code SessionContext}
* only ensures the caller still has a live handle.
*
*
Requires the {@code runtime-metrics} Cargo feature on the native crate, plus {@code
* RUSTFLAGS="--cfg tokio_unstable"} at build time.
*/
public final class RuntimeStats {
private final int numWorkers;
private final long liveTasksCount;
private final long globalQueueDepth;
private final long elapsedNanos;
private final long totalBusyNanos;
private final long totalParkCount;
private final long totalPollsCount;
private final long totalNoopCount;
private final long totalStealCount;
private final long totalLocalScheduleCount;
private final long totalOverflowCount;
public RuntimeStats(
int numWorkers,
long liveTasksCount,
long globalQueueDepth,
long elapsedNanos,
long totalBusyNanos,
long totalParkCount,
long totalPollsCount,
long totalNoopCount,
long totalStealCount,
long totalLocalScheduleCount,
long totalOverflowCount) {
this.numWorkers = numWorkers;
this.liveTasksCount = liveTasksCount;
this.globalQueueDepth = globalQueueDepth;
this.elapsedNanos = elapsedNanos;
this.totalBusyNanos = totalBusyNanos;
this.totalParkCount = totalParkCount;
this.totalPollsCount = totalPollsCount;
this.totalNoopCount = totalNoopCount;
this.totalStealCount = totalStealCount;
this.totalLocalScheduleCount = totalLocalScheduleCount;
this.totalOverflowCount = totalOverflowCount;
}
/** Number of OS worker threads driving the multi-threaded Tokio runtime. */
public int numWorkers() {
return numWorkers;
}
/** Number of tasks currently scheduled (alive) on the runtime. */
public long liveTasksCount() {
return liveTasksCount;
}
/** Tasks waiting in the global injection queue, not yet picked up by a worker. */
public long globalQueueDepth() {
return globalQueueDepth;
}
/** Wall-clock nanoseconds covered by this snapshot's interval (since runtime start). */
public long elapsedNanos() {
return elapsedNanos;
}
/** Total nanoseconds workers spent doing work (sum across workers). */
public long totalBusyNanos() {
return totalBusyNanos;
}
/** Times a worker has parked itself (gone idle waiting for work). */
public long totalParkCount() {
return totalParkCount;
}
/** Total task polls completed across workers. */
public long totalPollsCount() {
return totalPollsCount;
}
/** Times a worker unparked but found no work (false wakeup). */
public long totalNoopCount() {
return totalNoopCount;
}
/** Times a worker successfully stole tasks from another worker. */
public long totalStealCount() {
return totalStealCount;
}
/** Tasks scheduled into a worker's local queue. */
public long totalLocalScheduleCount() {
return totalLocalScheduleCount;
}
/** Times a worker's local queue overflowed and pushed tasks to the injector. */
public long totalOverflowCount() {
return totalOverflowCount;
}
@Override
public String toString() {
return "RuntimeStats{numWorkers="
+ numWorkers
+ ", liveTasksCount="
+ liveTasksCount
+ ", globalQueueDepth="
+ globalQueueDepth
+ ", elapsedNanos="
+ elapsedNanos
+ ", totalBusyNanos="
+ totalBusyNanos
+ ", totalParkCount="
+ totalParkCount
+ ", totalPollsCount="
+ totalPollsCount
+ ", totalNoopCount="
+ totalNoopCount
+ ", totalStealCount="
+ totalStealCount
+ ", totalLocalScheduleCount="
+ totalLocalScheduleCount
+ ", totalOverflowCount="
+ totalOverflowCount
+ "}";
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ScalarFunction.java 0000664 0000000 0000000 00000007413 15205164533 0031323 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.types.pojo.Field;
/**
* A Java-implemented scalar SQL function. Implementations declare their own name, signature, and
* volatility, and evaluate one input batch at a time.
*
*
Mirrors DataFusion's {@code ScalarUDFImpl} trait. Wrap an instance in a {@link ScalarUdf} and
* register it via {@link SessionContext#registerUdf(ScalarUdf)} to make it callable from SQL or
* DataFrame plans.
*
*
Implementations may be invoked concurrently by DataFusion on multiple worker threads. If the
* implementation carries mutable state, the implementation must synchronize it.
*/
public interface ScalarFunction {
/** SQL name under which this function is invoked (e.g., {@code "add_one"}). */
String name();
/**
* Declared argument fields, in positional order. The function is registered with an exact
* signature; calls whose argument types do not match exactly are rejected.
*
*
Each entry is an Arrow {@link Field} -- a name plus a {@code FieldType} plus an optional
* list of child fields. Use {@link Field#nullable(String,
* org.apache.arrow.vector.types.pojo.ArrowType)} for primitive types (e.g. {@code
* Field.nullable("arg0", new ArrowType.Int(32, true))}). Nested types like {@code List}, {@code
* Struct}, and {@code Map} require the children list to carry element / member / key / value type
* information; constructing a {@code Field} via {@code new Field(name, FieldType, children)} is
* the canonical Arrow way to do that.
*/
List argFields();
/**
* Declared return field. The returned {@link ColumnarValue}'s vector must have this exact type,
* including any nested children. Same construction rules as {@link #argFields()}.
*/
Field returnField();
/**
* Volatility classification. Use {@link Volatility#IMMUTABLE} for pure functions, {@link
* Volatility#STABLE} for functions deterministic within a query, and {@link Volatility#VOLATILE}
* for non-deterministic functions.
*/
Volatility volatility();
/**
* Compute the function result for one input batch.
*
* @param allocator the {@link BufferAllocator} that MUST be used for any new Arrow vector
* allocation, including the result. Buffers allocated from other allocators will not survive
* the JNI handoff.
* @param args the per-arg {@link ColumnarValue}s and the batch row count. Each {@link
* ColumnarValue} is a read-only view; the implementation must NOT close its underlying
* vector.
* @return a {@link ColumnarValue} of the declared return type. If {@link ColumnarValue.Array},
* the underlying vector must have length {@code args.rowCount()}; if {@link
* ColumnarValue.Scalar}, length 1. Ownership of the returned vector transfers to the
* framework; the implementation must NOT close it.
*/
ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args);
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ScalarFunctionArgs.java 0000664 0000000 0000000 00000003431 15205164533 0032134 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.util.List;
import java.util.Objects;
/**
* Bundle of inputs passed to {@link ScalarFunction#evaluate}: the per-arg {@link ColumnarValue}s
* (in declared order) and the batch row count DataFusion is driving.
*
*
Mirrors DataFusion's {@code datafusion::logical_expr::ScalarFunctionArgs}. {@code rowCount} is
* the only channel by which an array-returning UDF without array-typed inputs (all-scalar args, or
* nullary) can size its output. Nullary UDFs that prefer to broadcast a single value should return
* {@link ColumnarValue#scalar(org.apache.arrow.vector.FieldVector) ColumnarValue.scalar(...)}
* instead, which removes the need to consult {@code rowCount}.
*/
public record ScalarFunctionArgs(List args, int rowCount) {
public ScalarFunctionArgs {
args = List.copyOf(Objects.requireNonNull(args, "args"));
if (rowCount < 0) {
throw new IllegalArgumentException("rowCount must be >= 0, got " + rowCount);
}
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/ScalarUdf.java 0000664 0000000 0000000 00000006041 15205164533 0030250 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.util.List;
import java.util.Objects;
import org.apache.arrow.vector.types.pojo.Field;
/**
* A scalar UDF registration handle: pairs a {@link ScalarFunction} implementation with the metadata
* DataFusion needs to dispatch SQL calls to it.
*
*
Mirrors DataFusion's {@code ScalarUDF} struct. Construct one from a {@link ScalarFunction} via
* {@link #fromImpl(ScalarFunction)} (or the public constructor) and pass it to {@link
* SessionContext#registerUdf(ScalarUdf)}.
*/
public final class ScalarUdf {
private final ScalarFunction impl;
private final String name;
private final List argFields;
private final Field returnField;
private final Volatility volatility;
/**
* Wrap a {@link ScalarFunction} for registration. Each metadata getter is invoked once here and
* cached, so a user impl that allocates per call won't be double-evaluated at registration.
*
* @throws NullPointerException if {@code impl} or any of its declared metadata is null
*/
public ScalarUdf(ScalarFunction impl) {
this.impl = Objects.requireNonNull(impl, "impl");
this.name = Objects.requireNonNull(impl.name(), "impl.name()");
this.argFields = Objects.requireNonNull(impl.argFields(), "impl.argFields()");
this.returnField = Objects.requireNonNull(impl.returnField(), "impl.returnField()");
this.volatility = Objects.requireNonNull(impl.volatility(), "impl.volatility()");
}
/** Equivalent to {@code new ScalarUdf(impl)}; mirrors Rust's {@code ScalarUDF::new_from_impl}. */
public static ScalarUdf fromImpl(ScalarFunction impl) {
return new ScalarUdf(impl);
}
/** The wrapped implementation. */
public ScalarFunction impl() {
return impl;
}
/** SQL name; cached from {@link ScalarFunction#name()}. */
public String name() {
return name;
}
/** Declared argument fields; cached from {@link ScalarFunction#argFields()}. */
public List argFields() {
return argFields;
}
/** Declared return field; cached from {@link ScalarFunction#returnField()}. */
public Field returnField() {
return returnField;
}
/** Volatility classification; cached from {@link ScalarFunction#volatility()}. */
public Volatility volatility() {
return volatility;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/SessionContext.java 0000664 0000000 0000000 00000060274 15205164533 0031404 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.channels.Channels;
import java.util.ArrayList;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
import org.apache.arrow.vector.ipc.ReadChannel;
import org.apache.arrow.vector.ipc.message.MessageSerializer;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
/**
* A DataFusion session context.
*
*
Instances are not thread-safe. Concurrent calls to any of {@link #sql},
* {@link #registerParquet}, or {@link #close} from different threads can produce a use-after-free
* on the native side. Callers must externally synchronize, or confine each context to a single
* thread.
*/
public final class SessionContext implements AutoCloseable {
static {
NativeLibraryLoader.loadLibrary();
}
private long nativeHandle;
public SessionContext() {
this.nativeHandle = createSessionContext();
if (this.nativeHandle == 0) {
throw new RuntimeException("Failed to create native SessionContext");
}
}
SessionContext(byte[] optionsBytes) {
this.nativeHandle = createSessionContextWithOptions(optionsBytes);
if (this.nativeHandle == 0) {
throw new RuntimeException("Failed to create native SessionContext");
}
}
/** Start configuring a {@link SessionContext}. */
public static SessionContextBuilder builder() {
return new SessionContextBuilder();
}
/**
* Parse and plan {@code query}, returning a lazy {@link DataFrame}. The query is not executed
* until {@link DataFrame#collect} is called.
*/
public DataFrame sql(String query) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
long dfHandle = createDataFrame(nativeHandle, query);
return new DataFrame(dfHandle);
}
/**
* Decode a DataFusion-Proto {@code LogicalPlanNode} and return a lazy {@link DataFrame}. The plan
* is not executed until {@link DataFrame#collect} is called.
*
*
The bytes must be a serialized {@code datafusion.LogicalPlanNode} (see {@code
* org.apache.datafusion.protobuf.LogicalPlanNode}).
*
* @throws RuntimeException if the bytes are not a valid {@code LogicalPlanNode} or if logical
* planning fails.
*/
public DataFrame fromProto(byte[] planBytes) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
long dfHandle = createDataFrameFromProto(nativeHandle, planBytes);
return new DataFrame(dfHandle);
}
/**
* Snapshot the session's memory pool: bytes currently held and the peak observed since this
* session was created. Thread-safe; can be polled while queries run.
*
*
For multi-tenant attribution, place each tenant in its own {@link SessionContext}. Within a
* single session, the snapshot is the sum across all in-flight queries -- there is no
* per-DataFrame breakdown today.
*
*
The session's {@code MemoryPool} is wrapped transparently with a tracking adapter at
* construction time; the wrapper layers on top of whatever pool {@link
* SessionContextBuilder#memoryLimit(long, double)} produced (or DataFusion's default unbounded
* pool) and does not change pool semantics (limits, eviction, spilling).
*
*
What this counts: bytes reserved against the {@code MemoryPool} -- operator state for
* sorts, hash joins, aggregates, repartition buffers, and anything else that uses DataFusion's
* {@code MemoryReservation} machinery during execution.
*
*
What this does not count: memory held outside the pool, including record-batch
* buffers materialised by {@link DataFrame#cache()} (stored in an in-memory {@code MemTable} as
* plain {@code Vec} with no reservation), record-batch buffers that have crossed the
* FFI boundary into Arrow's Java allocator, and JVM-side allocations. Operator-level reservations
* are released as the plan unwinds, so a query that runs to completion typically returns {@code
* currentBytes} to ~0 even if the result set is large.
*
* @throws IllegalStateException if this context is closed.
* @throws RuntimeException if the native side has not registered a tracker for this handle
* (should not happen in practice -- tracker registration is done by the constructor).
*/
public MemoryUsage memoryUsage() {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
long[] values = memoryUsageNative(nativeHandle);
return new MemoryUsage(values[0], values[1]);
}
/**
* Snapshot operational counters from the underlying Tokio runtime: worker count, busy time, queue
* depth, etc. Thread-safe; can be polled while queries run.
*
*
The runtime is process-wide rather than per-session because the JNI library drives a single
* shared multi-threaded Tokio runtime. The {@link SessionContext} handle is checked only to
* ensure the caller still has a live session; the values returned are not session-specific.
*
*
Requires the {@code runtime-metrics} Cargo feature on the native crate (off by default).
* Rebuild with:
*
*
If invoked against a native binary built without the feature, this method throws {@link
* RuntimeException} with a message pointing at the rebuild command.
*
* @throws IllegalStateException if this context is closed.
* @throws RuntimeException if the native crate was built without the {@code runtime-metrics}
* feature.
*/
public RuntimeStats runtimeStats() {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
long[] s = runtimeStatsNative(nativeHandle);
return new RuntimeStats(
(int) s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7], s[8], s[9], s[10]);
}
/**
* Return the Arrow {@link Schema} of a registered table. Transferred via Arrow IPC; no {@link
* org.apache.arrow.memory.BufferAllocator} is required because a schema carries no buffer data.
*
* @throws RuntimeException if {@code tableName} is not registered in this context.
*/
public Schema tableSchema(String tableName) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
byte[] ipcBytes = tableSchemaIpc(nativeHandle, tableName);
try {
return MessageSerializer.deserializeSchema(
new ReadChannel(Channels.newChannel(new ByteArrayInputStream(ipcBytes))));
} catch (IOException e) {
throw new RuntimeException("Failed to deserialize IPC schema", e);
}
}
/**
* Read the current value of a {@code datafusion.*} config key. The key must be one DataFusion
* recognises (see {@link SessionContextBuilder#setOption(String, String)} for examples and the
* upstream configuration reference for the full list).
*
*
{@code datafusion.runtime.*} keys (memory limit, temp directory, cache sizes, etc) are not
* yet supported by this getter and will throw. Use the typed {@link
* SessionContextBuilder#memoryLimit(long, double)} and {@link
* SessionContextBuilder#tempDirectory(String)} setters at construction time instead. Round-trip
* support for the runtime subtree is tracked as a follow-up.
*
* @return the current value as a string, or {@code null} if the key is recognised but has no
* value set and no default.
* @throws IllegalArgumentException if {@code key} is {@code null}.
* @throws RuntimeException if the key is not recognised by DataFusion or is in the {@code
* datafusion.runtime.*} subtree.
* @throws IllegalStateException if this context is closed.
*/
public String getOption(String key) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
if (key == null) {
throw new IllegalArgumentException("getOption key must be non-null");
}
return getOptionNative(nativeHandle, key);
}
public void registerCsv(String name, String path) {
registerCsv(name, path, new CsvReadOptions());
}
/**
* Register a CSV file (or directory of CSV files) as a table with the supplied {@link
* CsvReadOptions}.
*
* @throws RuntimeException if registration fails (path not found, schema inference error, etc.).
*/
public void registerCsv(String name, String path, CsvReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
registerCsvWithOptions(
nativeHandle,
name,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
}
/** Read a CSV file as a {@link DataFrame} without registering it. */
public DataFrame readCsv(String path) {
return readCsv(path, new CsvReadOptions());
}
/**
* Read a CSV file as a {@link DataFrame} with the supplied {@link CsvReadOptions}.
*
* @throws RuntimeException if the read fails.
*/
public DataFrame readCsv(String path, CsvReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
long dfHandle =
readCsvWithOptions(
nativeHandle,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
return new DataFrame(dfHandle);
}
public void registerJson(String name, String path) {
registerJson(name, path, new NdJsonReadOptions());
}
/**
* Register a newline-delimited JSON file (or directory of NDJSON files) as a table with the
* supplied {@link NdJsonReadOptions}.
*
* @throws RuntimeException if registration fails (path not found, schema inference error, etc.).
*/
public void registerJson(String name, String path, NdJsonReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
if (name == null) {
throw new IllegalArgumentException("registerJson name must be non-null");
}
if (path == null) {
throw new IllegalArgumentException("registerJson path must be non-null");
}
if (options == null) {
throw new IllegalArgumentException("registerJson options must be non-null");
}
registerJsonWithOptions(
nativeHandle,
name,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
}
/** Read a newline-delimited JSON file as a {@link DataFrame} without registering it. */
public DataFrame readJson(String path) {
return readJson(path, new NdJsonReadOptions());
}
/**
* Read a newline-delimited JSON file as a {@link DataFrame} with the supplied {@link
* NdJsonReadOptions}.
*
* @throws RuntimeException if the read fails.
*/
public DataFrame readJson(String path, NdJsonReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
if (path == null) {
throw new IllegalArgumentException("readJson path must be non-null");
}
if (options == null) {
throw new IllegalArgumentException("readJson options must be non-null");
}
long dfHandle =
readJsonWithOptions(
nativeHandle,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
return new DataFrame(dfHandle);
}
public void registerParquet(String name, String path) {
registerParquet(name, path, new ParquetReadOptions());
}
/**
* Register a parquet file as a table with the supplied {@link ParquetReadOptions}.
*
* @throws RuntimeException if registration fails (path not found, schema mismatch, etc.).
*/
public void registerParquet(String name, String path, ParquetReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
registerParquetWithOptions(
nativeHandle,
name,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
}
/** Read a parquet file as a {@link DataFrame} without registering it. */
public DataFrame readParquet(String path) {
return readParquet(path, new ParquetReadOptions());
}
/**
* Read a parquet file as a {@link DataFrame} with the supplied {@link ParquetReadOptions}.
*
* @throws RuntimeException if the read fails.
*/
public DataFrame readParquet(String path, ParquetReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
long dfHandle =
readParquetWithOptions(
nativeHandle,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
return new DataFrame(dfHandle);
}
/** Register an Arrow IPC file (or directory of Arrow IPC files) as a table. */
public void registerArrow(String name, String path) {
registerArrow(name, path, new ArrowReadOptions());
}
/**
* Register an Arrow IPC file (or directory of Arrow IPC files) as a table with the supplied
* {@link ArrowReadOptions}.
*
* @throws IllegalArgumentException if any of {@code name}, {@code path}, or {@code options} is
* {@code null}.
* @throws RuntimeException if registration fails (path not found, schema mismatch, etc.).
*/
public void registerArrow(String name, String path, ArrowReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
if (name == null) {
throw new IllegalArgumentException("registerArrow name must be non-null");
}
if (path == null) {
throw new IllegalArgumentException("registerArrow path must be non-null");
}
if (options == null) {
throw new IllegalArgumentException("registerArrow options must be non-null");
}
registerArrowWithOptions(
nativeHandle,
name,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
}
/** Read an Arrow IPC file as a {@link DataFrame} without registering it. */
public DataFrame readArrow(String path) {
return readArrow(path, new ArrowReadOptions());
}
/**
* Read an Arrow IPC file as a {@link DataFrame} with the supplied {@link ArrowReadOptions}.
*
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
* @throws RuntimeException if the read fails.
*/
public DataFrame readArrow(String path, ArrowReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
if (path == null) {
throw new IllegalArgumentException("readArrow path must be non-null");
}
if (options == null) {
throw new IllegalArgumentException("readArrow options must be non-null");
}
long dfHandle =
readArrowWithOptions(
nativeHandle,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
return new DataFrame(dfHandle);
}
/** Register an Avro file (or directory of Avro files) as a table. */
public void registerAvro(String name, String path) {
registerAvro(name, path, new AvroReadOptions());
}
/**
* Register an Avro file (or directory of Avro files) as a table with the supplied {@link
* AvroReadOptions}.
*
* @throws IllegalArgumentException if any of {@code name}, {@code path}, or {@code options} is
* {@code null}.
* @throws RuntimeException if registration fails (path not found, schema mismatch, etc.).
*/
public void registerAvro(String name, String path, AvroReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
if (name == null) {
throw new IllegalArgumentException("registerAvro name must be non-null");
}
if (path == null) {
throw new IllegalArgumentException("registerAvro path must be non-null");
}
if (options == null) {
throw new IllegalArgumentException("registerAvro options must be non-null");
}
registerAvroWithOptions(
nativeHandle,
name,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
}
/** Read an Avro file as a {@link DataFrame} without registering it. */
public DataFrame readAvro(String path) {
return readAvro(path, new AvroReadOptions());
}
/**
* Read an Avro file as a {@link DataFrame} with the supplied {@link AvroReadOptions}.
*
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
* @throws RuntimeException if the read fails.
*/
public DataFrame readAvro(String path, AvroReadOptions options) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
if (path == null) {
throw new IllegalArgumentException("readAvro path must be non-null");
}
if (options == null) {
throw new IllegalArgumentException("readAvro options must be non-null");
}
long dfHandle =
readAvroWithOptions(
nativeHandle,
path,
options.toBytes(),
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
return new DataFrame(dfHandle);
}
/**
* Register a Java-implemented scalar UDF. After registration, the function can be invoked by SQL
* via the UDF's name or referenced in DataFusion plans deserialised with {@link #fromProto}.
*
*
The UDF is registered with an exact signature: the runtime will reject calls whose argument
* types do not match the declared {@link ScalarFunction#argFields()} exactly.
*
* @throws RuntimeException if registration fails (e.g., name already registered with an
* incompatible signature, schema serialisation failure).
*/
public void registerUdf(ScalarUdf udf) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
java.util.Objects.requireNonNull(udf, "udf");
ScalarFunction impl = udf.impl();
String name = udf.name();
Volatility volatility = udf.volatility();
List fields = new ArrayList<>(udf.argFields().size() + 1);
fields.add(udf.returnField());
fields.addAll(udf.argFields());
Schema signatureSchema = new Schema(fields);
byte[] signatureBytes = serializeSchemaIpc(signatureSchema);
registerScalarUdf(nativeHandle, name, signatureBytes, volatility.code(), impl);
}
/**
* Register a Java-implemented {@link TableProvider} under {@code name}. SQL queries that
* reference {@code name} call back into {@code provider} to fetch batches.
*
*
{@link TableProvider#schema()} is called once here, on the calling thread, and cached on the
* native side. {@link TableProvider#scan(org.apache.arrow.memory.BufferAllocator)} is called once
* per query that touches the table, on a Tokio worker thread; it must return a fresh, independent
* {@link org.apache.arrow.vector.ipc.ArrowReader} on every call, with its buffers allocated from
* the {@link org.apache.arrow.memory.BufferAllocator} the framework supplies.
*
*
This is the Java counterpart to DataFusion's Rust {@code SessionContext::register_table}.
*
* @throws IllegalArgumentException if {@code name} or {@code provider} is {@code null}.
* @throws IllegalStateException if {@code provider.schema()} returns {@code null}, or this
* context is closed.
* @throws RuntimeException if native registration fails.
*/
public void registerTable(String name, TableProvider provider) {
if (nativeHandle == 0) {
throw new IllegalStateException("SessionContext is closed");
}
if (name == null) {
throw new IllegalArgumentException("registerTable name must be non-null");
}
if (provider == null) {
throw new IllegalArgumentException("registerTable provider must be non-null");
}
Schema schema = provider.schema();
if (schema == null) {
throw new IllegalStateException("TableProvider.schema returned null");
}
byte[] schemaIpc = serializeSchemaIpc(schema);
registerTableNative(nativeHandle, name, schemaIpc, provider);
}
private static byte[] serializeSchemaIpc(Schema schema) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (BufferAllocator allocator = new RootAllocator();
VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator);
ArrowStreamWriter writer = new ArrowStreamWriter(root, null, Channels.newChannel(baos))) {
writer.start();
writer.end();
} catch (IOException e) {
throw new RuntimeException("Failed to serialize Arrow schema for JNI", e);
}
return baos.toByteArray();
}
@Override
public void close() {
if (nativeHandle != 0) {
closeSessionContext(nativeHandle);
nativeHandle = 0;
}
}
private static native long createSessionContext();
private static native long createSessionContextWithOptions(byte[] optionsBytes);
private static native long createDataFrame(long handle, String sql);
private static native long createDataFrameFromProto(long handle, byte[] planBytes);
private static native byte[] tableSchemaIpc(long handle, String tableName);
private static native String getOptionNative(long handle, String key);
private static native long[] memoryUsageNative(long handle);
private static native long[] runtimeStatsNative(long handle);
private static native void registerParquetWithOptions(
long handle, String name, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native long readParquetWithOptions(
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native void registerCsvWithOptions(
long handle, String name, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native long readCsvWithOptions(
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native void registerArrowWithOptions(
long handle, String name, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native long readArrowWithOptions(
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native void registerAvroWithOptions(
long handle, String name, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native long readAvroWithOptions(
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native void registerJsonWithOptions(
long handle, String name, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native long readJsonWithOptions(
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
private static native void closeSessionContext(long handle);
private static native void registerScalarUdf(
long handle, String name, byte[] signatureSchemaBytes, byte volatility, ScalarFunction impl);
private static native void registerTableNative(
long handle, String name, byte[] schemaIpcBytes, TableProvider provider);
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java 0000664 0000000 0000000 00000025042 15205164533 0032705 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.datafusion.protobuf.ConfigOption;
import org.apache.datafusion.protobuf.MemoryLimit;
import org.apache.datafusion.protobuf.SessionOptions;
/**
* Builder for a configured {@link SessionContext}. Each setter is optional; unset fields leave the
* DataFusion default in place.
*/
public final class SessionContextBuilder {
private Integer batchSize;
private Integer targetPartitions;
private Boolean collectStatistics;
private Boolean informationSchema;
private Long memoryLimitBytes;
private Double memoryLimitFraction;
private String tempDirectory;
private CacheManagerOptions cacheManager;
private final LinkedHashMap options = new LinkedHashMap<>();
private final List objectStores = new ArrayList<>();
SessionContextBuilder() {}
public SessionContextBuilder batchSize(int batchSize) {
if (batchSize <= 0) {
throw new IllegalArgumentException("batchSize must be positive, got " + batchSize);
}
this.batchSize = batchSize;
return this;
}
public SessionContextBuilder targetPartitions(int targetPartitions) {
if (targetPartitions <= 0) {
throw new IllegalArgumentException(
"targetPartitions must be positive, got " + targetPartitions);
}
this.targetPartitions = targetPartitions;
return this;
}
public SessionContextBuilder collectStatistics(boolean collectStatistics) {
this.collectStatistics = collectStatistics;
return this;
}
public SessionContextBuilder informationSchema(boolean informationSchema) {
this.informationSchema = informationSchema;
return this;
}
/**
* Cap the memory pool at {@code maxMemoryBytes}, reserving {@code fraction} of it for queries.
*/
public SessionContextBuilder memoryLimit(long maxMemoryBytes, double fraction) {
if (maxMemoryBytes <= 0) {
throw new IllegalArgumentException("maxMemoryBytes must be positive, got " + maxMemoryBytes);
}
if (fraction <= 0.0 || fraction > 1.0) {
throw new IllegalArgumentException("fraction must be in (0, 1], got " + fraction);
}
this.memoryLimitBytes = maxMemoryBytes;
this.memoryLimitFraction = fraction;
return this;
}
/** Directory the DiskManager uses for spill files. */
public SessionContextBuilder tempDirectory(String path) {
this.tempDirectory = path;
return this;
}
/**
* Set an arbitrary {@code datafusion.*} config option by string key. Mirrors DataFusion's {@code
* ConfigOptions::set(key, value)} API — see the DataFusion configuration reference for the full
* set of keys. Entries set this way are applied after the typed setters on this
* builder, so an explicit {@code setOption} call overrides a typed setter for the same knob.
*
*
{@code datafusion.runtime.*} keys (memory limit, temp directory, cache sizes, etc) are not
* yet supported by this setter and will throw at {@link #build()}. Use the typed {@link
* #memoryLimit(long, double)} and {@link #tempDirectory(String)} setters instead. Round-trip
* support for the runtime subtree is tracked as a follow-up.
*
*
Unknown keys or unparseable values are not validated here; they surface as a {@link
* RuntimeException} from {@link #build()} carrying DataFusion's error message.
*
* @throws IllegalArgumentException if {@code key} or {@code value} is {@code null}.
*/
public SessionContextBuilder setOption(String key, String value) {
if (key == null) {
throw new IllegalArgumentException("setOption key must be non-null");
}
if (value == null) {
throw new IllegalArgumentException("setOption value must be non-null");
}
// remove-then-put so a re-issued key moves to the end of the iteration
// order. LinkedHashMap.put on an existing key updates the value but keeps
// the original slot, which would re-order the wire-format relative to
// intervening keys. For overlapping side-effect keys (e.g.
// datafusion.optimizer.enable_dynamic_filter_pushdown rewrites the
// per-operator enable_*_dynamic_filter_pushdown flags) the umbrella key
// could otherwise apply *after* a later override of one of its
// sub-flags, silently undoing the override.
this.options.remove(key);
this.options.put(key, value);
return this;
}
/**
* Apply every entry of {@code entries} via {@link #setOption(String, String)}, in {@link
* LinkedHashMap} insertion order. Use this overload when you need the strict last-write-wins
* ordering guarantee for overlapping side-effect keys (e.g. {@code
* datafusion.optimizer.enable_dynamic_filter_pushdown} rewrites the per-operator {@code
* enable_*_dynamic_filter_pushdown} flags, so a per-operator override must come after the
* umbrella).
*
* @throws IllegalArgumentException if any key or value in {@code entries} is {@code null}.
*/
public SessionContextBuilder setOptions(LinkedHashMap entries) {
if (entries == null) {
throw new IllegalArgumentException("setOptions entries must be non-null");
}
for (Map.Entry e : entries.entrySet()) {
setOption(e.getKey(), e.getValue());
}
return this;
}
/**
* Apply every entry of {@code entries} via {@link #setOption(String, String)}. Iterates in
* whatever order the supplied {@link Map} produces, which for {@link java.util.HashMap} or {@link
* java.util.Properties} is unspecified.
*
*
This is the right overload for the common case where the caller's keys don't overlap with
* any upstream setter's side effects. If you do need order — see the {@link
* #setOptions(LinkedHashMap)} overload, which the compiler will resolve to automatically when you
* pass a {@code LinkedHashMap}.
*
* @throws IllegalArgumentException if any key or value in {@code entries} is {@code null}.
*/
public SessionContextBuilder setOptions(Map entries) {
if (entries == null) {
throw new IllegalArgumentException("setOptions entries must be non-null");
}
for (Map.Entry e : entries.entrySet()) {
setOption(e.getKey(), e.getValue());
}
return this;
}
/**
* Configure DataFusion's built-in {@code CacheManager} for the new context. Build the {@link
* CacheManagerOptions} via {@link CacheManagerOptions#builder()}; each cache slot is independent,
* so leaving a setter unset keeps the upstream default in place.
*
*
Calling this setter twice replaces the previous configuration — there is no incremental
* merge between calls. If you need a different cache configuration, build a new {@code
* CacheManagerOptions} from scratch.
*
* @throws IllegalArgumentException if {@code options} is {@code null}.
*/
public SessionContextBuilder cacheManager(CacheManagerOptions options) {
if (options == null) {
throw new IllegalArgumentException("cacheManager options must be non-null");
}
this.cacheManager = options;
return this;
}
/**
* Register an {@code object_store::ObjectStore} backend on the new context's {@code RuntimeEnv}.
* Build {@link ObjectStoreOptions} via the per-backend factories ({@link ObjectStoreOptions#s3},
* {@link ObjectStoreOptions#gcs}, {@link ObjectStoreOptions#http(String)}). The store is
* reachable inside the resulting {@link SessionContext} by URL — e.g. once an S3 store is
* registered for {@code my-bucket}, {@code ctx.registerParquet("orders",
* "s3://my-bucket/orders/")} will resolve through it.
*
*
Multiple registrations are applied in the order added; if two registrations resolve to the
* same URL, the later one wins (matching upstream {@code RuntimeEnv::register_object_store}).
*
*
If the underlying {@code object_store} cloud-backend Cargo feature is not built into the
* native library, {@link #build} surfaces a clear error rather than silently dropping the
* registration. The default {@code make} build enables all three backends (S3 / GCS / HTTP).
*
* @throws IllegalArgumentException if {@code options} is {@code null}.
*/
public SessionContextBuilder registerObjectStore(ObjectStoreOptions options) {
if (options == null) {
throw new IllegalArgumentException("registerObjectStore options must be non-null");
}
this.objectStores.add(options);
return this;
}
/**
* Construct a {@link SessionContext} with the configured options.
*
* @throws RuntimeException if the native side fails to construct the context.
*/
public SessionContext build() {
return new SessionContext(toBytes());
}
byte[] toBytes() {
SessionOptions.Builder b = SessionOptions.newBuilder();
if (batchSize != null) {
b.setBatchSize(batchSize);
}
if (targetPartitions != null) {
b.setTargetPartitions(targetPartitions);
}
if (collectStatistics != null) {
b.setCollectStatistics(collectStatistics);
}
if (informationSchema != null) {
b.setInformationSchema(informationSchema);
}
if (memoryLimitBytes != null && memoryLimitFraction != null) {
b.setMemoryLimit(
MemoryLimit.newBuilder()
.setMaxMemoryBytes(memoryLimitBytes)
.setMemoryFraction(memoryLimitFraction)
.build());
}
if (tempDirectory != null) {
b.setTempDirectory(tempDirectory);
}
if (cacheManager != null) {
b.setCacheManager(cacheManager.toProto());
}
for (Map.Entry e : options.entrySet()) {
b.addOptions(ConfigOption.newBuilder().setKey(e.getKey()).setValue(e.getValue()).build());
}
for (ObjectStoreOptions os : objectStores) {
b.addObjectStores(os.toRegistration());
}
return b.build().toByteArray();
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/SimpleTableProvider.java 0000664 0000000 0000000 00000005161 15205164533 0032322 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import java.util.function.Function;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.Schema;
/**
* A {@link TableProvider} that pairs a fixed {@link Schema} with a function that opens a fresh
* {@link ArrowReader} for each scan. Provided as a convenience for the common case where there is
* no projection / filter pushdown to implement.
*
*
Each call to {@link #scan(BufferAllocator)} invokes the supplied function and returns whatever
* {@link ArrowReader} it produces, so the function MUST return a fresh, independent reader on every
* invocation (see the contract on {@link TableProvider#scan(BufferAllocator)}).
*
*
As {@link TableProvider} grows additional methods in the future, this class will provide
* defaults so existing callers keep working without changes.
*/
public final class SimpleTableProvider implements TableProvider {
private final Schema schema;
private final Function scanFn;
/**
* @param schema the table schema; returned as-is from {@link #schema()}
* @param scanFn called on every {@link #scan(BufferAllocator)} with the framework-supplied
* allocator; must return a fresh, independent {@link ArrowReader} each time
*/
public SimpleTableProvider(Schema schema, Function scanFn) {
if (schema == null) {
throw new IllegalArgumentException("schema must be non-null");
}
if (scanFn == null) {
throw new IllegalArgumentException("scanFn must be non-null");
}
this.schema = schema;
this.scanFn = scanFn;
}
@Override
public Schema schema() {
return schema;
}
@Override
public ArrowReader scan(BufferAllocator allocator) {
return scanFn.apply(allocator);
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/SortExpr.java 0000664 0000000 0000000 00000005525 15205164533 0030200 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* A single sort key passed to {@link DataFrame#sort(SortExpr...)}, mirroring DataFusion's {@code
* expr::Sort{ expr, asc, nulls_first }}. Build via {@link #asc(String)} or {@link #desc(String)};
* tweak null placement via {@link #nullsFirst(boolean)}.
*
*
v1 accepts column names only -- the {@code column} string is interpreted as a column
* reference, not a SQL expression. Complex sort keys (e.g. {@code "a + b"}) are intentionally
* deferred until the Java binding gains an {@code Expr} builder.
*
*
Defaults match DataFusion: {@link #asc(String)} sorts ascending with NULLs last; {@link
* #desc(String)} sorts descending with NULLs first.
*/
public final class SortExpr {
private final String column;
private final boolean ascending;
private boolean nullsFirst;
private SortExpr(String column, boolean ascending, boolean nullsFirst) {
this.column = column;
this.ascending = ascending;
this.nullsFirst = nullsFirst;
}
/** Sort the given column ascending, with NULLs placed last. */
public static SortExpr asc(String column) {
if (column == null) {
throw new IllegalArgumentException("SortExpr column must be non-null");
}
return new SortExpr(column, true, false);
}
/** Sort the given column descending, with NULLs placed first. */
public static SortExpr desc(String column) {
if (column == null) {
throw new IllegalArgumentException("SortExpr column must be non-null");
}
return new SortExpr(column, false, true);
}
/** Override the default NULL placement. {@code true} = NULLs first, {@code false} = last. */
public SortExpr nullsFirst(boolean v) {
this.nullsFirst = v;
return this;
}
/** The column name this sort key references. */
public String column() {
return column;
}
/** {@code true} if ascending, {@code false} if descending. */
public boolean ascending() {
return ascending;
}
/** {@code true} if NULLs are placed first, {@code false} if last. */
public boolean nullsFirst() {
return nullsFirst;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/TableProvider.java 0000664 0000000 0000000 00000005565 15205164533 0031160 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.Schema;
/**
* A Java-implemented table that can be registered with a {@link SessionContext} via {@link
* SessionContext#registerTable(String, TableProvider)}. Mirrors the role of DataFusion's Rust
* {@code TableProvider} trait, but at present only exposes the methods needed for a full table
* scan; future versions may add filter/projection pushdown and multi-partition support as default
* methods so existing implementations keep working.
*
*
{@link SimpleTableProvider} is a ready-made implementation for the common case of "I have a
* schema and a function that returns an {@link ArrowReader}".
*
*
Each call to {@link #scan(BufferAllocator)} must return a fresh, independent {@link
* ArrowReader} so that queries which touch the table more than once (self-joins, {@code UNION ALL},
* repeated reads) work correctly. The returned reader is closed by the framework when the stream
* ends.
*
*
The schema returned by {@link #schema()} is captured once at registration time. Every batch
* produced by every {@code ArrowReader} returned from {@link #scan(BufferAllocator)} must conform
* to it; a mismatch fails the query.
*/
public interface TableProvider {
/** The fixed schema of this table. Called once, at registration time. */
Schema schema();
/**
* Open a fresh batch stream for this table. Called once per physical scan of the table — a single
* query may invoke this more than once (self-joins, {@code UNION ALL} over the same table, etc.).
*
*
Each invocation MUST return an independent {@link ArrowReader}. The reader's schema MUST
* equal {@link #schema()}. The reader's buffers MUST be allocated from {@code allocator} (or from
* a child of it) — the framework needs the reader's allocator hierarchy to share a root with the
* one it passes here. The allocator contract mirrors the one on {@link ScalarFunction#evaluate}.
*/
ArrowReader scan(BufferAllocator allocator);
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/UnnestOptions.java 0000664 0000000 0000000 00000003227 15205164533 0031237 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Configuration knobs for {@link DataFrame#unnestColumns(UnnestOptions, String...)}, mirroring
* DataFusion's {@code UnnestOptions}. Defaults match upstream: {@code preserveNulls = true}.
*
*
Per-column recursion (DataFusion's {@code recursions}) is intentionally not exposed yet — it
* needs a richer column-pair representation and is filed separately.
*/
public final class UnnestOptions {
private boolean preserveNulls = true;
/**
* When {@code true} (the default), nulls in the input column are preserved as null rows in the
* output. When {@code false}, nulls and empty lists are dropped.
*/
public UnnestOptions preserveNulls(boolean v) {
this.preserveNulls = v;
return this;
}
/** The current {@code preserveNulls} setting. */
public boolean preserveNulls() {
return preserveNulls;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/Volatility.java 0000664 0000000 0000000 00000003062 15205164533 0030544 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
/**
* Volatility classification for a UDF. Mirrors DataFusion's {@code Volatility} enum.
*
*
*
{@link #IMMUTABLE} — pure function: same inputs always produce the same output; safe to
* constant-fold and common-subexpression-eliminate.
*
{@link #STABLE} — deterministic within a single query but not across queries (e.g., {@code
* now()}).
*
{@link #VOLATILE} — may return a different value on every call (e.g., {@code random()}).
*
*/
public enum Volatility {
IMMUTABLE((byte) 0),
STABLE((byte) 1),
VOLATILE((byte) 2);
private final byte code;
Volatility(byte code) {
this.code = code;
}
/** Stable byte code for FFI. */
public byte code() {
return code;
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/internal/ 0000775 0000000 0000000 00000000000 15205164533 0027354 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/internal/JniBridge.java 0000664 0000000 0000000 00000015223 15205164533 0032057 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.internal;
import java.util.ArrayList;
import java.util.List;
import org.apache.arrow.c.ArrowArray;
import org.apache.arrow.c.ArrowArrayStream;
import org.apache.arrow.c.ArrowSchema;
import org.apache.arrow.c.Data;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.datafusion.ColumnarValue;
import org.apache.datafusion.ScalarFunction;
import org.apache.datafusion.ScalarFunctionArgs;
import org.apache.datafusion.TableProvider;
/** Internal trampoline invoked from native code on every UDF call. Not part of the public API. */
public final class JniBridge {
/**
* Shared allocator for UDF inputs/outputs. Created once at class-load time; never closed.
* Outstanding allocations are released by the FFI structs' release callbacks when the native side
* drops them after {@code from_ffi}.
*/
private static final RootAllocator ALLOCATOR = new RootAllocator();
private JniBridge() {}
/** argKind byte signalling a {@link ColumnarValue.Array} arg. */
private static final byte KIND_ARRAY = 0;
/** argKind byte signalling a {@link ColumnarValue.Scalar} arg. */
private static final byte KIND_SCALAR = 1;
/**
* Invoke a scalar UDF for one batch. Called from native code; not for application use.
*
*
Args arrive split into two struct arrays: {@code arrayArgs*} of length {@code rowCount}
* holding the {@link ColumnarValue.Array} arguments in their relative order, and {@code
* scalarArgs*} of length 1 holding the {@link ColumnarValue.Scalar} arguments. {@code argKinds}
* records the original positional order so the bridge can interleave them back into a single
* {@code List} for the user.
*
* @return {@link #KIND_ARRAY} if the UDF returned an Array, {@link #KIND_SCALAR} if it returned a
* Scalar. The native caller uses this to reconstruct the right {@code ColumnarValue} variant.
*/
public static byte invokeScalarUdf(
ScalarFunction impl,
long arrayArgsArrayAddr,
long arrayArgsSchemaAddr,
long scalarArgsArrayAddr,
long scalarArgsSchemaAddr,
byte[] argKinds,
long resultArrayAddr,
long resultSchemaAddr,
int rowCount) {
ArrowArray arrayArr = ArrowArray.wrap(arrayArgsArrayAddr);
ArrowSchema arraySch = ArrowSchema.wrap(arrayArgsSchemaAddr);
ArrowArray scalarArr = ArrowArray.wrap(scalarArgsArrayAddr);
ArrowSchema scalarSch = ArrowSchema.wrap(scalarArgsSchemaAddr);
ArrowArray resultArr = ArrowArray.wrap(resultArrayAddr);
ArrowSchema resultSch = ArrowSchema.wrap(resultSchemaAddr);
try (VectorSchemaRoot arrayRoot =
Data.importVectorSchemaRoot(ALLOCATOR, arrayArr, arraySch, null);
VectorSchemaRoot scalarRoot =
Data.importVectorSchemaRoot(ALLOCATOR, scalarArr, scalarSch, null)) {
List arrayFields = arrayRoot.getFieldVectors();
List scalarFields = scalarRoot.getFieldVectors();
List args = new ArrayList<>(argKinds.length);
int arrayIdx = 0;
int scalarIdx = 0;
for (byte kind : argKinds) {
if (kind == KIND_ARRAY) {
args.add(ColumnarValue.array(arrayFields.get(arrayIdx++)));
} else if (kind == KIND_SCALAR) {
args.add(ColumnarValue.scalar(scalarFields.get(scalarIdx++)));
} else {
throw new IllegalStateException("Unknown argKind byte: " + kind);
}
}
ColumnarValue result = impl.evaluate(ALLOCATOR, new ScalarFunctionArgs(args, rowCount));
if (result == null) {
throw new IllegalStateException("ScalarFunction.evaluate returned null");
}
FieldVector resultVec = result.vector();
byte resultKind;
int expectedLen;
if (result instanceof ColumnarValue.Array) {
resultKind = KIND_ARRAY;
expectedLen = rowCount;
} else {
resultKind = KIND_SCALAR;
expectedLen = 1;
}
if (resultVec.getValueCount() != expectedLen) {
try {
throw new IllegalStateException(
"ScalarFunction.evaluate returned "
+ (resultKind == KIND_ARRAY ? "Array" : "Scalar")
+ " vector with "
+ resultVec.getValueCount()
+ " rows; expected "
+ expectedLen);
} finally {
resultVec.close();
}
}
try {
Data.exportVector(ALLOCATOR, resultVec, null, resultArr, resultSch);
} finally {
resultVec.close();
}
return resultKind;
}
}
/**
* Open a fresh batch stream from a Java {@link TableProvider} and export it through the supplied
* Arrow C Data Interface address. Called from native code; not for application use.
*
*
{@link TableProvider#scan(org.apache.arrow.memory.BufferAllocator)} is called with {@link
* #ALLOCATOR} so that the reader's buffers share the same allocator root required by {@link
* Data#exportArrayStream}.
*
*
On success, ownership of the returned reader transfers to the FFI stream's release callback,
* so the native side closing the stream also closes the reader. On any failure during export, the
* reader is closed here before the exception propagates.
*/
public static void invokeTableScan(TableProvider provider, long ffiStreamAddr) {
ArrowReader reader = provider.scan(ALLOCATOR);
if (reader == null) {
throw new IllegalStateException("TableProvider.scan returned null");
}
ArrowArrayStream stream = ArrowArrayStream.wrap(ffiStreamAddr);
try {
Data.exportArrayStream(ALLOCATOR, reader, stream);
} catch (Throwable t) {
try {
reader.close();
} catch (Exception ignored) {
// best-effort cleanup; original error wins
}
throw t;
}
}
}
apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/proto/ 0000775 0000000 0000000 00000000000 15205164533 0026703 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/main/java/org/apache/datafusion/proto/SchemaConverter.java 0000664 0000000 0000000 00000017374 15205164533 0032652 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.proto;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.arrow.vector.types.DateUnit;
import org.apache.arrow.vector.types.FloatingPointPrecision;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import datafusion_common.DatafusionCommon;
/**
* Convert between Arrow Java {@link Schema} and the {@code datafusion_common.Schema} protobuf shape
* used by DataFusion plan messages such as {@code ListingTableScanNode.schema}.
*
*
Supports the primitive Arrow types this project's tests exercise (Bool, signed/unsigned Int
* 8..64, Float32/64, Utf8, Utf8View, LargeUtf8, Date32, Decimal128). Anything else raises {@link
* UnsupportedOperationException} with a message naming the offending type.
*/
public final class SchemaConverter {
private SchemaConverter() {}
public static DatafusionCommon.Schema toProto(Schema arrow) {
DatafusionCommon.Schema.Builder builder = DatafusionCommon.Schema.newBuilder();
for (Field f : arrow.getFields()) {
builder.addColumns(fieldToProto(f));
}
if (arrow.getCustomMetadata() != null) {
builder.putAllMetadata(arrow.getCustomMetadata());
}
return builder.build();
}
public static Schema fromProto(DatafusionCommon.Schema proto) {
List fields = new ArrayList<>(proto.getColumnsCount());
for (DatafusionCommon.Field f : proto.getColumnsList()) {
fields.add(fieldFromProto(f));
}
Map metadata = new LinkedHashMap<>(proto.getMetadataMap());
return new Schema(fields, metadata);
}
static DatafusionCommon.Field fieldToProto(Field f) {
DatafusionCommon.Field.Builder b =
DatafusionCommon.Field.newBuilder()
.setName(f.getName())
.setArrowType(arrowTypeToProto(f.getType()))
.setNullable(f.isNullable());
if (f.getMetadata() != null) {
b.putAllMetadata(f.getMetadata());
}
return b.build();
}
static Field fieldFromProto(DatafusionCommon.Field f) {
ArrowType type = arrowTypeFromProto(f.getArrowType());
FieldType ft = new FieldType(f.getNullable(), type, null, f.getMetadataMap());
return new Field(f.getName(), ft, null);
}
static DatafusionCommon.ArrowType arrowTypeToProto(ArrowType t) {
DatafusionCommon.EmptyMessage empty = DatafusionCommon.EmptyMessage.getDefaultInstance();
DatafusionCommon.ArrowType.Builder b = DatafusionCommon.ArrowType.newBuilder();
if (t instanceof ArrowType.Bool) {
return b.setBOOL(empty).build();
}
if (t instanceof ArrowType.Int) {
ArrowType.Int i = (ArrowType.Int) t;
if (i.getIsSigned()) {
switch (i.getBitWidth()) {
case 8:
return b.setINT8(empty).build();
case 16:
return b.setINT16(empty).build();
case 32:
return b.setINT32(empty).build();
case 64:
return b.setINT64(empty).build();
default:
throw new UnsupportedOperationException(
"Arrow type Int signed width "
+ i.getBitWidth()
+ " not yet supported by SchemaConverter");
}
} else {
switch (i.getBitWidth()) {
case 8:
return b.setUINT8(empty).build();
case 16:
return b.setUINT16(empty).build();
case 32:
return b.setUINT32(empty).build();
case 64:
return b.setUINT64(empty).build();
default:
throw new UnsupportedOperationException(
"Arrow type Int unsigned width "
+ i.getBitWidth()
+ " not yet supported by SchemaConverter");
}
}
}
if (t instanceof ArrowType.FloatingPoint) {
FloatingPointPrecision p = ((ArrowType.FloatingPoint) t).getPrecision();
if (p == FloatingPointPrecision.SINGLE) return b.setFLOAT32(empty).build();
if (p == FloatingPointPrecision.DOUBLE) return b.setFLOAT64(empty).build();
throw new UnsupportedOperationException(
"Arrow type FloatingPoint " + p + " not yet supported by SchemaConverter");
}
if (t instanceof ArrowType.Utf8) {
return b.setUTF8(empty).build();
}
if (t instanceof ArrowType.Utf8View) {
return b.setUTF8VIEW(empty).build();
}
if (t instanceof ArrowType.LargeUtf8) {
return b.setLARGEUTF8(empty).build();
}
if (t instanceof ArrowType.Date) {
DateUnit u = ((ArrowType.Date) t).getUnit();
if (u == DateUnit.DAY) return b.setDATE32(empty).build();
throw new UnsupportedOperationException(
"Arrow type Date " + u + " not yet supported by SchemaConverter");
}
if (t instanceof ArrowType.Decimal) {
ArrowType.Decimal d = (ArrowType.Decimal) t;
if (d.getBitWidth() != 128) {
throw new UnsupportedOperationException(
"Arrow type Decimal bit width "
+ d.getBitWidth()
+ " not yet supported by SchemaConverter");
}
return b.setDECIMAL128(
DatafusionCommon.Decimal128Type.newBuilder()
.setPrecision(d.getPrecision())
.setScale(d.getScale())
.build())
.build();
}
throw new UnsupportedOperationException(
"Arrow type " + t.getClass().getSimpleName() + " not yet supported by SchemaConverter");
}
static ArrowType arrowTypeFromProto(DatafusionCommon.ArrowType p) {
switch (p.getArrowTypeEnumCase()) {
case BOOL:
return ArrowType.Bool.INSTANCE;
case INT8:
return new ArrowType.Int(8, true);
case INT16:
return new ArrowType.Int(16, true);
case INT32:
return new ArrowType.Int(32, true);
case INT64:
return new ArrowType.Int(64, true);
case UINT8:
return new ArrowType.Int(8, false);
case UINT16:
return new ArrowType.Int(16, false);
case UINT32:
return new ArrowType.Int(32, false);
case UINT64:
return new ArrowType.Int(64, false);
case FLOAT32:
return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE);
case FLOAT64:
return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
case UTF8:
return ArrowType.Utf8.INSTANCE;
case UTF8_VIEW:
return ArrowType.Utf8View.INSTANCE;
case LARGE_UTF8:
return ArrowType.LargeUtf8.INSTANCE;
case DATE32:
return new ArrowType.Date(DateUnit.DAY);
case DECIMAL128:
DatafusionCommon.Decimal128Type d = p.getDECIMAL128();
return new ArrowType.Decimal(d.getPrecision(), d.getScale(), 128);
default:
throw new UnsupportedOperationException(
"datafusion_common.ArrowType "
+ p.getArrowTypeEnumCase()
+ " not yet supported by SchemaConverter");
}
}
}
apache-datafusion-java-0.1.0/core/src/test/ 0000775 0000000 0000000 00000000000 15205164533 0020505 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/ 0000775 0000000 0000000 00000000000 15205164533 0021426 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/ 0000775 0000000 0000000 00000000000 15205164533 0022215 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/ 0000775 0000000 0000000 00000000000 15205164533 0023436 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/ 0000775 0000000 0000000 00000000000 15205164533 0025573 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/ArrowReadOptionsTest.java 0000664 0000000 0000000 00000004202 15205164533 0032536 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import java.util.List;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.protobuf.ArrowReadOptionsProto;
import org.junit.jupiter.api.Test;
import com.google.protobuf.InvalidProtocolBufferException;
class ArrowReadOptionsTest {
@Test
void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
ArrowReadOptionsProto p = ArrowReadOptionsProto.parseFrom(new ArrowReadOptions().toBytes());
assertEquals(".arrow", p.getFileExtension());
}
@Test
void fileExtensionRoundTripsThroughProto() throws InvalidProtocolBufferException {
ArrowReadOptionsProto p =
ArrowReadOptionsProto.parseFrom(new ArrowReadOptions().fileExtension(".ipc").toBytes());
assertEquals(".ipc", p.getFileExtension());
}
@Test
void schemaIsHeldByReferenceAndNotInProto() {
Schema schema =
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
ArrowReadOptions opts = new ArrowReadOptions().schema(schema);
assertSame(schema, opts.schema());
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/AvroReadOptionsTest.java 0000664 0000000 0000000 00000004165 15205164533 0032363 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import java.util.List;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.protobuf.AvroReadOptionsProto;
import org.junit.jupiter.api.Test;
import com.google.protobuf.InvalidProtocolBufferException;
class AvroReadOptionsTest {
@Test
void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
AvroReadOptionsProto p = AvroReadOptionsProto.parseFrom(new AvroReadOptions().toBytes());
assertEquals(".avro", p.getFileExtension());
}
@Test
void fileExtensionRoundTripsThroughProto() throws InvalidProtocolBufferException {
AvroReadOptionsProto p =
AvroReadOptionsProto.parseFrom(new AvroReadOptions().fileExtension(".av").toBytes());
assertEquals(".av", p.getFileExtension());
}
@Test
void schemaIsHeldByReferenceAndNotInProto() {
Schema schema =
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
AvroReadOptions opts = new AvroReadOptions().schema(schema);
assertSame(schema, opts.schema());
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/CacheManagerOptionsTest.java 0000664 0000000 0000000 00000016052 15205164533 0033154 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
import org.apache.datafusion.protobuf.CacheManagerOptionsProto;
import org.apache.datafusion.protobuf.SessionOptions;
import org.junit.jupiter.api.Test;
class CacheManagerOptionsTest {
@Test
void allThreeCachesRoundTripThroughProto() throws Exception {
byte[] bytes =
SessionContext.builder()
.cacheManager(
CacheManagerOptions.builder()
.fileMetadataCache(64L << 20)
.listFilesCache(8L << 20, Duration.ofMinutes(5))
.fileStatisticsCache(true)
.build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
assertTrue(parsed.hasCacheManager());
CacheManagerOptionsProto cm = parsed.getCacheManager();
assertTrue(cm.hasFileMetadataCacheMaxBytes());
assertEquals(64L << 20, cm.getFileMetadataCacheMaxBytes());
assertTrue(cm.hasListFilesCache());
assertTrue(cm.getListFilesCache().hasMaxBytes());
assertEquals(8L << 20, cm.getListFilesCache().getMaxBytes());
assertTrue(cm.getListFilesCache().hasTtlMillis());
assertEquals(Duration.ofMinutes(5).toMillis(), cm.getListFilesCache().getTtlMillis());
assertTrue(cm.hasFileStatisticsCacheEnabled());
assertTrue(cm.getFileStatisticsCacheEnabled());
}
@Test
void unsetSettersAreAbsentInProto() throws Exception {
byte[] bytes =
SessionContext.builder()
.cacheManager(CacheManagerOptions.builder().fileMetadataCache(64L << 20).build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
CacheManagerOptionsProto cm = parsed.getCacheManager();
assertTrue(cm.hasFileMetadataCacheMaxBytes());
// The other two travel as unset, not as zero/empty -- the Rust side
// distinguishes "leave upstream default in place" from "explicitly off".
assertFalse(cm.hasListFilesCache());
assertFalse(cm.hasFileStatisticsCacheEnabled());
}
@Test
void listFilesCacheNullTtlIsUnsetOnTheWire() throws Exception {
byte[] bytes =
SessionContext.builder()
.cacheManager(CacheManagerOptions.builder().listFilesCache(8L << 20, null).build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
CacheManagerOptionsProto cm = parsed.getCacheManager();
assertTrue(cm.hasListFilesCache());
assertTrue(cm.getListFilesCache().hasMaxBytes());
assertEquals(8L << 20, cm.getListFilesCache().getMaxBytes());
// ttl_millis is the channel for "infinite" (None on the Rust side) --
// it has to travel as unset, not as 0.
assertFalse(cm.getListFilesCache().hasTtlMillis());
}
@Test
void fileMetadataCacheZeroIsAccepted() throws Exception {
// Upstream allows 0 (the cache is constructed with a 0-byte budget).
// Java mirrors that contract.
byte[] bytes =
SessionContext.builder()
.cacheManager(CacheManagerOptions.builder().fileMetadataCache(0).build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
CacheManagerOptionsProto cm = parsed.getCacheManager();
assertTrue(cm.hasFileMetadataCacheMaxBytes());
assertEquals(0L, cm.getFileMetadataCacheMaxBytes());
}
@Test
void fileStatisticsCacheFalseTravelsExplicitlyOnTheWire() throws Exception {
// `fileStatisticsCache(false)` is a different end-state from "never
// called this setter" only on the wire -- the Rust side treats both as
// "leave the slot None" today, but distinguishing the two on the wire
// lets us assert the bool actually round-tripped.
byte[] bytes =
SessionContext.builder()
.cacheManager(CacheManagerOptions.builder().fileStatisticsCache(false).build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
CacheManagerOptionsProto cm = parsed.getCacheManager();
assertTrue(cm.hasFileStatisticsCacheEnabled());
assertFalse(cm.getFileStatisticsCacheEnabled());
}
@Test
void unsetCacheManagerIsAbsentInProto() throws Exception {
// Sanity check that builders that never call .cacheManager(...) emit no
// cache_manager field at all -- the existing-callers-see-no-change
// contract relies on this.
byte[] bytes = SessionContext.builder().batchSize(8192).toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
assertFalse(parsed.hasCacheManager());
}
@Test
void rejectsNegativeFileMetadataCacheMaxBytes() {
CacheManagerOptions.Builder b = CacheManagerOptions.builder();
assertThrows(IllegalArgumentException.class, () -> b.fileMetadataCache(-1));
}
@Test
void rejectsNegativeListFilesCacheMaxBytes() {
CacheManagerOptions.Builder b = CacheManagerOptions.builder();
assertThrows(IllegalArgumentException.class, () -> b.listFilesCache(-1, null));
}
@Test
void rejectsNegativeListFilesCacheTtl() {
CacheManagerOptions.Builder b = CacheManagerOptions.builder();
assertThrows(IllegalArgumentException.class, () -> b.listFilesCache(0, Duration.ofMillis(-1)));
}
@Test
void cacheManagerRejectsNull() {
SessionContextBuilder b = SessionContext.builder();
assertThrows(IllegalArgumentException.class, () -> b.cacheManager(null));
}
@Test
void cacheManagerSetterReplacesPreviousValue() throws Exception {
// Setter replaces; doesn't merge. Calling fileMetadataCache(64) then
// listFilesCache(8) on a fresh builder must NOT carry the file-metadata
// setting forward.
byte[] bytes =
SessionContext.builder()
.cacheManager(CacheManagerOptions.builder().fileMetadataCache(64L << 20).build())
.cacheManager(CacheManagerOptions.builder().listFilesCache(8L << 20, null).build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
CacheManagerOptionsProto cm = parsed.getCacheManager();
// Only list-files survives -- the second .cacheManager(...) call replaced
// the first.
assertFalse(cm.hasFileMetadataCacheMaxBytes());
assertTrue(cm.hasListFilesCache());
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/ColumnarValueTest.java 0000664 0000000 0000000 00000005324 15205164533 0032057 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.junit.jupiter.api.Test;
class ColumnarValueTest {
private static final ArrowType INT32 = new ArrowType.Int(32, true);
@Test
void array_factory_returnsArrayVariant() {
try (BufferAllocator allocator = new RootAllocator();
IntVector v = new IntVector("v", allocator)) {
v.allocateNew(3);
v.setValueCount(3);
ColumnarValue cv = ColumnarValue.array(v);
assertSame(v, cv.vector());
assertEquals(INT32, cv.dataType());
}
}
@Test
void scalar_factory_returnsScalarVariant() {
try (BufferAllocator allocator = new RootAllocator();
IntVector v = new IntVector("v", allocator)) {
v.allocateNew(1);
v.set(0, 42);
v.setValueCount(1);
ColumnarValue cv = ColumnarValue.scalar(v);
assertSame(v, cv.vector());
assertEquals(INT32, cv.dataType());
}
}
@Test
void scalar_factory_rejectsNonOneLength() {
try (BufferAllocator allocator = new RootAllocator();
IntVector v = new IntVector("v", allocator)) {
v.allocateNew(2);
v.setValueCount(2);
IllegalArgumentException ex =
assertThrows(IllegalArgumentException.class, () -> ColumnarValue.scalar(v));
assertEquals("Scalar vector must have valueCount == 1, got 2", ex.getMessage());
}
}
@Test
void array_factory_rejectsNull() {
assertThrows(NullPointerException.class, () -> ColumnarValue.array(null));
}
@Test
void scalar_factory_rejectsNull() {
assertThrows(NullPointerException.class, () -> ColumnarValue.scalar(null));
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/CsvReadOptionsTest.java 0000664 0000000 0000000 00000011062 15205164533 0032201 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.protobuf.CsvReadOptionsProto;
import org.junit.jupiter.api.Test;
import com.google.protobuf.InvalidProtocolBufferException;
class CsvReadOptionsTest {
@Test
void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
CsvReadOptionsProto p = CsvReadOptionsProto.parseFrom(new CsvReadOptions().toBytes());
assertTrue(p.getHasHeader());
assertEquals((int) ',', p.getDelimiter());
assertEquals((int) '"', p.getQuote());
assertEquals(".csv", p.getFileExtension());
assertEquals(
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED,
p.getFileCompressionType());
assertFalse(p.hasTerminator());
assertFalse(p.hasEscape());
assertFalse(p.hasComment());
assertFalse(p.hasNewlinesInValues());
assertFalse(p.hasSchemaInferMaxRecords());
}
@Test
void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferException {
CsvReadOptions opts =
new CsvReadOptions()
.hasHeader(false)
.delimiter((byte) '|')
.quote((byte) '\'')
.terminator((byte) '\n')
.escape((byte) '\\')
.comment((byte) '#')
.newlinesInValues(true)
.schemaInferMaxRecords(10L)
.fileExtension(".tsv")
.fileCompressionType(FileCompressionType.GZIP);
CsvReadOptionsProto p = CsvReadOptionsProto.parseFrom(opts.toBytes());
assertFalse(p.getHasHeader());
assertEquals((int) '|', p.getDelimiter());
assertEquals((int) '\'', p.getQuote());
assertEquals((int) '\n', p.getTerminator());
assertEquals((int) '\\', p.getEscape());
assertEquals((int) '#', p.getComment());
assertTrue(p.getNewlinesInValues());
assertEquals(10L, p.getSchemaInferMaxRecords());
assertEquals(".tsv", p.getFileExtension());
assertEquals(
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP,
p.getFileCompressionType());
}
@Test
void schemaIsHeldByReferenceAndNotInProto() {
Schema schema =
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
CsvReadOptions opts = new CsvReadOptions().schema(schema);
assertSame(schema, opts.schema());
}
@Test
void allCompressionTypesMapThroughProto() throws InvalidProtocolBufferException {
for (FileCompressionType t : FileCompressionType.values()) {
CsvReadOptionsProto p =
CsvReadOptionsProto.parseFrom(new CsvReadOptions().fileCompressionType(t).toBytes());
assertEquals(
"FILE_COMPRESSION_TYPE_" + t.name(),
p.getFileCompressionType().name(),
"mismatch for " + t);
}
}
@Test
void schemaInferMaxRecordsRejectsNegative() {
// The proto wire field is uint64, so a negative long would be reinterpreted
// as a huge unsigned value on the Rust side and silently expand schema
// inference across the full dataset. Reject at the Java setter instead.
CsvReadOptions opts = new CsvReadOptions();
assertThrows(IllegalArgumentException.class, () -> opts.schemaInferMaxRecords(-1L));
assertThrows(IllegalArgumentException.class, () -> opts.schemaInferMaxRecords(Long.MIN_VALUE));
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/CsvWriteOptionsTest.java 0000664 0000000 0000000 00000006351 15205164533 0032425 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.datafusion.protobuf.CsvWriteOptionsProto;
import org.junit.jupiter.api.Test;
class CsvWriteOptionsTest {
@Test
void defaultsLeaveEverythingUnset() throws Exception {
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(new CsvWriteOptions().toBytes());
assertFalse(p.hasSingleFileOutput());
assertEquals(0, p.getPartitionColsCount());
assertFalse(p.hasHasHeader());
assertFalse(p.hasDelimiter());
assertFalse(p.hasQuote());
assertFalse(p.hasEscape());
assertFalse(p.hasNullValue());
assertFalse(p.hasFileCompressionType());
}
@Test
void fluentSettersRoundTripThroughProto() throws Exception {
CsvWriteOptions opts =
new CsvWriteOptions()
.singleFileOutput(true)
.partitionCols("region", "year")
.hasHeader(false)
.delimiter((byte) '|')
.quote((byte) '\'')
.escape((byte) '\\')
.nullValue("\\N")
.fileCompressionType(FileCompressionType.GZIP);
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(opts.toBytes());
assertTrue(p.getSingleFileOutput());
assertEquals(2, p.getPartitionColsCount());
assertEquals("region", p.getPartitionCols(0));
assertEquals("year", p.getPartitionCols(1));
assertFalse(p.getHasHeader());
assertEquals((int) '|', p.getDelimiter());
assertEquals((int) '\'', p.getQuote());
assertEquals((int) '\\', p.getEscape());
assertEquals("\\N", p.getNullValue());
assertEquals(
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP,
p.getFileCompressionType());
}
@Test
void partitionColsResetOnSubsequentCalls() throws Exception {
CsvWriteOptions opts = new CsvWriteOptions().partitionCols("a", "b", "c").partitionCols("only");
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(opts.toBytes());
assertEquals(1, p.getPartitionColsCount());
assertEquals("only", p.getPartitionCols(0));
}
@Test
void delimiterAcceptsHighByteWithoutSignExtension() throws Exception {
CsvWriteOptions opts = new CsvWriteOptions().delimiter((byte) 0xC2);
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(opts.toBytes());
assertEquals(0xC2, p.getDelimiter());
}
}
DataFrameExecuteStreamTest.java 0000664 0000000 0000000 00000017122 15205164533 0033545 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class DataFrameExecuteStreamTest {
/**
* Write a CSV with `rows` integer rows, one column `x`. Used in tests that need a real file scan
* so DataFusion's batching honors {@code batch_size} -- in-memory {@code VALUES} plans get
* coalesced into a single batch in some DataFusion versions, which would make those tests
* brittle.
*/
private static Path writeRowsCsv(Path dir, int rows) throws IOException {
StringBuilder sb = new StringBuilder("x\n");
for (int i = 1; i <= rows; i++) {
sb.append(i).append('\n');
}
Path file = dir.resolve("rows.csv");
Files.writeString(file, sb.toString());
return file;
}
@Test
void executeStreamYieldsTheSameRowsAsCollect() throws Exception {
String sql = "SELECT * FROM (VALUES (1), (2), (3), (4), (5)) AS t(x)";
long collected = 0;
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(sql);
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
collected += reader.getVectorSchemaRoot().getRowCount();
}
}
long streamed = 0;
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(sql);
ArrowReader reader = df.executeStream(allocator)) {
while (reader.loadNextBatch()) {
streamed += reader.getVectorSchemaRoot().getRowCount();
}
}
assertEquals(5L, collected);
assertEquals(collected, streamed);
}
@Test
void executeStreamConsumesTheDataFrame() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
DataFrame df = ctx.sql("SELECT 1");
try (ArrowReader reader = df.executeStream(allocator)) {
assertTrue(reader.loadNextBatch());
}
// After a successful executeStream, the DataFrame's native handle is
// released. A second collect/executeStream/count must throw.
assertThrows(IllegalStateException.class, () -> df.executeStream(allocator));
assertThrows(IllegalStateException.class, () -> df.collect(allocator));
assertThrows(IllegalStateException.class, df::count);
// close() on an already-streamed DataFrame is a no-op (no double-free).
df.close();
}
}
@Test
void executeStreamReadsBatchByBatch(@TempDir Path tempDir) throws Exception {
// CSV with 5 rows scanned at batch_size=2 reliably yields multiple batches
// across DataFusion versions, where an in-memory VALUES plan can be
// coalesced into a single batch by the planner. The point of this test is
// to pin "executeStream actually streams" without coupling to planner
// batching behavior that may shift in upstream releases.
Path csv = writeRowsCsv(tempDir, 5);
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = SessionContext.builder().batchSize(2).build()) {
ctx.registerCsv("rows", csv.toAbsolutePath().toString());
try (DataFrame df = ctx.sql("SELECT x FROM rows");
ArrowReader reader = df.executeStream(allocator)) {
int batches = 0;
long total = 0;
int maxBatchSize = 0;
while (reader.loadNextBatch()) {
batches++;
VectorSchemaRoot root = reader.getVectorSchemaRoot();
total += root.getRowCount();
maxBatchSize = Math.max(maxBatchSize, root.getRowCount());
}
assertEquals(5L, total);
assertTrue(batches >= 2, "expected multiple batches with batchSize=2, got " + batches);
assertTrue(maxBatchSize <= 2, "expected each batch <= 2 rows, got " + maxBatchSize);
}
}
}
@Test
void executeStreamSurvivesEarlyClose() throws Exception {
// Close the reader after the first batch and confirm no native panic /
// resource leak. The DataFrame is already consumed; explicit close on it
// must remain a no-op.
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = SessionContext.builder().batchSize(1).build();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)");
ArrowReader reader = df.executeStream(allocator)) {
assertTrue(reader.loadNextBatch());
}
}
@Test
void executeStreamOverParquetMatchesCollectRowCount() throws Exception {
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
Assumptions.assumeTrue(
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerParquet("lineitem", lineitem.toAbsolutePath().toString());
long collected;
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM lineitem");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector v = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
collected = v.get(0);
}
long streamed = 0;
try (DataFrame df = ctx.sql("SELECT l_orderkey FROM lineitem");
ArrowReader reader = df.executeStream(allocator)) {
while (reader.loadNextBatch()) {
streamed += reader.getVectorSchemaRoot().getRowCount();
}
}
assertEquals(collected, streamed);
}
}
@Test
void executeStreamColumnValuesAreCorrect() throws Exception {
// Pin actual cell values, not just row counts: a regression that
// shipped wrong values per batch must be caught.
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = SessionContext.builder().batchSize(2).build();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (10), (20), (30), (40)) AS t(x) ORDER BY x");
ArrowReader reader = df.executeStream(allocator)) {
java.util.List seen = new java.util.ArrayList<>();
while (reader.loadNextBatch()) {
BigIntVector v = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
for (int i = 0; i < v.getValueCount(); i++) {
seen.add(v.get(i));
}
}
assertEquals(java.util.List.of(10L, 20L, 30L, 40L), seen);
}
}
}
DataFrameIntrospectionTest.java 0000664 0000000 0000000 00000021154 15205164533 0033627 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.HashSet;
import java.util.Set;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.Test;
class DataFrameIntrospectionTest {
@Test
void schemaReturnsArrowSchema() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x, 'foo' AS s")) {
Schema schema = df.schema();
assertEquals(2, schema.getFields().size());
assertEquals("x", schema.getFields().get(0).getName());
assertTrue(schema.getFields().get(0).getType() instanceof ArrowType.Int);
assertEquals("s", schema.getFields().get(1).getName());
assertTrue(schema.getFields().get(1).getType() instanceof ArrowType.Utf8);
}
}
@Test
void schemaPreservesReceiver() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)")) {
Schema schema = df.schema();
assertNotNull(schema);
// Receiver still usable after schema().
assertEquals(3L, df.count());
}
}
@Test
void schemaThrowsAfterClose() {
try (SessionContext ctx = new SessionContext()) {
DataFrame df = ctx.sql("SELECT 1");
df.close();
assertThrows(IllegalStateException.class, df::schema);
}
}
@Test
void explainReturnsHumanReadablePlan() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT x FROM (VALUES (1), (2), (3)) AS t(x) WHERE x > 1");
DataFrame explained = df.explain(false, false);
ArrowReader reader = explained.collect(allocator)) {
StringBuilder labels = new StringBuilder();
StringBuilder bodies = new StringBuilder();
while (reader.loadNextBatch()) {
VectorSchemaRoot root = reader.getVectorSchemaRoot();
// Column 0 = plan-type label, Column 1 = plan text.
VarCharVector planType = (VarCharVector) root.getVector(0);
VarCharVector planText = (VarCharVector) root.getVector(1);
for (int i = 0; i < root.getRowCount(); i++) {
labels.append(new String(planType.get(i))).append('\n');
bodies.append(new String(planText.get(i))).append('\n');
}
}
String labelsDump = labels.toString();
String bodiesDump = bodies.toString();
assertTrue(labelsDump.contains("logical_plan"), () -> "labels: " + labelsDump);
assertTrue(bodiesDump.contains("Filter"), () -> "bodies: " + bodiesDump);
assertTrue(bodiesDump.contains("Projection"), () -> "bodies: " + bodiesDump);
}
}
@Test
void explainVerboseAnalyzeProducesMoreRowsThanLazy() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
long lazyRows;
try (DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)");
DataFrame e = df.explain(false, false);
ArrowReader reader = e.collect(allocator)) {
lazyRows = countAllRows(reader);
}
long verboseRows;
try (DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)");
DataFrame e = df.explain(true, true);
ArrowReader reader = e.collect(allocator)) {
verboseRows = countAllRows(reader);
}
assertTrue(
verboseRows > lazyRows,
() -> "verbose+analyze rows " + verboseRows + " not greater than lazy " + lazyRows);
}
}
@Test
void explainPreservesReceiver() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)")) {
try (DataFrame e = df.explain(false, false)) {
// Just ensure it's a valid handle.
assertNotNull(e.schema());
}
// Receiver still usable.
assertEquals(2L, df.count());
}
}
@Test
void explainThrowsAfterClose() {
try (SessionContext ctx = new SessionContext()) {
DataFrame df = ctx.sql("SELECT 1");
df.close();
assertThrows(IllegalStateException.class, () -> df.explain(false, false));
}
}
@Test
void cacheMaterializesPlan() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT x FROM (VALUES (1), (2), (3), (4)) AS t(x) WHERE x > 1");
DataFrame cached = df.cache()) {
assertEquals(3L, cached.count());
// Schema is preserved after caching.
Schema schema = cached.schema();
assertEquals(1, schema.getFields().size());
assertEquals("x", schema.getFields().get(0).getName());
}
}
@Test
void cachePreservesReceiver() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)")) {
try (DataFrame cached = df.cache()) {
assertEquals(2L, cached.count());
}
// Receiver still usable after cache().
assertEquals(2L, df.count());
}
}
@Test
void cacheThrowsAfterClose() {
try (SessionContext ctx = new SessionContext()) {
DataFrame df = ctx.sql("SELECT 1");
df.close();
assertThrows(IllegalStateException.class, df::cache);
}
}
@Test
void describeReturnsSummaryStats() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1, 'a'), (2, 'b'), (3, 'c')) AS t(x, s)");
DataFrame stats = df.describe()) {
// First column of describe() output is named "describe" and lists the statistic name.
Schema schema = stats.schema();
assertEquals("describe", schema.getFields().get(0).getName());
Set seen = new HashSet<>();
try (ArrowReader reader = stats.collect(allocator)) {
while (reader.loadNextBatch()) {
VectorSchemaRoot root = reader.getVectorSchemaRoot();
VarCharVector labels = (VarCharVector) root.getVector(0);
for (int i = 0; i < root.getRowCount(); i++) {
seen.add(new String(labels.get(i)));
}
}
}
// DataFusion 53.1 reports these seven labels.
assertTrue(seen.contains("count"), () -> "labels=" + seen);
assertTrue(seen.contains("null_count"), () -> "labels=" + seen);
assertTrue(seen.contains("mean"), () -> "labels=" + seen);
assertTrue(seen.contains("std"), () -> "labels=" + seen);
assertTrue(seen.contains("min"), () -> "labels=" + seen);
assertTrue(seen.contains("max"), () -> "labels=" + seen);
assertTrue(seen.contains("median"), () -> "labels=" + seen);
}
}
@Test
void describePreservesReceiver() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)")) {
try (DataFrame stats = df.describe()) {
assertNotNull(stats.schema());
}
// Receiver still usable after describe().
assertEquals(3L, df.count());
}
}
@Test
void describeThrowsAfterClose() {
try (SessionContext ctx = new SessionContext()) {
DataFrame df = ctx.sql("SELECT 1");
df.close();
assertThrows(IllegalStateException.class, df::describe);
}
}
private static long countAllRows(ArrowReader reader) throws Exception {
long n = 0;
while (reader.loadNextBatch()) {
n += reader.getVectorSchemaRoot().getRowCount();
}
return n;
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/DataFrameJoinTest.java 0000664 0000000 0000000 00000026443 15205164533 0031753 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test;
class DataFrameJoinTest {
// Two relations with one matching key (1, 2) and a few unmatched rows on each side.
// left: (1,'a'), (2,'b'), (3,'c'); right: (1,10), (2,20), (4,40).
private static final String LEFT_SQL =
"SELECT * FROM (VALUES (1, 'a'), (2, 'b'), (3, 'c')) AS l(id, s)";
private static final String RIGHT_SQL =
"SELECT * FROM (VALUES (1, 10), (2, 20), (4, 40)) AS r(id, v)";
@Test
void innerJoinOnSingleColumn() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.INNER, new String[] {"id"}, new String[] {"id"})) {
assertEquals(2L, joined.count()); // (1,'a',1,10) and (2,'b',2,20)
}
}
@Test
void innerJoinOnMultipleColumns() {
try (SessionContext ctx = new SessionContext();
DataFrame l =
ctx.sql("SELECT * FROM (VALUES (1, 'x', 100), (2, 'y', 200)) AS t(a, b, l_other)");
DataFrame r =
ctx.sql("SELECT * FROM (VALUES (1, 'x', 'p'), (2, 'z', 'q')) AS t(a2, b2, r_other)");
DataFrame joined =
l.join(r, JoinType.INNER, new String[] {"a", "b"}, new String[] {"a2", "b2"})) {
assertEquals(1L, joined.count()); // only (1,'x') matches on both keys
}
}
@Test
void leftJoinPreservesUnmatchedLeft() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.LEFT, new String[] {"id"}, new String[] {"id"})) {
// 3 left rows; unmatched (3,'c') gets nulls on the right side.
assertEquals(3L, joined.count());
}
}
@Test
void rightJoinPreservesUnmatchedRight() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.RIGHT, new String[] {"id"}, new String[] {"id"})) {
// 3 right rows; unmatched (4,40) gets nulls on the left side.
assertEquals(3L, joined.count());
}
}
@Test
void fullJoinPreservesBothSides() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.FULL, new String[] {"id"}, new String[] {"id"})) {
// 2 matched rows + 1 unmatched-left + 1 unmatched-right = 4.
assertEquals(4L, joined.count());
}
}
@Test
void leftSemiJoinReturnsLeftMatchedOnly() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.LEFT_SEMI, new String[] {"id"}, new String[] {"id"})) {
// Only the 2 left rows that have a matching right row.
// Output projects left side only (id, s) — right columns dropped.
assertEquals(2L, joined.count());
}
}
@Test
void leftAntiJoinReturnsLeftUnmatchedOnly() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.LEFT_ANTI, new String[] {"id"}, new String[] {"id"})) {
// Only the 1 left row (3,'c') with no right match. Output projects left side only.
assertEquals(1L, joined.count());
}
}
@Test
void rightSemiJoinReturnsRightMatchedOnly() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.RIGHT_SEMI, new String[] {"id"}, new String[] {"id"})) {
// Output projects right side only (id, v) — left columns dropped.
assertEquals(2L, joined.count());
}
}
@Test
void rightAntiJoinReturnsRightUnmatchedOnly() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.RIGHT_ANTI, new String[] {"id"}, new String[] {"id"})) {
assertEquals(1L, joined.count()); // (4, 40)
}
}
@Test
void leftMarkJoinAddsMarkColumn() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.LEFT_MARK, new String[] {"id"}, new String[] {"id"})) {
// One row per left row, plus a 'mark' boolean column.
assertEquals(3L, joined.count());
}
}
@Test
void joinWithResidualFilter() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined =
left.join(right, JoinType.INNER, new String[] {"id"}, new String[] {"id"}, "v >= 20")) {
// Without the filter: 2 matched rows. With v >= 20: only (2,'b',2,20).
assertEquals(1L, joined.count());
}
}
@Test
void joinOnSingleEqualityPredicate() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined = left.joinOn(right, JoinType.INNER, "l.id = r.id")) {
assertEquals(2L, joined.count());
}
}
@Test
void joinOnInequalityPredicate() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined = left.joinOn(right, JoinType.INNER, "l.id < r.id")) {
// Pairs (1,'a')<(2,20), (1,'a')<(4,40), (2,'b')<(4,40), (3,'c')<(4,40) = 4.
assertEquals(4L, joined.count());
}
}
@Test
void joinOnMultiplePredicates() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL);
DataFrame joined = left.joinOn(right, JoinType.INNER, "l.id = r.id", "r.v > 15")) {
// Equality narrows to (1,'a',1,10) and (2,'b',2,20); v > 15 leaves only the second.
assertEquals(1L, joined.count());
}
}
@Test
void semiJoinWithFilterToleratesSharedUnqualifiedColumn() {
// Regression for issue surfaced in code review: when both inputs carry an unqualified
// column with the same name (here, `tag`) that the residual filter does NOT reference,
// the join must still plan. Earlier the Rust side merged the schemas via
// DFSchema::join, whose check_names rejected the duplicate before parsing the filter.
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql("SELECT 1 AS id, 'l' AS tag");
DataFrame right = ctx.sql("SELECT 1 AS rid, 99 AS rv, 'r' AS tag");
DataFrame joined =
left.join(
right, JoinType.LEFT_SEMI, new String[] {"id"}, new String[] {"rid"}, "rv > 0")) {
assertEquals(1L, joined.count());
}
}
@Test
void joinOnToleratesSharedUnqualifiedColumn() {
// Same regression as the previous test, but exercises the joinOn predicate path.
// Uses LEFT_SEMI so the output schema is one-sided -- INNER joins on inputs that share
// an unqualified column name are genuinely ambiguous in the result and rejected by
// upstream's build_join_schema, which is not specific to our code.
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql("SELECT 1 AS id, 'l' AS tag");
DataFrame right = ctx.sql("SELECT 1 AS rid, 99 AS rv, 'r' AS tag");
DataFrame joined = left.joinOn(right, JoinType.LEFT_SEMI, "id = rid", "rv > 0")) {
assertEquals(1L, joined.count());
}
}
@Test
void joinPreservesReceivers() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL)) {
try (DataFrame joined =
left.join(right, JoinType.INNER, new String[] {"id"}, new String[] {"id"})) {
assertEquals(2L, joined.count());
}
// Both receivers still usable after join().
assertEquals(3L, left.count());
assertEquals(3L, right.count());
}
}
@Test
void joinThrowsWhenLeftClosed() {
try (SessionContext ctx = new SessionContext();
DataFrame right = ctx.sql(RIGHT_SQL)) {
DataFrame left = ctx.sql(LEFT_SQL);
left.close();
assertThrows(
IllegalStateException.class,
() -> left.join(right, JoinType.INNER, new String[] {"id"}, new String[] {"id"}));
}
}
@Test
void joinThrowsWhenRightClosed() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL)) {
DataFrame right = ctx.sql(RIGHT_SQL);
right.close();
assertThrows(
IllegalStateException.class,
() -> left.join(right, JoinType.INNER, new String[] {"id"}, new String[] {"id"}));
}
}
@Test
void joinNullArgumentValidation() {
try (SessionContext ctx = new SessionContext();
DataFrame left = ctx.sql(LEFT_SQL);
DataFrame right = ctx.sql(RIGHT_SQL)) {
String[] cols = new String[] {"id"};
assertThrows(
IllegalArgumentException.class, () -> left.join(null, JoinType.INNER, cols, cols));
assertThrows(IllegalArgumentException.class, () -> left.join(right, null, cols, cols));
assertThrows(
IllegalArgumentException.class, () -> left.join(right, JoinType.INNER, null, cols));
assertThrows(
IllegalArgumentException.class, () -> left.join(right, JoinType.INNER, cols, null));
assertThrows(
IllegalArgumentException.class,
() -> left.join(right, JoinType.INNER, new String[] {"id", "id"}, cols));
assertThrows(
IllegalArgumentException.class, () -> left.join(right, JoinType.INNER, cols, cols, null));
assertThrows(IllegalArgumentException.class, () -> left.joinOn(null, JoinType.INNER, "1=1"));
assertThrows(IllegalArgumentException.class, () -> left.joinOn(right, null, "1=1"));
assertThrows(IllegalArgumentException.class, () -> left.joinOn(right, JoinType.INNER));
assertThrows(
IllegalArgumentException.class, () -> left.joinOn(right, JoinType.INNER, (String) null));
}
}
}
DataFrameTransformationsTest.java 0000664 0000000 0000000 00000065623 15205164533 0034171 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
class DataFrameTransformationsTest {
@Test
void countReturnsRowCount() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)")) {
assertEquals(3L, df.count());
}
}
@Test
void showDoesNotThrow() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)")) {
df.show();
}
}
@Test
void showWithLimitDoesNotThrow() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)")) {
df.show(0);
df.show(1);
df.show(1_000_000);
}
}
@Test
void selectProjectsAndReordersColumns() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b, 3 AS c");
DataFrame projected = source.select("b", "a");
ArrowReader reader = projected.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
assertArrayEquals(
new String[] {"b", "a"},
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
}
}
@Test
void selectIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b")) {
try (DataFrame first = source.select("a")) {
assertEquals(1L, first.count());
}
try (DataFrame second = source.select("b")) {
assertEquals(1L, second.count());
}
assertEquals(1L, source.count());
}
}
@Test
void filterRemovesRows() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (2), (3), (4)) AS t(x)");
DataFrame filtered = source.filter("x > 2")) {
assertEquals(2L, filtered.count());
}
}
@Test
void filterIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (2), (3), (4)) AS t(x)")) {
try (DataFrame filtered = source.filter("x > 2")) {
assertEquals(2L, filtered.count());
}
assertEquals(4L, source.count());
}
}
@Test
void chainFilterSelectCount() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b UNION ALL SELECT 10 AS a, 20 AS b");
DataFrame chained = source.filter("a > 5").select("b")) {
assertEquals(1L, chained.count());
}
}
@Test
void methodsThrowAfterClose() {
try (SessionContext ctx = new SessionContext()) {
DataFrame df = ctx.sql("SELECT 1 AS x");
df.close();
assertThrows(IllegalStateException.class, () -> df.select("x"));
assertThrows(IllegalStateException.class, () -> df.filter("x > 0"));
assertThrows(IllegalStateException.class, () -> df.limit(1));
assertThrows(IllegalStateException.class, () -> df.limit(0, 1));
assertThrows(IllegalStateException.class, df::distinct);
assertThrows(IllegalStateException.class, () -> df.dropColumns("x"));
assertThrows(IllegalStateException.class, () -> df.withColumnRenamed("x", "y"));
assertThrows(IllegalStateException.class, () -> df.withColumn("y", "x + 1"));
assertThrows(IllegalStateException.class, () -> df.unnestColumns("x"));
assertThrows(IllegalStateException.class, () -> df.sort(SortExpr.asc("x")));
assertThrows(IllegalStateException.class, () -> df.repartitionRoundRobin(1));
assertThrows(IllegalStateException.class, () -> df.repartitionHash(1, "x"));
assertThrows(IllegalStateException.class, df::count);
assertThrows(IllegalStateException.class, df::show);
assertThrows(IllegalStateException.class, () -> df.show(5));
}
}
@Test
void methodsThrowAfterCollect() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
try (ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
}
assertThrows(IllegalStateException.class, () -> df.select("x"));
assertThrows(IllegalStateException.class, () -> df.filter("x > 0"));
assertThrows(IllegalStateException.class, () -> df.limit(1));
assertThrows(IllegalStateException.class, () -> df.limit(0, 1));
assertThrows(IllegalStateException.class, df::distinct);
assertThrows(IllegalStateException.class, () -> df.dropColumns("x"));
assertThrows(IllegalStateException.class, () -> df.withColumnRenamed("x", "y"));
assertThrows(IllegalStateException.class, () -> df.withColumn("y", "x + 1"));
assertThrows(IllegalStateException.class, () -> df.unnestColumns("x"));
assertThrows(IllegalStateException.class, () -> df.sort(SortExpr.asc("x")));
assertThrows(IllegalStateException.class, () -> df.repartitionRoundRobin(1));
assertThrows(IllegalStateException.class, () -> df.repartitionHash(1, "x"));
assertThrows(IllegalStateException.class, df::count);
assertThrows(IllegalStateException.class, df::show);
assertThrows(IllegalStateException.class, () -> df.show(5));
}
}
@Test
void selectInvalidColumnThrows() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(RuntimeException.class, () -> df.select("not_a_column"));
}
}
@Test
void filterMalformedPredicateThrows() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(RuntimeException.class, () -> df.filter("this is not sql"));
}
}
@Test
void lineitemFilterCountAgainstSqlBaseline() throws Exception {
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
Assumptions.assumeTrue(
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
try (SessionContext ctx = new SessionContext()) {
ctx.registerParquet("lineitem", lineitem.toAbsolutePath().toString());
long viaDataFrame;
try (DataFrame df = ctx.sql("SELECT * FROM lineitem");
DataFrame filtered = df.filter("l_orderkey < 100")) {
viaDataFrame = filtered.count();
}
long viaSql;
try (BufferAllocator allocator = new RootAllocator();
DataFrame df = ctx.sql("SELECT COUNT(*) FROM lineitem WHERE l_orderkey < 100");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
viaSql = ((BigIntVector) root.getVector(0)).get(0);
}
assertEquals(viaSql, viaDataFrame);
}
}
@Test
void limitTakesFirstNRows() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (2), (3), (4), (5)) AS t(x)");
DataFrame limited = source.limit(2)) {
assertEquals(2L, limited.count());
}
}
@Test
void limitWithSkipDropsLeadingRows() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (2), (3), (4), (5)) AS t(x)");
DataFrame limited = source.limit(2, 2)) {
assertEquals(2L, limited.count());
}
}
@Test
void limitIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)")) {
try (DataFrame limited = source.limit(1)) {
assertEquals(1L, limited.count());
}
assertEquals(3L, source.count());
}
}
@Test
void limitRejectsNegativeArgs() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(IllegalArgumentException.class, () -> df.limit(-1));
assertThrows(IllegalArgumentException.class, () -> df.limit(-1, 0));
assertThrows(IllegalArgumentException.class, () -> df.limit(0, -1));
}
}
@Test
void distinctRemovesDuplicates() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (1), (2), (2), (3)) AS t(x)");
DataFrame deduped = source.distinct()) {
assertEquals(3L, deduped.count());
}
}
@Test
void distinctIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (1), (2)) AS t(x)")) {
try (DataFrame deduped = source.distinct()) {
assertEquals(2L, deduped.count());
}
assertEquals(3L, source.count());
}
}
@Test
void dropColumnsRemovesNamedColumns() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b, 3 AS c");
DataFrame dropped = source.dropColumns("b");
ArrowReader reader = dropped.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertArrayEquals(
new String[] {"a", "c"},
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
}
}
@Test
void dropColumnsIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b")) {
try (DataFrame dropped = source.dropColumns("a")) {
assertEquals(1L, dropped.count());
}
assertEquals(1L, source.count());
}
}
@Test
void dropColumnsSilentlyIgnoresUnknownNames() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x");
DataFrame dropped = df.dropColumns("not_a_column");
ArrowReader reader = dropped.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertArrayEquals(
new String[] {"x"},
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
}
}
@Test
void withColumnRenamedChangesColumnName() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b");
DataFrame renamed = source.withColumnRenamed("a", "alpha");
ArrowReader reader = renamed.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertArrayEquals(
new String[] {"alpha", "b"},
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
}
}
@Test
void withColumnRenamedIsNonDestructive() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b")) {
try (DataFrame renamed = source.withColumnRenamed("a", "alpha")) {
assertEquals(1L, renamed.count());
}
try (DataFrame again = source.select("a");
ArrowReader reader = again.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertArrayEquals(
new String[] {"a"},
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
}
}
}
@Test
void withColumnRenamedUnknownColumnIsNoOp() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x");
DataFrame renamed = df.withColumnRenamed("not_a_column", "y");
ArrowReader reader = renamed.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertArrayEquals(
new String[] {"x"},
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
}
}
@Test
void withColumnAppendsNewColumn() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b");
DataFrame extended = source.withColumn("c", "a + b");
ArrowReader reader = extended.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
assertArrayEquals(
new String[] {"a", "b", "c"},
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
assertEquals(3L, ((BigIntVector) root.getVector("c")).get(0));
}
}
@Test
void withColumnReplacesExistingColumn() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b");
DataFrame replaced = source.withColumn("b", "a * 10");
ArrowReader reader = replaced.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertArrayEquals(
new String[] {"a", "b"},
root.getSchema().getFields().stream().map(f -> f.getName()).toArray(String[]::new));
assertEquals(10L, ((BigIntVector) root.getVector("b")).get(0));
}
}
@Test
void withColumnIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS a, 2 AS b")) {
try (DataFrame extended = source.withColumn("c", "a + b")) {
assertEquals(1L, extended.count());
}
assertEquals(1L, source.count());
}
}
@Test
void withColumnUnknownColumnRefThrows() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(RuntimeException.class, () -> df.withColumn("y", "not_a_column + 1"));
}
}
@Test
void withColumnRejectsNullArgs() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(IllegalArgumentException.class, () -> df.withColumn(null, "x + 1"));
assertThrows(IllegalArgumentException.class, () -> df.withColumn("y", null));
}
}
@Test
void unnestColumnsExpandsList() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS id, [10, 20, 30] AS vals");
DataFrame unnested = source.unnestColumns("vals");
ArrowReader reader = unnested.collect(allocator)) {
int rows = 0;
while (reader.loadNextBatch()) {
rows += reader.getVectorSchemaRoot().getRowCount();
}
assertEquals(3, rows);
}
}
@Test
void unnestColumnsPreserveNullsTrueKeepsNullRow() throws Exception {
String sql =
"SELECT id, vals FROM (VALUES "
+ "(1, [10, 20]), "
+ "(2, CAST(NULL AS BIGINT[])), "
+ "(3, [30])) AS t(id, vals)";
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql(sql);
DataFrame unnested = source.unnestColumns(new UnnestOptions().preserveNulls(true), "vals");
ArrowReader reader = unnested.collect(allocator)) {
int rows = 0;
while (reader.loadNextBatch()) {
rows += reader.getVectorSchemaRoot().getRowCount();
}
assertEquals(4, rows);
}
}
@Test
void unnestColumnsPreserveNullsFalseDropsNullRow() throws Exception {
String sql =
"SELECT id, vals FROM (VALUES "
+ "(1, [10, 20]), "
+ "(2, CAST(NULL AS BIGINT[])), "
+ "(3, [30])) AS t(id, vals)";
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql(sql);
DataFrame unnested =
source.unnestColumns(new UnnestOptions().preserveNulls(false), "vals");
ArrowReader reader = unnested.collect(allocator)) {
int rows = 0;
while (reader.loadNextBatch()) {
rows += reader.getVectorSchemaRoot().getRowCount();
}
assertEquals(3, rows);
}
}
@Test
void unnestColumnsIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT 1 AS id, [10, 20] AS vals")) {
try (DataFrame unnested = source.unnestColumns("vals")) {
assertEquals(2L, unnested.count());
}
assertEquals(1L, source.count());
}
}
@Test
void unnestColumnsUnknownColumnThrows() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(RuntimeException.class, () -> df.unnestColumns("not_a_column"));
}
}
@Test
void unnestColumnsRejectsNullArgs() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x, [1, 2] AS vals")) {
assertThrows(
IllegalArgumentException.class, () -> df.unnestColumns((UnnestOptions) null, "vals"));
assertThrows(
IllegalArgumentException.class,
() -> df.unnestColumns(new UnnestOptions(), (String[]) null));
}
}
// -- sort -----------------------------------------------------------------
@Test
void sortAscOrdersAscending() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (3), (1), (2)) AS t(x)");
DataFrame sorted = df.sort(SortExpr.asc("x"));
ArrowReader reader = sorted.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
BigIntVector x = (BigIntVector) root.getVector("x");
assertEquals(3, x.getValueCount());
assertEquals(1L, x.get(0));
assertEquals(2L, x.get(1));
assertEquals(3L, x.get(2));
}
}
@Test
void sortDescOrdersDescending() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (3), (1), (2)) AS t(x)");
DataFrame sorted = df.sort(SortExpr.desc("x"));
ArrowReader reader = sorted.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
BigIntVector x = (BigIntVector) root.getVector("x");
assertEquals(3, x.getValueCount());
assertEquals(3L, x.get(0));
assertEquals(2L, x.get(1));
assertEquals(1L, x.get(2));
}
}
@Test
void sortMultipleKeysMixedDirection() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1, 1), (1, 2), (2, 1)) AS t(x, y)");
DataFrame sorted = df.sort(SortExpr.asc("x"), SortExpr.desc("y"));
ArrowReader reader = sorted.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
BigIntVector x = (BigIntVector) root.getVector("x");
BigIntVector y = (BigIntVector) root.getVector("y");
// (1,2), (1,1), (2,1) -- x asc, y desc within each x group.
assertEquals(1L, x.get(0));
assertEquals(2L, y.get(0));
assertEquals(1L, x.get(1));
assertEquals(1L, y.get(1));
assertEquals(2L, x.get(2));
assertEquals(1L, y.get(2));
}
}
@Test
void sortNullsFirstPlacesNullsFirst() throws Exception {
String sql =
"SELECT * FROM (VALUES (CAST(1 AS BIGINT)), (CAST(NULL AS BIGINT)),"
+ " (CAST(2 AS BIGINT))) AS t(x)";
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(sql);
DataFrame sorted = df.sort(SortExpr.asc("x").nullsFirst(true));
ArrowReader reader = sorted.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
BigIntVector x = (BigIntVector) root.getVector("x");
assertEquals(3, x.getValueCount());
assertTrue(x.isNull(0), "expected NULL first");
assertEquals(1L, x.get(1));
assertEquals(2L, x.get(2));
}
}
@Test
void sortNullsLastPlacesNullsLast() throws Exception {
String sql =
"SELECT * FROM (VALUES (CAST(1 AS BIGINT)), (CAST(NULL AS BIGINT)),"
+ " (CAST(2 AS BIGINT))) AS t(x)";
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(sql);
DataFrame sorted = df.sort(SortExpr.asc("x").nullsFirst(false));
ArrowReader reader = sorted.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
BigIntVector x = (BigIntVector) root.getVector("x");
assertEquals(3, x.getValueCount());
assertEquals(1L, x.get(0));
assertEquals(2L, x.get(1));
assertTrue(x.isNull(2), "expected NULL last");
}
}
@Test
void sortIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (3), (1), (2)) AS t(x)")) {
try (DataFrame sorted = source.sort(SortExpr.asc("x"))) {
assertEquals(3L, sorted.count());
}
assertEquals(3L, source.count());
}
}
@Test
void sortRejectsNullArgs() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(IllegalArgumentException.class, () -> df.sort((SortExpr[]) null));
assertThrows(IllegalArgumentException.class, () -> df.sort(new SortExpr[] {null}));
}
}
@Test
void sortUnknownColumnThrows() {
// Sort plan-builds lazily; DataFusion may not validate column names until the plan is
// actually executed. Drive a count() to force planning + optimization.
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(
RuntimeException.class,
() -> {
try (DataFrame sorted = df.sort(SortExpr.asc("not_a_column"))) {
sorted.count();
}
});
}
}
// -- repartitionRoundRobin -------------------------------------------------
@Test
void repartitionRoundRobinPreservesRows() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (2), (3), (4), (5)) AS t(x)");
DataFrame repartitioned = source.repartitionRoundRobin(3)) {
assertEquals(5L, repartitioned.count());
}
}
@Test
void repartitionRoundRobinIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)")) {
try (DataFrame rp = source.repartitionRoundRobin(2)) {
assertEquals(3L, rp.count());
}
assertEquals(3L, source.count());
}
}
@Test
void repartitionRoundRobinRejectsNonPositive() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(IllegalArgumentException.class, () -> df.repartitionRoundRobin(0));
assertThrows(IllegalArgumentException.class, () -> df.repartitionRoundRobin(-1));
}
}
// -- repartitionHash -------------------------------------------------------
@Test
void repartitionHashPreservesRows() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (2), (3), (4), (5)) AS t(x)");
DataFrame repartitioned = source.repartitionHash(3, "x")) {
assertEquals(5L, repartitioned.count());
}
}
@Test
void repartitionHashIsNonDestructive() {
try (SessionContext ctx = new SessionContext();
DataFrame source = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)")) {
try (DataFrame rp = source.repartitionHash(2, "x")) {
assertEquals(3L, rp.count());
}
assertEquals(3L, source.count());
}
}
@Test
void repartitionHashRejectsBadArgs() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(IllegalArgumentException.class, () -> df.repartitionHash(0, "x"));
assertThrows(IllegalArgumentException.class, () -> df.repartitionHash(-1, "x"));
assertThrows(IllegalArgumentException.class, () -> df.repartitionHash(2, (String[]) null));
assertThrows(IllegalArgumentException.class, () -> df.repartitionHash(2));
assertThrows(
IllegalArgumentException.class, () -> df.repartitionHash(2, new String[] {null}));
}
}
@Test
void repartitionHashUnknownColumnThrows() {
// Repartition plan-builds lazily; DataFusion may not validate column names until the plan
// is actually executed. Drive a count() to force planning + optimization.
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1 AS x")) {
assertThrows(
RuntimeException.class,
() -> {
try (DataFrame rp = df.repartitionHash(2, "not_a_column")) {
rp.count();
}
});
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/DataFrameWriteCsvTest.java 0000664 0000000 0000000 00000016014 15205164533 0032613 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Stream;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class DataFrameWriteCsvTest {
private static Path writeCsv(Path dir, String name, String contents) throws IOException {
Path file = dir.resolve(name);
Files.writeString(file, contents);
return file;
}
private static long countRowsAt(Path dirOrFile, CsvReadOptions readOpts) throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerCsv("t", dirOrFile.toAbsolutePath().toString(), readOpts);
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM t");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
return ((BigIntVector) root.getVector(0)).get(0);
}
}
}
@Test
void writeCsvRoundTripsRowCount(@TempDir Path tempDir) throws Exception {
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n2,bob\n3,carol\n");
Path out = tempDir.resolve("out");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
df.writeCsv(out.toString());
}
assertEquals(3L, countRowsAt(out, new CsvReadOptions()));
}
@Test
void writeCsvDefaultsToDirectoryEvenWithExtensionInPath(@TempDir Path tempDir) throws Exception {
// The Javadoc promises "directory unless overridden via singleFileOutput(true)". DataFusion's
// own DataFrameWriteOptions defaults to Automatic mode, where an extension in the path
// (".csv" here) silently flips the output to a single file. The native handler explicitly
// pins the default to directory mode so this contract holds regardless of path shape.
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n2,bob\n3,carol\n");
Path out = tempDir.resolve("out.csv");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
df.writeCsv(out.toString());
}
assertTrue(Files.isDirectory(out), "expected directory output at " + out + ", got a file");
assertEquals(3L, countRowsAt(out, new CsvReadOptions()));
}
@Test
void writeCsvSingleFileProducesOneFile(@TempDir Path tempDir) throws Exception {
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n2,bob\n3,carol\n");
Path out = tempDir.resolve("out.csv");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
df.writeCsv(out.toString(), new CsvWriteOptions().singleFileOutput(true));
}
assertTrue(Files.isRegularFile(out), "expected single file at " + out);
assertEquals(3L, countRowsAt(out, new CsvReadOptions()));
}
@Test
void writeCsvWithCustomDelimiter(@TempDir Path tempDir) throws Exception {
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n2,bob\n");
Path out = tempDir.resolve("out.csv");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
df.writeCsv(
out.toString(), new CsvWriteOptions().singleFileOutput(true).delimiter((byte) '|'));
}
String written = Files.readString(out);
assertTrue(written.contains("|"), "expected pipe delimiter in output, got: " + written);
assertEquals(2L, countRowsAt(out, new CsvReadOptions().delimiter((byte) '|')));
}
@Test
void writeCsvWithGzipCompressionRoundTrips(@TempDir Path tempDir) throws Exception {
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n2,bob\n3,carol\n4,dan\n");
Path out = tempDir.resolve("gz-out");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
df.writeCsv(
out.toString(), new CsvWriteOptions().fileCompressionType(FileCompressionType.GZIP));
}
try (Stream stream = Files.walk(out)) {
List files = stream.filter(Files::isRegularFile).toList();
assertTrue(!files.isEmpty(), "expected at least one part-file under " + out);
for (Path p : files) {
assertTrue(
p.getFileName().toString().endsWith(".gz"),
"expected .gz suffix on " + p.getFileName());
}
}
CsvReadOptions readOpts =
new CsvReadOptions().fileCompressionType(FileCompressionType.GZIP).fileExtension(".csv.gz");
assertEquals(4L, countRowsAt(out, readOpts));
}
@Test
void writeCsvRetainsDataFrame(@TempDir Path tempDir) throws Exception {
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n2,bob\n3,carol\n");
Path out = tempDir.resolve("retained");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
df.writeCsv(out.toString());
assertEquals(3L, df.count());
}
}
@Test
void writeCsvRejectsNullPath(@TempDir Path tempDir) throws Exception {
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
assertThrows(IllegalArgumentException.class, () -> df.writeCsv(null));
}
}
@Test
void writeCsvRejectsNullOptions(@TempDir Path tempDir) throws Exception {
Path src = writeCsv(tempDir, "src.csv", "id,name\n1,alice\n");
Path out = tempDir.resolve("out");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readCsv(src.toAbsolutePath().toString())) {
assertThrows(IllegalArgumentException.class, () -> df.writeCsv(out.toString(), null));
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/DataFrameWriteJsonTest.java 0000664 0000000 0000000 00000013722 15205164533 0032774 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.stream.Stream;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class DataFrameWriteJsonTest {
/**
* Build a 3-row DataFrame {@code (id BIGINT, name VARCHAR)}, write it as JSON via the SUT, then
* read the result back through {@link SessionContext#registerJson} and run {@code COUNT(*)}.
* Returns the row count.
*/
private static long countRowsAt(Path dirOrFile, NdJsonReadOptions readOpts) throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerJson("t", dirOrFile.toAbsolutePath().toString(), readOpts);
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM t");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
return ((BigIntVector) root.getVector(0)).get(0);
}
}
}
/** Materialise a 3-row table the test can hand to {@code writeJson}. */
private static String threeRowSql() {
return "SELECT * FROM (VALUES (CAST(1 AS BIGINT), 'alice'),"
+ " (CAST(2 AS BIGINT), 'bob'), (CAST(3 AS BIGINT), 'carol')) AS t(id, name)";
}
@Test
void writeJsonRoundTripsRowCount(@TempDir Path tempDir) throws Exception {
Path out = tempDir.resolve("out");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(threeRowSql())) {
df.writeJson(out.toString());
}
assertEquals(3L, countRowsAt(out, new NdJsonReadOptions()));
}
@Test
void writeJsonSingleFileProducesOneFile(@TempDir Path tempDir) throws Exception {
Path out = tempDir.resolve("out.json");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(threeRowSql())) {
df.writeJson(out.toString(), new JsonWriteOptions().singleFileOutput(true));
}
assertTrue(Files.isRegularFile(out), "expected single file at " + out);
assertEquals(3L, countRowsAt(out, new NdJsonReadOptions()));
}
@Test
void writeJsonWithGzipCompressionRoundTrips(@TempDir Path tempDir) throws Exception {
Path out = tempDir.resolve("gz-out");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(threeRowSql())) {
df.writeJson(
out.toString(), new JsonWriteOptions().fileCompressionType(FileCompressionType.GZIP));
}
try (Stream stream = Files.walk(out)) {
List files = stream.filter(Files::isRegularFile).toList();
assertTrue(!files.isEmpty(), "expected at least one part-file under " + out);
for (Path p : files) {
assertTrue(
p.getFileName().toString().endsWith(".gz"),
"expected .gz suffix on " + p.getFileName());
}
}
NdJsonReadOptions readOpts =
new NdJsonReadOptions()
.fileCompressionType(FileCompressionType.GZIP)
.fileExtension(".json.gz");
assertEquals(3L, countRowsAt(out, readOpts));
}
@Test
void writeJsonDefaultsToDirectoryEvenWithExtensionInPath(@TempDir Path tempDir) throws Exception {
// The Javadoc promises "directory unless overridden via singleFileOutput(true)". DataFusion's
// own DataFrameWriteOptions defaults to Automatic mode, where an extension in the path
// (".json" here) silently flips the output to a single file. The native handler explicitly
// pins the default to directory mode so this contract holds regardless of path shape.
Path out = tempDir.resolve("out.json");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(threeRowSql())) {
df.writeJson(out.toString());
}
assertTrue(Files.isDirectory(out), "expected directory output at " + out + ", got a file");
assertEquals(3L, countRowsAt(out, new NdJsonReadOptions()));
}
@Test
void writeJsonRetainsDataFrame(@TempDir Path tempDir) throws Exception {
Path out = tempDir.resolve("retained");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(threeRowSql())) {
df.writeJson(out.toString());
assertEquals(3L, df.count());
}
}
@Test
void writeJsonRejectsNullPath() {
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(threeRowSql())) {
assertThrows(IllegalArgumentException.class, () -> df.writeJson(null));
}
}
@Test
void writeJsonRejectsNullOptions(@TempDir Path tempDir) {
Path out = tempDir.resolve("out");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql(threeRowSql())) {
assertThrows(IllegalArgumentException.class, () -> df.writeJson(out.toString(), null));
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/DataFrameWriteParquetTest.java0000664 0000000 0000000 00000012016 15205164533 0033477 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class DataFrameWriteParquetTest {
private static final Path LINEITEM = Path.of("tpch-data/sf1/lineitem.parquet");
private static final long LINEITEM_ROWS = 6_001_215L;
private static void assumeLineitem() {
Assumptions.assumeTrue(
Files.exists(LINEITEM), "TPC-H SF1 data not found; run `make tpch-data` first");
}
private static long countRowsAt(Path path) throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerParquet("t", path.toAbsolutePath().toString());
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM t");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
return ((BigIntVector) root.getVector(0)).get(0);
}
}
}
@Test
void writeParquetRoundTripsRowCount(@TempDir Path tempDir) throws Exception {
assumeLineitem();
Path out = tempDir.resolve("out");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readParquet(LINEITEM.toAbsolutePath().toString())) {
df.writeParquet(out.toString());
}
assertEquals(LINEITEM_ROWS, countRowsAt(out));
}
@Test
void writeParquetSingleFileProducesOneFile(@TempDir Path tempDir) throws Exception {
assumeLineitem();
Path out = tempDir.resolve("out.parquet");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readParquet(LINEITEM.toAbsolutePath().toString())) {
df.writeParquet(out.toString(), new ParquetWriteOptions().singleFileOutput(true));
}
assertTrue(Files.isRegularFile(out), "expected single file at " + out);
assertEquals(LINEITEM_ROWS, countRowsAt(out));
}
@Test
void writeParquetWithCompressionRoundTrips(@TempDir Path tempDir) throws Exception {
assumeLineitem();
Path out = tempDir.resolve("zstd-out");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readParquet(LINEITEM.toAbsolutePath().toString())) {
df.writeParquet(out.toString(), new ParquetWriteOptions().compression("zstd(3)"));
}
assertEquals(LINEITEM_ROWS, countRowsAt(out));
}
@Test
void writeParquetRetainsDataFrame(@TempDir Path tempDir) throws Exception {
assumeLineitem();
Path out = tempDir.resolve("retained");
try (SessionContext ctx = new SessionContext();
DataFrame df = ctx.readParquet(LINEITEM.toAbsolutePath().toString())) {
df.writeParquet(out.toString());
assertEquals(LINEITEM_ROWS, df.count());
}
}
@Test
void writeParquetDefaultsToDirectoryEvenWithExtensionInPath(@TempDir Path tempDir)
throws Exception {
// The Javadoc promises "directory unless overridden via singleFileOutput(true)". DataFusion's
// own DataFrameWriteOptions defaults to Automatic mode, where an extension in the path
// (".parquet" here) silently flips the output to a single file. The native handler explicitly
// pins the default to directory mode so this contract holds regardless of path shape.
//
// Uses an in-memory 3-row source so this regression test runs without TPC-H fixtures.
Path out = tempDir.resolve("out.parquet");
try (SessionContext ctx = new SessionContext();
DataFrame df =
ctx.sql(
"SELECT * FROM (VALUES (CAST(1 AS BIGINT), 'alice'), (CAST(2 AS BIGINT), 'bob'),"
+ " (CAST(3 AS BIGINT), 'carol')) AS t(id, name)")) {
df.writeParquet(out.toString());
}
assertTrue(Files.isDirectory(out), "expected directory output at " + out + ", got a file");
assertEquals(3L, countRowsAt(out));
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/DataFusionExceptionTest.java 0000664 0000000 0000000 00000007265 15205164533 0033224 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertSame;
import org.junit.jupiter.api.Test;
class DataFusionExceptionTest {
@Test
void parentExtendsRuntimeException() {
DataFusionException e = new DataFusionException("x");
assertInstanceOf(RuntimeException.class, e);
assertEquals("x", e.getMessage());
}
@Test
void parentCauseConstructorPropagatesCause() {
Throwable cause = new IllegalStateException("inner");
DataFusionException e = new DataFusionException("outer", cause);
assertEquals("outer", e.getMessage());
assertSame(cause, e.getCause());
}
@Test
void allSubclassesExtendDataFusionException() {
// Each subclass must extend the parent so callers can `catch
// (DataFusionException)` for "anything from DataFusion" and `catch
// (RuntimeException)` keeps working unchanged.
assertInstanceOf(DataFusionException.class, new PlanException("x"));
assertInstanceOf(DataFusionException.class, new ExecutionException("x"));
assertInstanceOf(DataFusionException.class, new ResourcesExhaustedException("x"));
assertInstanceOf(DataFusionException.class, new IoException("x"));
assertInstanceOf(DataFusionException.class, new NotImplementedException("x"));
assertInstanceOf(DataFusionException.class, new ConfigurationException("x"));
}
@Test
void allSubclassesExtendRuntimeException() {
// Existing callers use `catch (RuntimeException)`. A typed-exception PR
// that breaks them is a regression even if the new types are correct.
assertInstanceOf(RuntimeException.class, new PlanException("x"));
assertInstanceOf(RuntimeException.class, new ExecutionException("x"));
assertInstanceOf(RuntimeException.class, new ResourcesExhaustedException("x"));
assertInstanceOf(RuntimeException.class, new IoException("x"));
assertInstanceOf(RuntimeException.class, new NotImplementedException("x"));
assertInstanceOf(RuntimeException.class, new ConfigurationException("x"));
}
@Test
void allSubclassesAcceptCauseConstructor() {
// The (String, Throwable) constructor is the seam upstream issue #55
// plugs into for Java-throwable propagation from JVM upcalls. v1 has no
// production caller for it, but it must exist on every subclass so #55
// doesn't have to add it later.
Throwable cause = new IllegalStateException("c");
assertSame(cause, new PlanException("m", cause).getCause());
assertSame(cause, new ExecutionException("m", cause).getCause());
assertSame(cause, new ResourcesExhaustedException("m", cause).getCause());
assertSame(cause, new IoException("m", cause).getCause());
assertSame(cause, new NotImplementedException("m", cause).getCause());
assertSame(cause, new ConfigurationException("m", cause).getCause());
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/JsonWriteOptionsTest.java 0000664 0000000 0000000 00000004712 15205164533 0032602 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.datafusion.protobuf.JsonWriteOptionsProto;
import org.junit.jupiter.api.Test;
class JsonWriteOptionsTest {
@Test
void defaultsLeaveEverythingUnset() throws Exception {
JsonWriteOptionsProto p = JsonWriteOptionsProto.parseFrom(new JsonWriteOptions().toBytes());
assertFalse(p.hasSingleFileOutput());
assertEquals(0, p.getPartitionColsCount());
assertFalse(p.hasFileCompressionType());
}
@Test
void fluentSettersRoundTripThroughProto() throws Exception {
JsonWriteOptions opts =
new JsonWriteOptions()
.singleFileOutput(true)
.partitionCols("region", "year")
.fileCompressionType(FileCompressionType.GZIP);
JsonWriteOptionsProto p = JsonWriteOptionsProto.parseFrom(opts.toBytes());
assertTrue(p.getSingleFileOutput());
assertEquals(2, p.getPartitionColsCount());
assertEquals("region", p.getPartitionCols(0));
assertEquals("year", p.getPartitionCols(1));
assertEquals(
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP,
p.getFileCompressionType());
}
@Test
void partitionColsResetOnSubsequentCalls() throws Exception {
JsonWriteOptions opts =
new JsonWriteOptions().partitionCols("a", "b", "c").partitionCols("only");
JsonWriteOptionsProto p = JsonWriteOptionsProto.parseFrom(opts.toBytes());
assertEquals(1, p.getPartitionColsCount());
assertEquals("only", p.getPartitionCols(0));
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/NdJsonReadOptionsTest.java 0000664 0000000 0000000 00000007423 15205164533 0032647 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import java.util.List;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.protobuf.NdJsonReadOptionsProto;
import org.junit.jupiter.api.Test;
import com.google.protobuf.InvalidProtocolBufferException;
class NdJsonReadOptionsTest {
@Test
void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
NdJsonReadOptionsProto p = NdJsonReadOptionsProto.parseFrom(new NdJsonReadOptions().toBytes());
assertEquals(".json", p.getFileExtension());
assertEquals(
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED,
p.getFileCompressionType());
assertFalse(p.hasSchemaInferMaxRecords());
}
@Test
void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferException {
NdJsonReadOptions opts =
new NdJsonReadOptions()
.fileExtension(".ndjson")
.fileCompressionType(FileCompressionType.GZIP)
.schemaInferMaxRecords(50L);
NdJsonReadOptionsProto p = NdJsonReadOptionsProto.parseFrom(opts.toBytes());
assertEquals(".ndjson", p.getFileExtension());
assertEquals(
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP,
p.getFileCompressionType());
assertEquals(50L, p.getSchemaInferMaxRecords());
}
@Test
void schemaIsHeldByReferenceAndNotInProto() {
Schema schema =
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
NdJsonReadOptions opts = new NdJsonReadOptions().schema(schema);
assertSame(schema, opts.schema());
}
@Test
void allCompressionTypesMapThroughProto() throws InvalidProtocolBufferException {
for (FileCompressionType t : FileCompressionType.values()) {
NdJsonReadOptionsProto p =
NdJsonReadOptionsProto.parseFrom(
new NdJsonReadOptions().fileCompressionType(t).toBytes());
assertEquals(
"FILE_COMPRESSION_TYPE_" + t.name(),
p.getFileCompressionType().name(),
"mismatch for " + t);
}
}
@Test
void schemaInferMaxRecordsRejectsNegative() {
// The proto wire field is uint64, so a negative long would be reinterpreted
// as a huge unsigned value on the Rust side and silently expand schema
// inference across the full dataset. Reject at the Java setter instead.
NdJsonReadOptions opts = new NdJsonReadOptions();
assertThrows(IllegalArgumentException.class, () -> opts.schemaInferMaxRecords(-1L));
assertThrows(IllegalArgumentException.class, () -> opts.schemaInferMaxRecords(Long.MIN_VALUE));
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/ObjectStoreOptionsTest.java 0000664 0000000 0000000 00000015576 15205164533 0033113 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import org.apache.datafusion.protobuf.ObjectStoreRegistration;
import org.apache.datafusion.protobuf.SessionOptions;
import org.junit.jupiter.api.Test;
class ObjectStoreOptionsTest {
@Test
void s3FullSetterSurfaceRoundTripsThroughProto() throws Exception {
byte[] bytes =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.s3()
.bucket("orders")
.region("us-east-1")
.endpoint("https://minio.internal:9000")
.accessKeyId("AKIA...")
.secretAccessKey("...")
.sessionToken("...")
.allowHttp(true)
.skipSignature(false)
.imdsv1Fallback(true)
.build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
List regs = parsed.getObjectStoresList();
assertEquals(1, regs.size());
ObjectStoreRegistration r = regs.get(0);
assertTrue(r.hasS3());
assertFalse(r.hasUrl(), "default URL is derived on the Rust side, not sent on the wire");
assertEquals("orders", r.getS3().getBucket());
assertEquals("us-east-1", r.getS3().getRegion());
assertEquals("https://minio.internal:9000", r.getS3().getEndpoint());
assertEquals("AKIA...", r.getS3().getAccessKeyId());
assertTrue(r.getS3().getAllowHttp());
assertFalse(r.getS3().getSkipSignature());
assertTrue(r.getS3().getImdsv1Fallback());
}
@Test
void s3UnsetFieldsAreAbsentInProto() throws Exception {
byte[] bytes =
SessionContext.builder()
.registerObjectStore(ObjectStoreOptions.s3().bucket("b").build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
ObjectStoreRegistration r = parsed.getObjectStores(0);
assertTrue(r.hasS3());
assertFalse(r.hasUrl());
assertEquals("b", r.getS3().getBucket());
// Every optional must travel as unset, not as zero/empty — the Rust side
// distinguishes "leave the upstream default in place" from "explicitly 0".
assertFalse(r.getS3().hasRegion());
assertFalse(r.getS3().hasEndpoint());
assertFalse(r.getS3().hasAccessKeyId());
assertFalse(r.getS3().hasSecretAccessKey());
assertFalse(r.getS3().hasSessionToken());
assertFalse(r.getS3().hasAllowHttp());
assertFalse(r.getS3().hasSkipSignature());
assertFalse(r.getS3().hasImdsv1Fallback());
}
@Test
void s3UrlOverrideRidesOnTheRegistrationNotTheBackendMessage() throws Exception {
byte[] bytes =
SessionContext.builder()
.registerObjectStore(ObjectStoreOptions.s3().bucket("b").url("s3a://b").build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
ObjectStoreRegistration r = parsed.getObjectStores(0);
assertTrue(r.hasUrl());
assertEquals("s3a://b", r.getUrl());
}
@Test
void s3RequiresBucket() {
assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.s3().build());
assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.s3().bucket("").build());
}
@Test
void gcsRoundTripsThroughProto() throws Exception {
byte[] bytes =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.gcs()
.bucket("data")
.serviceAccountKey("{\"type\":\"service_account\"}")
.build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
ObjectStoreRegistration r = parsed.getObjectStores(0);
assertTrue(r.hasGcs());
assertEquals("data", r.getGcs().getBucket());
assertEquals("{\"type\":\"service_account\"}", r.getGcs().getServiceAccountKey());
assertFalse(r.getGcs().hasServiceAccountPath());
}
@Test
void gcsRequiresBucketAndRejectsConflictingCreds() {
assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.gcs().build());
assertThrows(
IllegalArgumentException.class,
() ->
ObjectStoreOptions.gcs()
.bucket("b")
.serviceAccountKey("{}")
.serviceAccountPath("/tmp/sa.json")
.build());
}
@Test
void httpRoundTripsThroughProto() throws Exception {
byte[] bytes =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.http("https://example.com/").allowHttp(false).build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
ObjectStoreRegistration r = parsed.getObjectStores(0);
assertTrue(r.hasHttp());
assertTrue(r.hasUrl());
assertEquals("https://example.com/", r.getUrl());
assertFalse(r.getHttp().getAllowHttp());
}
@Test
void httpRequiresUrl() {
assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.http(null));
}
@Test
void multipleRegistrationsPreserveInsertionOrder() throws Exception {
byte[] bytes =
SessionContext.builder()
.registerObjectStore(ObjectStoreOptions.s3().bucket("first").build())
.registerObjectStore(ObjectStoreOptions.gcs().bucket("second").build())
.registerObjectStore(ObjectStoreOptions.s3().bucket("third").build())
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
List regs = parsed.getObjectStoresList();
assertEquals(3, regs.size());
assertEquals("first", regs.get(0).getS3().getBucket());
assertEquals("second", regs.get(1).getGcs().getBucket());
assertEquals("third", regs.get(2).getS3().getBucket());
}
@Test
void registerObjectStoreRejectsNull() {
SessionContextBuilder b = SessionContext.builder();
assertThrows(IllegalArgumentException.class, () -> b.registerObjectStore(null));
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/ParquetReadOptionsTest.java 0000664 0000000 0000000 00000005255 15205164533 0033076 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.protobuf.ParquetReadOptionsProto;
import org.junit.jupiter.api.Test;
import com.google.protobuf.InvalidProtocolBufferException;
class ParquetReadOptionsTest {
@Test
void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
ParquetReadOptionsProto p =
ParquetReadOptionsProto.parseFrom(new ParquetReadOptions().toBytes());
assertEquals(".parquet", p.getFileExtension());
assertFalse(p.hasParquetPruning());
assertFalse(p.hasSkipMetadata());
assertFalse(p.hasMetadataSizeHint());
}
@Test
void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferException {
ParquetReadOptions opts =
new ParquetReadOptions()
.fileExtension(".par")
.parquetPruning(true)
.skipMetadata(false)
.metadataSizeHint(4096L);
ParquetReadOptionsProto p = ParquetReadOptionsProto.parseFrom(opts.toBytes());
assertEquals(".par", p.getFileExtension());
assertTrue(p.getParquetPruning());
assertFalse(p.getSkipMetadata());
assertEquals(4096L, p.getMetadataSizeHint());
}
@Test
void schemaIsHeldByReferenceAndNotInProto() {
Schema schema =
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
ParquetReadOptions opts = new ParquetReadOptions().schema(schema);
assertSame(schema, opts.schema());
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/ParquetWriteOptionsTest.java 0000664 0000000 0000000 00000002711 15205164533 0033307 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import org.junit.jupiter.api.Test;
class ParquetWriteOptionsTest {
@Test
void defaultsAreAllNull() {
ParquetWriteOptions opts = new ParquetWriteOptions();
assertNull(opts.compression());
assertNull(opts.singleFileOutput());
}
@Test
void fluentSettersChainAndMutate() {
ParquetWriteOptions opts =
new ParquetWriteOptions().compression("zstd(3)").singleFileOutput(true);
assertEquals("zstd(3)", opts.compression());
assertEquals(Boolean.TRUE, opts.singleFileOutput());
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/PlatformTest.java 0000664 0000000 0000000 00000010664 15205164533 0031071 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test;
class PlatformTest {
@Test
void detectsLinuxFromOsName() {
assertEquals(Platform.Os.LINUX, Platform.detectOs("Linux"));
assertEquals(Platform.Os.LINUX, Platform.detectOs("linux"));
}
@Test
void detectsDarwinFromOsName() {
assertEquals(Platform.Os.DARWIN, Platform.detectOs("Mac OS X"));
assertEquals(Platform.Os.DARWIN, Platform.detectOs("macOS"));
assertEquals(Platform.Os.DARWIN, Platform.detectOs("Darwin"));
}
@Test
void detectsWindowsFromOsName() {
assertEquals(Platform.Os.WINDOWS, Platform.detectOs("Windows 10"));
assertEquals(Platform.Os.WINDOWS, Platform.detectOs("Windows Server 2019"));
}
@Test
void rejectsUnknownOs() {
assertThrows(UnsupportedOperationException.class, () -> Platform.detectOs("Solaris"));
}
@Test
void rejectsNullOs() {
assertThrows(UnsupportedOperationException.class, () -> Platform.detectOs(null));
}
@Test
void usesAmd64OnLinuxForX86_64Aliases() {
assertEquals("amd64", Platform.detectArch(Platform.Os.LINUX, "amd64"));
assertEquals("amd64", Platform.detectArch(Platform.Os.LINUX, "x86_64"));
assertEquals("amd64", Platform.detectArch(Platform.Os.LINUX, "x64"));
}
@Test
void usesX86_64OnDarwinForX86_64Aliases() {
assertEquals("x86_64", Platform.detectArch(Platform.Os.DARWIN, "amd64"));
assertEquals("x86_64", Platform.detectArch(Platform.Os.DARWIN, "x86_64"));
}
@Test
void usesAmd64OnWindowsForX86_64Aliases() {
assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "amd64"));
assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "x86_64"));
assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "x64"));
}
@Test
void usesAarch64ForArm64Aliases() {
assertEquals("aarch64", Platform.detectArch(Platform.Os.LINUX, "aarch64"));
assertEquals("aarch64", Platform.detectArch(Platform.Os.LINUX, "arm64"));
assertEquals("aarch64", Platform.detectArch(Platform.Os.DARWIN, "aarch64"));
assertEquals("aarch64", Platform.detectArch(Platform.Os.DARWIN, "arm64"));
}
@Test
void rejectsUnknownArch() {
assertThrows(
UnsupportedOperationException.class,
() -> Platform.detectArch(Platform.Os.LINUX, "ppc64le"));
}
@Test
void libFileNameUsesPlatformConventions() {
assertEquals("libdatafusion_jni.so", Platform.of("Linux", "amd64").libFileName());
assertEquals("libdatafusion_jni.so", Platform.of("Linux", "aarch64").libFileName());
assertEquals("libdatafusion_jni.dylib", Platform.of("Mac OS X", "x86_64").libFileName());
assertEquals("libdatafusion_jni.dylib", Platform.of("Mac OS X", "aarch64").libFileName());
assertEquals("datafusion_jni.dll", Platform.of("Windows 11", "amd64").libFileName());
}
@Test
void resourcePathMatchesSpec() {
assertEquals(
"/org/apache/datafusion/linux/amd64/libdatafusion_jni.so",
Platform.of("Linux", "amd64").resourcePath());
assertEquals(
"/org/apache/datafusion/linux/aarch64/libdatafusion_jni.so",
Platform.of("Linux", "aarch64").resourcePath());
assertEquals(
"/org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib",
Platform.of("Mac OS X", "x86_64").resourcePath());
assertEquals(
"/org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib",
Platform.of("Mac OS X", "aarch64").resourcePath());
assertEquals(
"/org/apache/datafusion/windows/amd64/datafusion_jni.dll",
Platform.of("Windows 11", "amd64").resourcePath());
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/ScalarFunctionArgsTest.java 0000664 0000000 0000000 00000005346 15205164533 0033036 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
import org.junit.jupiter.api.Test;
class ScalarFunctionArgsTest {
@Test
void construct_emptyArgs_zeroRows_ok() {
ScalarFunctionArgs a = new ScalarFunctionArgs(List.of(), 0);
assertEquals(List.of(), a.args());
assertEquals(0, a.rowCount());
}
@Test
void construct_rejectsNullArgs() {
assertThrows(NullPointerException.class, () -> new ScalarFunctionArgs(null, 3));
}
@Test
void construct_rejectsNegativeRowCount() {
assertThrows(IllegalArgumentException.class, () -> new ScalarFunctionArgs(List.of(), -1));
}
@Test
void construct_copiesArgsDefensively() {
try (BufferAllocator allocator = new RootAllocator();
IntVector v = new IntVector("v", allocator)) {
v.allocateNew(1);
v.setValueCount(1);
List source = new ArrayList<>();
source.add(ColumnarValue.scalar(v));
ScalarFunctionArgs a = new ScalarFunctionArgs(source, 1);
source.clear();
assertEquals(1, a.args().size());
assertThrows(UnsupportedOperationException.class, () -> a.args().clear());
}
}
@Test
void args_singletonCase_preservesValue() {
try (BufferAllocator allocator = new RootAllocator();
IntVector v = new IntVector("v", allocator)) {
v.allocateNew(1);
v.set(0, 7);
v.setValueCount(1);
ScalarFunctionArgs a =
new ScalarFunctionArgs(Collections.singletonList(ColumnarValue.scalar(v)), 5);
assertEquals(5, a.rowCount());
assertTrue(a.args().get(0) instanceof ColumnarValue.Scalar);
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/ScalarUdfTest.java 0000664 0000000 0000000 00000075167 15205164533 0031162 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.StructVector;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.junit.jupiter.api.Test;
class ScalarUdfTest {
private static final ArrowType INT32 = new ArrowType.Int(32, true);
private static final ArrowType INT64 = new ArrowType.Int(64, true);
private static final ArrowType FLOAT64 =
new ArrowType.FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE);
private static final ArrowType UTF8 = new ArrowType.Utf8();
/** Test-only base that supplies the four metadata getters from constructor args. */
abstract static class AbstractScalarFunction implements ScalarFunction {
private final String name;
private final List argFields;
private final Field returnField;
private final Volatility volatility;
AbstractScalarFunction(
String name, List argFields, Field returnField, Volatility volatility) {
this.name = name;
this.argFields = argFields;
this.returnField = returnField;
this.volatility = volatility;
}
@Override
public final String name() {
return name;
}
@Override
public final List argFields() {
return argFields;
}
@Override
public final Field returnField() {
return returnField;
}
@Override
public final Volatility volatility() {
return volatility;
}
}
/** Adds 1 to each row of an Int32 column. */
static final class AddOne extends AbstractScalarFunction {
AddOne() {
this("add_one", Volatility.IMMUTABLE);
}
AddOne(String name, Volatility volatility) {
super(name, List.of(Field.nullable("x", INT32)), Field.nullable("y", INT32), volatility);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
IntVector in = (IntVector) args.args().get(0).vector();
IntVector out = new IntVector("add_one_out", allocator);
int n = in.getValueCount();
out.allocateNew(n);
for (int i = 0; i < n; i++) {
if (in.isNull(i)) {
out.setNull(i);
} else {
out.set(i, in.get(i) + 1);
}
}
out.setValueCount(n);
return ColumnarValue.array(out);
}
}
@Test
void addOne_overConstantTable_returnsIncrementedValues() throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new AddOne()));
try (DataFrame df =
ctx.sql(
"SELECT add_one(x) AS y"
+ " FROM (VALUES (CAST(1 AS INT)), (CAST(2 AS INT)), (CAST(3 AS INT)))"
+ " AS t(x)");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
VectorSchemaRoot root = r.getVectorSchemaRoot();
IntVector y = (IntVector) root.getVector("y");
assertEquals(3, y.getValueCount());
assertEquals(2, y.get(0));
assertEquals(3, y.get(1));
assertEquals(4, y.get(2));
}
}
}
/** Concatenates two Utf8 columns. */
static final class Concat extends AbstractScalarFunction {
Concat() {
super(
"java_concat",
List.of(Field.nullable("a", UTF8), Field.nullable("b", UTF8)),
Field.nullable("c", UTF8),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
org.apache.arrow.vector.VarCharVector left =
(org.apache.arrow.vector.VarCharVector) args.args().get(0).vector();
org.apache.arrow.vector.VarCharVector right =
(org.apache.arrow.vector.VarCharVector) args.args().get(1).vector();
org.apache.arrow.vector.VarCharVector out =
new org.apache.arrow.vector.VarCharVector("concat_out", allocator);
int n = left.getValueCount();
out.allocateNew(n);
for (int i = 0; i < n; i++) {
if (left.isNull(i) || right.isNull(i)) {
out.setNull(i);
} else {
byte[] l = left.get(i);
byte[] r = right.get(i);
byte[] both = new byte[l.length + r.length];
System.arraycopy(l, 0, both, 0, l.length);
System.arraycopy(r, 0, both, l.length, r.length);
out.setSafe(i, both);
}
}
out.setValueCount(n);
return ColumnarValue.array(out);
}
}
@Test
void concat_overVarCharColumns_concatenatesValues() throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new Concat()));
try (DataFrame df =
ctx.sql(
"SELECT java_concat(a, b) AS c FROM (VALUES ('foo','bar'),('hello','!')) AS t(a,"
+ " b)");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
VectorSchemaRoot root = r.getVectorSchemaRoot();
org.apache.arrow.vector.VarCharVector c =
(org.apache.arrow.vector.VarCharVector) root.getVector("c");
assertEquals(2, c.getValueCount());
assertEquals("foobar", new String(c.get(0)));
assertEquals("hello!", new String(c.get(1)));
}
}
}
/** Squares a Float64 column. */
static final class Square extends AbstractScalarFunction {
Square() {
super(
"java_square",
List.of(Field.nullable("x", FLOAT64)),
Field.nullable("y", FLOAT64),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
org.apache.arrow.vector.Float8Vector in =
(org.apache.arrow.vector.Float8Vector) args.args().get(0).vector();
org.apache.arrow.vector.Float8Vector out =
new org.apache.arrow.vector.Float8Vector("square_out", allocator);
int n = in.getValueCount();
out.allocateNew(n);
for (int i = 0; i < n; i++) {
if (in.isNull(i)) {
out.setNull(i);
} else {
double v = in.get(i);
out.set(i, v * v);
}
}
out.setValueCount(n);
return ColumnarValue.array(out);
}
}
@Test
void square_overFloat64Column_squaresValues() throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new Square()));
try (DataFrame df = ctx.sql("SELECT java_square(x) AS y FROM (VALUES (2.0),(3.5)) AS t(x)");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
VectorSchemaRoot root = r.getVectorSchemaRoot();
org.apache.arrow.vector.Float8Vector y =
(org.apache.arrow.vector.Float8Vector) root.getVector("y");
assertEquals(2, y.getValueCount());
assertEquals(4.0, y.get(0), 0.0);
assertEquals(12.25, y.get(1), 0.0);
}
}
}
@Test
void addOne_invokedTwiceInOneSession_executesIndependently() throws Exception {
// Re-running the same UDF query twice exercises that GlobalRefs and JNI state
// don't accumulate across invocations within a session.
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new AddOne()));
for (int run = 0; run < 2; run++) {
try (DataFrame df = ctx.sql("SELECT add_one(CAST(5 AS INT)) AS y");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
IntVector y = (IntVector) r.getVectorSchemaRoot().getVector("y");
assertEquals(1, y.getValueCount());
assertEquals(6, y.get(0));
}
}
}
}
static final class ReturnsNull extends AbstractScalarFunction {
ReturnsNull() {
super(
"bad_null",
List.of(Field.nullable("x", INT32)),
Field.nullable("y", INT32),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
return null;
}
}
@Test
void udfReturningNull_surfacesIllegalStateException() {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new ReturnsNull()));
RuntimeException ex =
org.junit.jupiter.api.Assertions.assertThrows(
RuntimeException.class,
() -> {
try (DataFrame df = ctx.sql("SELECT bad_null(CAST(1 AS INT))");
ArrowReader r = df.collect(allocator)) {
while (r.loadNextBatch()) {}
}
});
org.junit.jupiter.api.Assertions.assertTrue(
ex.getMessage().contains("returned null"),
"expected error to mention 'returned null', got: " + ex.getMessage());
}
}
static final class WrongRowCount extends AbstractScalarFunction {
WrongRowCount() {
super(
"bad_rows",
List.of(Field.nullable("x", INT32)),
Field.nullable("y", INT32),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
IntVector in = (IntVector) args.args().get(0).vector();
IntVector out = new IntVector("out", allocator);
out.allocateNew(in.getValueCount() + 1); // off by one
for (int i = 0; i < in.getValueCount() + 1; i++) out.set(i, 0);
out.setValueCount(in.getValueCount() + 1);
return ColumnarValue.array(out);
}
}
@Test
void udfReturningWrongRowCount_surfacesIllegalStateException() {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new WrongRowCount()));
RuntimeException ex =
org.junit.jupiter.api.Assertions.assertThrows(
RuntimeException.class,
() -> {
try (DataFrame df = ctx.sql("SELECT bad_rows(CAST(1 AS INT))");
ArrowReader r = df.collect(allocator)) {
while (r.loadNextBatch()) {}
}
});
org.junit.jupiter.api.Assertions.assertTrue(
ex.getMessage().contains("expected") && ex.getMessage().contains("rows"),
"expected error to mention row mismatch, got: " + ex.getMessage());
}
}
static final class WrongType extends AbstractScalarFunction {
WrongType() {
super(
"bad_type",
List.of(Field.nullable("x", INT32)),
Field.nullable("y", INT32),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
// Declared return type is Int32; return Float64.
org.apache.arrow.vector.Float8Vector out =
new org.apache.arrow.vector.Float8Vector("out", allocator);
FieldVector in = args.args().get(0).vector();
out.allocateNew(in.getValueCount());
for (int i = 0; i < in.getValueCount(); i++) out.set(i, 0.0);
out.setValueCount(in.getValueCount());
return ColumnarValue.array(out);
}
}
@Test
void udfReturningWrongType_surfacesTypeMismatch() {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new WrongType()));
RuntimeException ex =
org.junit.jupiter.api.Assertions.assertThrows(
RuntimeException.class,
() -> {
try (DataFrame df = ctx.sql("SELECT bad_type(CAST(1 AS INT))");
ArrowReader r = df.collect(allocator)) {
while (r.loadNextBatch()) {}
}
});
org.junit.jupiter.api.Assertions.assertTrue(
ex.getMessage().toLowerCase().contains("type"),
"expected error to mention type mismatch, got: " + ex.getMessage());
}
}
static final class ThrowsIAE extends AbstractScalarFunction {
ThrowsIAE() {
super(
"boom",
List.of(Field.nullable("x", INT32)),
Field.nullable("y", INT32),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
throw new IllegalArgumentException("custom boom from UDF");
}
}
@Test
void udfThrowingException_propagatesClassAndMessage() {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new ThrowsIAE()));
RuntimeException ex =
org.junit.jupiter.api.Assertions.assertThrows(
RuntimeException.class,
() -> {
try (DataFrame df = ctx.sql("SELECT boom(CAST(1 AS INT))");
ArrowReader r = df.collect(allocator)) {
while (r.loadNextBatch()) {}
}
});
String msg = ex.getMessage();
org.junit.jupiter.api.Assertions.assertTrue(
msg.contains("IllegalArgumentException"), "expected class name in error, got: " + msg);
org.junit.jupiter.api.Assertions.assertTrue(
msg.contains("custom boom from UDF"), "expected user message in error, got: " + msg);
}
}
@Test
void twoUdfsInOneSession_bothCallable() throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new AddOne()));
ctx.registerUdf(new ScalarUdf(new Square()));
try (DataFrame df =
ctx.sql("SELECT add_one(CAST(10 AS INT)) AS a, java_square(CAST(3 AS DOUBLE)) AS b");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
VectorSchemaRoot root = r.getVectorSchemaRoot();
IntVector a = (IntVector) root.getVector("a");
org.apache.arrow.vector.Float8Vector b =
(org.apache.arrow.vector.Float8Vector) root.getVector("b");
assertEquals(11, a.get(0));
assertEquals(9.0, b.get(0), 0.0);
}
}
}
@Test
void registerSameNameAfterCloseInNewSession_works() throws Exception {
for (int round = 0; round < 2; round++) {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new AddOne()));
try (DataFrame df = ctx.sql("SELECT add_one(CAST(7 AS INT))");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
IntVector v = (IntVector) r.getVectorSchemaRoot().getVector(0);
assertEquals(8, v.get(0));
}
}
}
}
@Test
void udfAppliedToMultiRowQuery_processesAllRows() throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new AddOne()));
String values =
java.util.stream.IntStream.rangeClosed(1, 100)
.mapToObj(i -> "(CAST(" + i + " AS INT))")
.collect(java.util.stream.Collectors.joining(", "));
try (DataFrame df =
ctx.sql("SELECT add_one(x) AS y FROM (VALUES " + values + ") AS t(x) ORDER BY y");
ArrowReader r = df.collect(allocator)) {
long total = 0;
long rows = 0;
while (r.loadNextBatch()) {
IntVector y = (IntVector) r.getVectorSchemaRoot().getVector("y");
for (int i = 0; i < y.getValueCount(); i++) {
total += y.get(i);
rows++;
}
}
assertEquals(100, rows);
// Sum of 2..101 = (2+101)*100/2 = 5150
assertEquals(5150L, total);
}
}
}
/**
* Nullary UDF returning a length-1 Float8 vector. Marked VOLATILE so DataFusion's constant folder
* does not collapse the call before reaching us. Exercises the path that the abandoned PR #57
* added a separate rowCount parameter for: a nullary UDF can now broadcast its value through
* {@link ColumnarValue#scalar(FieldVector)} and the framework handles per-row expansion.
*/
static final class JavaPi extends AbstractScalarFunction {
JavaPi() {
super("java_pi", List.of(), Field.nullable("p", FLOAT64), Volatility.VOLATILE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
org.apache.arrow.vector.Float8Vector out =
new org.apache.arrow.vector.Float8Vector("pi_out", allocator);
out.allocateNew(1);
out.set(0, Math.PI);
out.setValueCount(1);
return ColumnarValue.scalar(out);
}
}
@Test
void nullaryScalarReturnUdf_overMultiRowQuery_broadcasts() throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new JavaPi()));
try (DataFrame df = ctx.sql("SELECT java_pi() AS p FROM (VALUES (1), (2), (3)) AS t(x)");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
VectorSchemaRoot root = r.getVectorSchemaRoot();
org.apache.arrow.vector.Float8Vector p =
(org.apache.arrow.vector.Float8Vector) root.getVector("p");
assertEquals(3, p.getValueCount());
assertEquals(Math.PI, p.get(0), 0.0);
assertEquals(Math.PI, p.get(1), 0.0);
assertEquals(Math.PI, p.get(2), 0.0);
}
}
}
/**
* UDF over (int_col, int_literal). On every invocation it asserts that arg 0 is an Array and arg
* 1 is a Scalar (length-1 vector). Proves the FFI protocol preserves scalar-ness end-to-end
* rather than materialising the literal to a length-N array on the native side.
*/
static final class AssertSecondArgIsScalar extends AbstractScalarFunction {
AssertSecondArgIsScalar() {
super(
"assert_scalar_arg",
List.of(Field.nullable("a", INT32), Field.nullable("b", INT32)),
Field.nullable("y", INT32),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
if (!(args.args().get(0) instanceof ColumnarValue.Array)) {
throw new AssertionError(
"arg 0 expected Array, got " + args.args().get(0).getClass().getSimpleName());
}
if (!(args.args().get(1) instanceof ColumnarValue.Scalar)) {
throw new AssertionError(
"arg 1 expected Scalar, got " + args.args().get(1).getClass().getSimpleName());
}
IntVector left = (IntVector) args.args().get(0).vector();
IntVector right = (IntVector) args.args().get(1).vector();
if (right.getValueCount() != 1) {
throw new AssertionError(
"Scalar arg vector should have length 1, got " + right.getValueCount());
}
int rightVal = right.get(0);
IntVector out = new IntVector("out", allocator);
int n = left.getValueCount();
out.allocateNew(n);
for (int i = 0; i < n; i++) {
if (left.isNull(i)) {
out.setNull(i);
} else {
out.set(i, left.get(i) + rightVal);
}
}
out.setValueCount(n);
return ColumnarValue.array(out);
}
}
@Test
void scalarLiteralArg_arrivesAsScalarColumnarValue() throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new AssertSecondArgIsScalar()));
try (DataFrame df =
ctx.sql(
"SELECT assert_scalar_arg(x, CAST(100 AS INT)) AS y"
+ " FROM (VALUES (CAST(1 AS INT)), (CAST(2 AS INT)), (CAST(3 AS INT)))"
+ " AS t(x)");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
VectorSchemaRoot root = r.getVectorSchemaRoot();
IntVector y = (IntVector) root.getVector("y");
assertEquals(3, y.getValueCount());
assertEquals(101, y.get(0));
assertEquals(102, y.get(1));
assertEquals(103, y.get(2));
}
}
}
/** UDF that ignores its input and returns a constant Scalar. */
static final class IgnoreInputReturnFortyTwo extends AbstractScalarFunction {
IgnoreInputReturnFortyTwo() {
super(
"forty_two",
List.of(Field.nullable("x", INT32)),
Field.nullable("y", INT32),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
IntVector out = new IntVector("out", allocator);
out.allocateNew(1);
out.set(0, 42);
out.setValueCount(1);
return ColumnarValue.scalar(out);
}
}
@Test
void udfReturningScalar_isBroadcastByFramework() throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new IgnoreInputReturnFortyTwo()));
try (DataFrame df =
ctx.sql(
"SELECT forty_two(x) AS y"
+ " FROM (VALUES (CAST(1 AS INT)), (CAST(2 AS INT)),"
+ " (CAST(3 AS INT)), (CAST(4 AS INT)), (CAST(5 AS INT))) AS t(x)");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
VectorSchemaRoot root = r.getVectorSchemaRoot();
IntVector y = (IntVector) root.getVector("y");
assertEquals(5, y.getValueCount());
for (int i = 0; i < 5; i++) {
assertEquals(42, y.get(i));
}
}
}
}
@Test
void volatilityBytesRoundTrip_forAllThreeKinds() throws Exception {
for (Volatility v : Volatility.values()) {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
String registeredName = "add_one_" + v.name().toLowerCase();
ctx.registerUdf(new ScalarUdf(new AddOne(registeredName, v)));
try (DataFrame df = ctx.sql("SELECT " + registeredName + "(CAST(0 AS INT))");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
IntVector y = (IntVector) r.getVectorSchemaRoot().getVector(0);
assertEquals(1, y.get(0));
}
}
}
}
// ---------------------------------------------------------------------
// Nested-type UDF tests. These pinpoint the regression #58 fixed.
// ---------------------------------------------------------------------
/**
* UDF taking a {@code List} argument and returning its length as Int32. Exercises that
* nested Arrow types -- whose element / member types live on the parent {@link Field}'s {@code
* children} list, not inside {@link ArrowType} -- can be declared and registered.
*/
static final class ListLength extends AbstractScalarFunction {
ListLength() {
super(
"java_list_length",
List.of(
new Field(
"vals",
FieldType.nullable(new ArrowType.List()),
List.of(Field.nullable("item", INT32)))),
Field.nullable("len", INT32),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
ListVector in = (ListVector) args.args().get(0).vector();
IntVector out = new IntVector("len_out", allocator);
int n = in.getValueCount();
out.allocateNew(n);
for (int i = 0; i < n; i++) {
if (in.isNull(i)) {
out.setNull(i);
} else {
out.set(i, in.getElementEndIndex(i) - in.getElementStartIndex(i));
}
}
out.setValueCount(n);
return ColumnarValue.array(out);
}
}
@Test
void udfWithListArg_canBeRegistered() {
// Smoke test: registration alone must succeed for nested-type UDFs. Without #58's fix the
// schema-IPC writer rejects List with no children.
try (SessionContext ctx = new SessionContext()) {
ctx.registerUdf(new ScalarUdf(new ListLength()));
}
}
@Test
void udfWithListArg_canBeInvokedFromSql() throws Exception {
// End-to-end: the registered UDF is callable from SQL with literal list arguments and the
// body sees the right element type. SELECT java_list_length([10, 20, 30]) -> 3.
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new ListLength()));
try (DataFrame df =
ctx.sql(
"SELECT java_list_length(make_array(CAST(10 AS INT), CAST(20 AS INT),"
+ " CAST(30 AS INT))) AS n");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
IntVector n = (IntVector) r.getVectorSchemaRoot().getVector("n");
assertEquals(1, n.getValueCount());
assertEquals(3, n.get(0));
}
}
}
/**
* UDF taking a {@code Struct} and returning the sum {@code a + b} as Int64.
* Confirms the fix is structural rather than List-specific.
*/
static final class SumStructFields extends AbstractScalarFunction {
SumStructFields() {
super(
"java_sum_struct",
List.of(
new Field(
"ab",
FieldType.nullable(new ArrowType.Struct()),
List.of(Field.nullable("a", INT32), Field.nullable("b", INT32)))),
Field.nullable("s", INT64),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
StructVector in = (StructVector) args.args().get(0).vector();
IntVector a = (IntVector) in.getChild("a");
IntVector b = (IntVector) in.getChild("b");
org.apache.arrow.vector.BigIntVector out =
new org.apache.arrow.vector.BigIntVector("sum_out", allocator);
int n = in.getValueCount();
out.allocateNew(n);
for (int i = 0; i < n; i++) {
if (in.isNull(i) || a.isNull(i) || b.isNull(i)) {
out.setNull(i);
} else {
out.set(i, (long) a.get(i) + (long) b.get(i));
}
}
out.setValueCount(n);
return ColumnarValue.array(out);
}
}
@Test
void udfWithStructArg_canBeRegistered() {
// Struct child fields ride through the same Field children list as List elements; if the fix
// works for List it should work for Struct. This pins that.
try (SessionContext ctx = new SessionContext()) {
ctx.registerUdf(new ScalarUdf(new SumStructFields()));
}
}
@Test
void udfWithStructArg_canBeInvokedFromSql() throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new SumStructFields()));
try (DataFrame df =
ctx.sql(
"SELECT java_sum_struct(named_struct('a', CAST(3 AS INT), 'b', CAST(4 AS INT)))"
+ " AS s");
ArrowReader r = df.collect(allocator)) {
assertEquals(true, r.loadNextBatch());
org.apache.arrow.vector.BigIntVector s =
(org.apache.arrow.vector.BigIntVector) r.getVectorSchemaRoot().getVector("s");
assertEquals(1, s.getValueCount());
assertEquals(7L, s.get(0));
}
}
}
/**
* UDF declaring a non-nullable return Field. DataFusion's default {@code return_field_from_args}
* wraps the return type in a fresh always-nullable Field, so without the {@code
* JavaScalarUdf::return_field_from_args} override the planner sees this UDF's output as nullable
* even though the Java caller said otherwise.
*/
static final class NonNullableConstOne extends AbstractScalarFunction {
NonNullableConstOne() {
super(
"java_const_one_nn",
List.of(),
new Field("v", new FieldType(false, INT32, null), null),
Volatility.IMMUTABLE);
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
// Nullary -- broadcast a single value as a Scalar. The framework expands it to
// args.rowCount() rows downstream, so we only need a length-1 vector here.
IntVector out = new IntVector("out", allocator);
out.allocateNew(1);
out.set(0, 1);
out.setValueCount(1);
return ColumnarValue.scalar(out);
}
}
@Test
void udfWithNonNullableReturnField_preservesNullabilityInResultSchema() throws Exception {
// The result column's schema must reflect the declared non-nullable return Field. Before the
// fix the JavaScalarUdf only stored a DataType; DataFusion's default return_field_from_args
// synthesised a fresh always-nullable Field, so the column came back as nullable.
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new NonNullableConstOne()));
try (DataFrame df = ctx.sql("SELECT java_const_one_nn() AS v");
ArrowReader r = df.collect(allocator)) {
assertTrue(r.loadNextBatch());
Field resultField = r.getVectorSchemaRoot().getSchema().findField("v");
assertFalse(
resultField.isNullable(),
"expected declared non-nullable return Field to round-trip through registration,"
+ " got nullable=true");
}
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/SessionContextArrowTest.java 0000664 0000000 0000000 00000017263 15205164533 0033312 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowFileWriter;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class SessionContextArrowTest {
/**
* Write three rows of {@code (id INT, name UTF8)} as a single Arrow IPC file using arrow-vector's
* built-in file writer. Returns the path the test can hand to {@code registerArrow} / {@code
* readArrow}.
*/
private static Path writePeopleArrow(Path dir, String name) throws IOException {
Schema schema =
new Schema(
List.of(
new Field("id", FieldType.notNullable(new ArrowType.Int(32, true)), null),
new Field("name", FieldType.notNullable(new ArrowType.Utf8()), null)));
Path file = dir.resolve(name);
try (BufferAllocator allocator = new RootAllocator();
VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
IntVector id = (IntVector) root.getVector("id");
VarCharVector nameVec = (VarCharVector) root.getVector("name");
id.allocateNew(3);
nameVec.allocateNew(3);
id.set(0, 1);
id.set(1, 2);
id.set(2, 3);
nameVec.setSafe(0, "alice".getBytes());
nameVec.setSafe(1, "bob".getBytes());
nameVec.setSafe(2, "carol".getBytes());
root.setRowCount(3);
try (FileChannel ch =
FileChannel.open(file, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
ArrowFileWriter writer = new ArrowFileWriter(root, null, ch)) {
writer.start();
writer.writeBatch();
writer.end();
}
}
return file;
}
@Test
void registerArrowInfersSchemaAndCounts(@TempDir Path tempDir) throws Exception {
Path file = writePeopleArrow(tempDir, "people.arrow");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerArrow("people", file.toAbsolutePath().toString());
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM people");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector count = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
assertEquals(3L, count.get(0));
}
try (DataFrame df = ctx.sql("SELECT name FROM people WHERE id = 2");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
VarCharVector names = (VarCharVector) root.getVector(0);
assertEquals("bob", new String(names.get(0)));
}
}
}
@Test
void readArrowYieldsTheStoredRows(@TempDir Path tempDir) throws Exception {
Path file = writePeopleArrow(tempDir, "people.arrow");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.readArrow(file.toAbsolutePath().toString());
ArrowReader reader = df.collect(allocator)) {
long total = 0;
while (reader.loadNextBatch()) {
total += reader.getVectorSchemaRoot().getRowCount();
}
assertEquals(3L, total);
}
}
@Test
void registerArrowWithCustomExtension(@TempDir Path tempDir) throws Exception {
Path file = writePeopleArrow(tempDir, "people.ipc");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerArrow(
"t", file.toAbsolutePath().toString(), new ArrowReadOptions().fileExtension(".ipc"));
try (DataFrame df = ctx.sql("SELECT SUM(id) FROM t");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector sum = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
assertEquals(6L, sum.get(0));
}
}
}
@Test
void readArrowWithExplicitSchemaIsAccepted(@TempDir Path tempDir) throws Exception {
// Explicit schema overrides on-read inference. We supply the same schema the
// file actually has, so query results stay correct; the test pins that the
// explicit-schema code path is plumbed through and accepted.
Path file = writePeopleArrow(tempDir, "people.arrow");
Schema schema =
new Schema(
List.of(
new Field("id", FieldType.notNullable(new ArrowType.Int(32, true)), null),
new Field("name", FieldType.notNullable(new ArrowType.Utf8()), null)));
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df =
ctx.readArrow(file.toAbsolutePath().toString(), new ArrowReadOptions().schema(schema));
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(3, root.getRowCount());
assertEquals("id", root.getSchema().getFields().get(0).getName());
assertEquals("name", root.getSchema().getFields().get(1).getName());
}
}
@Test
void registerArrowRejectsNullArguments() {
try (SessionContext ctx = new SessionContext()) {
ArrowReadOptions opts = new ArrowReadOptions();
assertThrows(IllegalArgumentException.class, () -> ctx.registerArrow(null, "/p"));
assertThrows(IllegalArgumentException.class, () -> ctx.registerArrow("t", null));
assertThrows(IllegalArgumentException.class, () -> ctx.registerArrow(null, "/p", opts));
assertThrows(IllegalArgumentException.class, () -> ctx.registerArrow("t", null, opts));
assertThrows(IllegalArgumentException.class, () -> ctx.registerArrow("t", "/p", null));
}
}
@Test
void readArrowRejectsNullArguments() {
try (SessionContext ctx = new SessionContext()) {
ArrowReadOptions opts = new ArrowReadOptions();
assertThrows(IllegalArgumentException.class, () -> ctx.readArrow(null));
assertThrows(IllegalArgumentException.class, () -> ctx.readArrow(null, opts));
assertThrows(IllegalArgumentException.class, () -> ctx.readArrow("/p", null));
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/SessionContextAvroTest.java 0000664 0000000 0000000 00000017530 15205164533 0033124 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.avro.SchemaBuilder;
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.generic.GenericRecord;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class SessionContextAvroTest {
/**
* Write three rows of {@code (id long, name string)} as a single Avro object container file using
* the canonical Apache Avro Java writer. Returns the path the test can hand to {@code
* registerAvro} / {@code readAvro}.
*
*
Avro's logical types are minimal here on purpose: the goal is to pin that DataFusion's Avro
* reader sees the file at all and that the JNI plumbing wires schema inference and SQL correctly.
* Richer Avro type coverage is a job for upstream's own datasource tests.
*/
private static Path writePeopleAvro(Path dir, String name) throws IOException {
org.apache.avro.Schema avroSchema =
SchemaBuilder.record("Person")
.namespace("org.apache.datafusion.test")
.fields()
.name("id")
.type()
.longType()
.noDefault()
.name("name")
.type()
.stringType()
.noDefault()
.endRecord();
Path file = dir.resolve(name);
GenericDatumWriter datumWriter = new GenericDatumWriter<>(avroSchema);
try (DataFileWriter writer = new DataFileWriter<>(datumWriter)) {
writer.create(avroSchema, file.toFile());
for (Object[] row :
new Object[][] {
{1L, "alice"}, {2L, "bob"}, {3L, "carol"},
}) {
GenericRecord rec = new GenericData.Record(avroSchema);
rec.put("id", row[0]);
rec.put("name", row[1]);
writer.append(rec);
}
}
return file;
}
@Test
void registerAvroInfersSchemaAndCounts(@TempDir Path tempDir) throws Exception {
Path file = writePeopleAvro(tempDir, "people.avro");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerAvro("people", file.toAbsolutePath().toString());
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM people");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector count = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
assertEquals(3L, count.get(0));
}
try (DataFrame df = ctx.sql("SELECT name FROM people WHERE id = 2");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
VarCharVector names = (VarCharVector) root.getVector(0);
assertEquals("bob", new String(names.get(0)));
}
}
}
@Test
void readAvroYieldsTheStoredRows(@TempDir Path tempDir) throws Exception {
Path file = writePeopleAvro(tempDir, "people.avro");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.readAvro(file.toAbsolutePath().toString());
ArrowReader reader = df.collect(allocator)) {
long total = 0;
while (reader.loadNextBatch()) {
total += reader.getVectorSchemaRoot().getRowCount();
}
assertEquals(3L, total);
}
}
@Test
void registerAvroWithCustomExtension(@TempDir Path tempDir) throws Exception {
Path file = writePeopleAvro(tempDir, "people.av");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerAvro(
"t", file.toAbsolutePath().toString(), new AvroReadOptions().fileExtension(".av"));
try (DataFrame df = ctx.sql("SELECT SUM(id) FROM t");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector sum = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
assertEquals(6L, sum.get(0));
}
}
}
@Test
void readAvroWithExplicitSchemaIsAccepted(@TempDir Path tempDir) throws Exception {
// Explicit schema overrides on-read inference. We supply the same schema the file actually
// has, so query results stay correct; the test pins that the explicit-schema code path is
// plumbed through and accepted (same as the Arrow / NDJSON readers).
Path file = writePeopleAvro(tempDir, "people.avro");
Schema schema =
new Schema(
List.of(
new Field("id", FieldType.nullable(new ArrowType.Int(64, true)), null),
new Field("name", FieldType.nullable(new ArrowType.Utf8()), null)));
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df =
ctx.readAvro(file.toAbsolutePath().toString(), new AvroReadOptions().schema(schema));
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(3, root.getRowCount());
assertEquals("id", root.getSchema().getFields().get(0).getName());
assertEquals("name", root.getSchema().getFields().get(1).getName());
}
}
@Test
void registerAvroRejectsNullArguments() {
try (SessionContext ctx = new SessionContext()) {
AvroReadOptions opts = new AvroReadOptions();
assertThrows(IllegalArgumentException.class, () -> ctx.registerAvro(null, "/p"));
assertThrows(IllegalArgumentException.class, () -> ctx.registerAvro("t", null));
assertThrows(IllegalArgumentException.class, () -> ctx.registerAvro(null, "/p", opts));
assertThrows(IllegalArgumentException.class, () -> ctx.registerAvro("t", null, opts));
assertThrows(IllegalArgumentException.class, () -> ctx.registerAvro("t", "/p", null));
}
}
@Test
void readAvroRejectsNullArguments() {
try (SessionContext ctx = new SessionContext()) {
AvroReadOptions opts = new AvroReadOptions();
assertThrows(IllegalArgumentException.class, () -> ctx.readAvro(null));
assertThrows(IllegalArgumentException.class, () -> ctx.readAvro(null, opts));
assertThrows(IllegalArgumentException.class, () -> ctx.readAvro("/p", null));
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java0000664 0000000 0000000 00000035243 15205164533 0033604 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Path;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.datafusion.protobuf.ConfigOption;
import org.apache.datafusion.protobuf.SessionOptions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class SessionContextBuilderTest {
@Test
void protoRoundTripPreservesAllFields() throws Exception {
byte[] bytes =
SessionContext.builder()
.batchSize(8192)
.targetPartitions(4)
.collectStatistics(true)
.informationSchema(true)
.memoryLimit(1L << 30, 0.8)
.tempDirectory("/tmp/df")
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
assertTrue(parsed.hasBatchSize());
assertEquals(8192L, parsed.getBatchSize());
assertTrue(parsed.hasTargetPartitions());
assertEquals(4L, parsed.getTargetPartitions());
assertTrue(parsed.hasCollectStatistics());
assertTrue(parsed.getCollectStatistics());
assertTrue(parsed.hasInformationSchema());
assertTrue(parsed.getInformationSchema());
assertTrue(parsed.hasMemoryLimit());
assertEquals(1L << 30, parsed.getMemoryLimit().getMaxMemoryBytes());
assertEquals(0.8, parsed.getMemoryLimit().getMemoryFraction(), 1e-9);
assertTrue(parsed.hasTempDirectory());
assertEquals("/tmp/df", parsed.getTempDirectory());
}
@Test
void unsetFieldsAreAbsentInProto() throws Exception {
byte[] bytes = SessionContext.builder().batchSize(8192).toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
assertTrue(parsed.hasBatchSize());
assertFalse(parsed.hasTargetPartitions());
assertFalse(parsed.hasCollectStatistics());
assertFalse(parsed.hasInformationSchema());
assertFalse(parsed.hasMemoryLimit());
assertFalse(parsed.hasTempDirectory());
}
@Test
void informationSchemaEnabledMakesMetaQueryRun() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = SessionContext.builder().informationSchema(true).build();
DataFrame df = ctx.sql("SELECT table_name FROM information_schema.tables");
ArrowReader reader = df.collect(allocator)) {
// information_schema.tables always has at least one row.
assertTrue(reader.loadNextBatch());
}
}
@Test
void informationSchemaDisabledByDefaultThrows() {
try (SessionContext ctx = SessionContext.builder().build()) {
assertThrows(
RuntimeException.class,
() -> ctx.sql("SELECT table_name FROM information_schema.tables"));
}
}
@Test
void buildWithEveryKnobSetCanExecuteSelectOne(@TempDir Path tempDir) throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx =
SessionContext.builder()
.batchSize(8192)
.targetPartitions(4)
.collectStatistics(true)
.informationSchema(true)
.memoryLimit(1L << 30, 0.8)
.tempDirectory(tempDir.toAbsolutePath().toString())
.build();
DataFrame df = ctx.sql("SELECT 1");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
assertEquals(1, reader.getVectorSchemaRoot().getRowCount());
}
}
@Test
void setOptionRoundTripsThroughProto() throws Exception {
byte[] bytes =
SessionContext.builder()
.setOption("datafusion.execution.parquet.pushdown_filters", "true")
.setOption("datafusion.optimizer.prefer_hash_join", "true")
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
List opts = parsed.getOptionsList();
assertEquals(2, opts.size());
assertEquals("datafusion.execution.parquet.pushdown_filters", opts.get(0).getKey());
assertEquals("true", opts.get(0).getValue());
assertEquals("datafusion.optimizer.prefer_hash_join", opts.get(1).getKey());
assertEquals("true", opts.get(1).getValue());
}
@Test
void setOptionsBulkRoundTripsThroughProto() throws Exception {
Map entries = new LinkedHashMap<>();
entries.put("datafusion.execution.time_zone", "UTC");
entries.put("datafusion.optimizer.default_filter_selectivity", "10");
byte[] bytes = SessionContext.builder().setOptions(entries).toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
List opts = parsed.getOptionsList();
assertEquals(2, opts.size());
assertEquals("datafusion.execution.time_zone", opts.get(0).getKey());
assertEquals("UTC", opts.get(0).getValue());
assertEquals("datafusion.optimizer.default_filter_selectivity", opts.get(1).getKey());
assertEquals("10", opts.get(1).getValue());
}
@Test
void setOptionRejectsNullKeyOrValue() {
SessionContextBuilder b = SessionContext.builder();
assertThrows(IllegalArgumentException.class, () -> b.setOption(null, "v"));
assertThrows(IllegalArgumentException.class, () -> b.setOption("k", null));
assertThrows(IllegalArgumentException.class, () -> b.setOptions(null));
}
@Test
void setOptionAffectsRunningSession() {
try (SessionContext ctx =
SessionContext.builder()
.setOption("datafusion.execution.parquet.pushdown_filters", "true")
.build()) {
assertEquals("true", ctx.getOption("datafusion.execution.parquet.pushdown_filters"));
}
}
@Test
void setOptionLastWriteWinsWithinMap() throws Exception {
byte[] bytes =
SessionContext.builder()
.setOption("datafusion.execution.batch_size", "1024")
.setOption("datafusion.execution.batch_size", "8192")
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
// setOption dedups by key (remove-then-put), so only one entry reaches
// the wire — the most-recently-written value, at the position of the
// most recent setOption call.
List opts = parsed.getOptionsList();
assertEquals(1, opts.size());
assertEquals("datafusion.execution.batch_size", opts.get(0).getKey());
assertEquals("8192", opts.get(0).getValue());
}
@Test
void setOptionOverridesTypedSetterAtConstruction() {
// Typed setter says batch_size = 8192; setOption overrides to 1024.
// The override is a real semantic change, not just absence of a throw —
// assert that's what DataFusion is actually running with.
try (SessionContext ctx =
SessionContext.builder()
.batchSize(8192)
.setOption("datafusion.execution.batch_size", "1024")
.build()) {
assertEquals("1024", ctx.getOption("datafusion.execution.batch_size"));
}
}
@Test
void unknownOptionKeyThrowsAtBuild() {
SessionContextBuilder b =
SessionContext.builder().setOption("datafusion.execution.bogus_made_up_key", "x");
RuntimeException thrown = assertThrows(RuntimeException.class, b::build);
// The DataFusion error message includes the bad key text — assert it's
// surfaced rather than swallowed.
assertTrue(
thrown.getMessage() != null && thrown.getMessage().contains("bogus_made_up_key"),
"expected DataFusion error to mention bad key, got: " + thrown.getMessage());
}
@Test
void getOptionReadsTypedSetterValue() {
try (SessionContext ctx = SessionContext.builder().batchSize(4096).build()) {
assertEquals("4096", ctx.getOption("datafusion.execution.batch_size"));
}
}
@Test
void getOptionWithNoExplicitValueReturnsDefault() {
// batch_size has a non-null default in DataFusion, so the getter must
// return the default rather than null when the user never set it.
try (SessionContext ctx = SessionContext.builder().build()) {
String batchSize = ctx.getOption("datafusion.execution.batch_size");
assertTrue(
batchSize != null && !batchSize.isEmpty(),
"expected default batch_size, got: " + batchSize);
}
}
@Test
void getOptionRejectsNullKey() {
try (SessionContext ctx = SessionContext.builder().build()) {
assertThrows(IllegalArgumentException.class, () -> ctx.getOption(null));
}
}
@Test
void getOptionThrowsOnUnknownKey() {
try (SessionContext ctx = SessionContext.builder().build()) {
RuntimeException thrown =
assertThrows(RuntimeException.class, () -> ctx.getOption("datafusion.no.such.key"));
assertTrue(
thrown.getMessage() != null && thrown.getMessage().contains("no.such.key"),
"expected error to mention bad key, got: " + thrown.getMessage());
}
}
@Test
void getOptionThrowsAfterClose() {
SessionContext ctx = SessionContext.builder().build();
ctx.close();
assertThrows(
IllegalStateException.class, () -> ctx.getOption("datafusion.execution.batch_size"));
}
@Test
void setOptionAppliesInInsertionOrder() throws Exception {
// Some upstream setters have side effects on other keys. For example,
// setting `datafusion.optimizer.enable_dynamic_filter_pushdown` rewrites
// the per-operator `enable_*_dynamic_filter_pushdown` flags. The caller's
// last write therefore must win. Pin the on-the-wire order; the Rust side
// applies in this order and DataFusion's per-key setter sees the umbrella
// first, the override second.
byte[] bytes =
SessionContext.builder()
.setOption("datafusion.optimizer.enable_dynamic_filter_pushdown", "true")
.setOption("datafusion.optimizer.enable_topk_dynamic_filter_pushdown", "false")
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
List opts = parsed.getOptionsList();
assertEquals(2, opts.size());
assertEquals("datafusion.optimizer.enable_dynamic_filter_pushdown", opts.get(0).getKey());
assertEquals("datafusion.optimizer.enable_topk_dynamic_filter_pushdown", opts.get(1).getKey());
// And on the running session, the topk override wins over the umbrella.
try (SessionContext ctx =
SessionContext.builder()
.setOption("datafusion.optimizer.enable_dynamic_filter_pushdown", "true")
.setOption("datafusion.optimizer.enable_topk_dynamic_filter_pushdown", "false")
.build()) {
assertEquals(
"false", ctx.getOption("datafusion.optimizer.enable_topk_dynamic_filter_pushdown"));
assertEquals("true", ctx.getOption("datafusion.optimizer.enable_dynamic_filter_pushdown"));
}
}
@Test
void setOptionRepeatedKeyMovesToEndOfIterationOrder() throws Exception {
// Codex regression: a LinkedHashMap.put on an existing key updates the
// value but keeps the *original* slot. If the caller writes
// topk=false → umbrella=true → topk=false (re-write)
// the re-write of topk would otherwise stay at position 0, putting the
// umbrella *after* it on the wire. The umbrella's setter has the side
// effect of resetting topk back to true, silently undoing the override.
// Pin both the wire-level order and the running-session behaviour.
byte[] bytes =
SessionContext.builder()
.setOption("datafusion.optimizer.enable_topk_dynamic_filter_pushdown", "false")
.setOption("datafusion.optimizer.enable_dynamic_filter_pushdown", "true")
.setOption("datafusion.optimizer.enable_topk_dynamic_filter_pushdown", "false")
.toBytes();
SessionOptions parsed = SessionOptions.parseFrom(bytes);
List opts = parsed.getOptionsList();
assertEquals(2, opts.size());
// Umbrella applies first; the re-issued topk override is last.
assertEquals("datafusion.optimizer.enable_dynamic_filter_pushdown", opts.get(0).getKey());
assertEquals("datafusion.optimizer.enable_topk_dynamic_filter_pushdown", opts.get(1).getKey());
try (SessionContext ctx =
SessionContext.builder()
.setOption("datafusion.optimizer.enable_topk_dynamic_filter_pushdown", "false")
.setOption("datafusion.optimizer.enable_dynamic_filter_pushdown", "true")
.setOption("datafusion.optimizer.enable_topk_dynamic_filter_pushdown", "false")
.build()) {
assertEquals(
"false", ctx.getOption("datafusion.optimizer.enable_topk_dynamic_filter_pushdown"));
}
}
@Test
void setOptionRejectsRuntimeKeysAtBuild() {
// datafusion.runtime.* lives on a separate config object (RuntimeEnv) and
// round-tripping through getOption/setOption has subtle correctness
// pitfalls (lazy default-tempdir creation, integer K/M/G truncation,
// OS-specific path separators). Reject runtime keys with a clear pointer
// to the typed setters until a follow-up PR designs proper support.
SessionContextBuilder b =
SessionContext.builder().setOption("datafusion.runtime.memory_limit", "2G");
RuntimeException thrown = assertThrows(RuntimeException.class, b::build);
assertTrue(
thrown.getMessage() != null
&& thrown.getMessage().contains("datafusion.runtime")
&& thrown.getMessage().contains("memoryLimit"),
"expected runtime-key error to point at typed setters, got: " + thrown.getMessage());
}
@Test
void getOptionRejectsRuntimeKeys() {
try (SessionContext ctx = SessionContext.builder().build()) {
RuntimeException thrown =
assertThrows(
RuntimeException.class, () -> ctx.getOption("datafusion.runtime.memory_limit"));
assertTrue(
thrown.getMessage() != null && thrown.getMessage().contains("datafusion.runtime"),
"expected runtime-key error, got: " + thrown.getMessage());
}
}
}
SessionContextCacheManagerTest.java 0000664 0000000 0000000 00000010552 15205164533 0034431 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.time.Duration;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Test;
class SessionContextCacheManagerTest {
@Test
void fileMetadataCacheBuildsAndContextIsUsable() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx =
SessionContext.builder()
.cacheManager(CacheManagerOptions.builder().fileMetadataCache(64L << 20).build())
.build();
DataFrame df = ctx.sql("SELECT 1");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
assertEquals(1, reader.getVectorSchemaRoot().getRowCount());
}
}
@Test
void listFilesCacheWithFiniteTtlBuilds() {
try (SessionContext ctx =
SessionContext.builder()
.cacheManager(
CacheManagerOptions.builder()
.listFilesCache(8L << 20, Duration.ofMinutes(5))
.build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void listFilesCacheWithInfiniteTtlBuilds() {
// null TTL must reach the Rust side as None (infinite). Construction
// succeeds; the only failure mode would be the JNI layer mistranslating
// null as "TTL=0" and the upstream constructor rejecting it.
try (SessionContext ctx =
SessionContext.builder()
.cacheManager(CacheManagerOptions.builder().listFilesCache(8L << 20, null).build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void fileStatisticsCacheBuilds() {
try (SessionContext ctx =
SessionContext.builder()
.cacheManager(CacheManagerOptions.builder().fileStatisticsCache(true).build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void allThreeCachesTogetherBuildAndContextIsUsable() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx =
SessionContext.builder()
.cacheManager(
CacheManagerOptions.builder()
.fileMetadataCache(64L << 20)
.listFilesCache(8L << 20, Duration.ofMinutes(5))
.fileStatisticsCache(true)
.build())
.build();
DataFrame df = ctx.sql("SELECT 1");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
assertEquals(1, reader.getVectorSchemaRoot().getRowCount());
}
}
@Test
void emptyCacheManagerOptionsIsHarmlessNoop() {
// builder().build() with nothing set is on the wire as "all three slots
// unset". The Rust side treats that as "skip with_cache_manager
// entirely" and the context construction succeeds.
try (SessionContext ctx =
SessionContext.builder().cacheManager(CacheManagerOptions.builder().build()).build()) {
assertNotNull(ctx);
}
}
@Test
void fileMetadataCacheZeroIsHarmless() {
// Upstream allows a 0-byte cap (cache constructed but evicts every
// insert). Java should not reject it.
try (SessionContext ctx =
SessionContext.builder()
.cacheManager(CacheManagerOptions.builder().fileMetadataCache(0).build())
.build()) {
assertNotNull(ctx);
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/SessionContextCsvTest.java 0000664 0000000 0000000 00000011200 15205164533 0032734 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class SessionContextCsvTest {
private static Path writeCsv(Path dir, String name, String contents) throws IOException {
Path file = dir.resolve(name);
Files.writeString(file, contents);
return file;
}
@Test
void registerCsvWithHeaderInfersSchemaAndCounts(@TempDir Path tempDir) throws Exception {
Path csv = writeCsv(tempDir, "people.csv", "id,name\n1,alice\n2,bob\n3,carol\n");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerCsv("people", csv.toAbsolutePath().toString());
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM people");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector count = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
assertEquals(3L, count.get(0));
}
try (DataFrame df = ctx.sql("SELECT name FROM people WHERE id = 2");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
VarCharVector names = (VarCharVector) root.getVector(0);
assertEquals("bob", new String(names.get(0)));
}
}
}
@Test
void readCsvWithExplicitSchemaAndNoHeader(@TempDir Path tempDir) throws Exception {
Path csv = writeCsv(tempDir, "headerless.csv", "1|alice\n2|bob\n");
Schema schema =
new Schema(
List.of(
new Field("id", FieldType.nullable(new ArrowType.Int(64, true)), null),
new Field("name", FieldType.nullable(new ArrowType.Utf8()), null)));
CsvReadOptions opts =
new CsvReadOptions().hasHeader(false).delimiter((byte) '|').schema(schema);
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.readCsv(csv.toAbsolutePath().toString(), opts);
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(2, root.getRowCount());
assertEquals("id", root.getSchema().getFields().get(0).getName());
assertEquals("name", root.getSchema().getFields().get(1).getName());
}
}
@Test
void registerCsvWithCustomExtension(@TempDir Path tempDir) throws Exception {
Path csv = writeCsv(tempDir, "data.tsv", "x\ty\n10\t20\n30\t40\n");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerCsv(
"t",
csv.toAbsolutePath().toString(),
new CsvReadOptions().delimiter((byte) '\t').fileExtension(".tsv"));
try (DataFrame df = ctx.sql("SELECT SUM(x) + SUM(y) FROM t");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector v = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
assertEquals(100L, v.get(0));
}
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/SessionContextJsonTest.java 0000664 0000000 0000000 00000013514 15205164533 0033124 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class SessionContextJsonTest {
private static Path writeJson(Path dir, String name, String contents) throws IOException {
Path file = dir.resolve(name);
Files.writeString(file, contents);
return file;
}
@Test
void registerJsonInfersSchemaAndCounts(@TempDir Path tempDir) throws Exception {
Path file =
writeJson(
tempDir,
"people.json",
"{\"id\":1,\"name\":\"alice\"}\n"
+ "{\"id\":2,\"name\":\"bob\"}\n"
+ "{\"id\":3,\"name\":\"carol\"}\n");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerJson("people", file.toAbsolutePath().toString());
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM people");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector count = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
assertEquals(3L, count.get(0));
}
try (DataFrame df = ctx.sql("SELECT name FROM people WHERE id = 2");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
VarCharVector names = (VarCharVector) root.getVector(0);
assertEquals("bob", new String(names.get(0)));
}
}
}
@Test
void readJsonWithExplicitSchema(@TempDir Path tempDir) throws Exception {
Path file =
writeJson(
tempDir, "headerless.json", "{\"id\":10,\"name\":\"x\"}\n{\"id\":20,\"name\":\"y\"}\n");
Schema schema =
new Schema(
List.of(
new Field("id", FieldType.nullable(new ArrowType.Int(64, true)), null),
new Field("name", FieldType.nullable(new ArrowType.Utf8()), null)));
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df =
ctx.readJson(file.toAbsolutePath().toString(), new NdJsonReadOptions().schema(schema));
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(2, root.getRowCount());
assertEquals("id", root.getSchema().getFields().get(0).getName());
assertEquals("name", root.getSchema().getFields().get(1).getName());
}
}
@Test
void registerJsonWithCustomExtension(@TempDir Path tempDir) throws Exception {
Path file = writeJson(tempDir, "data.ndjson", "{\"x\":10,\"y\":20}\n{\"x\":30,\"y\":40}\n");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerJson(
"t", file.toAbsolutePath().toString(), new NdJsonReadOptions().fileExtension(".ndjson"));
try (DataFrame df = ctx.sql("SELECT SUM(x) + SUM(y) FROM t");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector v = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
assertEquals(100L, v.get(0));
}
}
}
@Test
void registerJsonRejectsNullArguments() {
try (SessionContext ctx = new SessionContext()) {
NdJsonReadOptions opts = new NdJsonReadOptions();
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson(null, "/p"));
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson("t", null));
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson(null, "/p", opts));
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson("t", null, opts));
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson("t", "/p", null));
}
}
@Test
void readJsonRejectsNullArguments() {
try (SessionContext ctx = new SessionContext()) {
NdJsonReadOptions opts = new NdJsonReadOptions();
assertThrows(IllegalArgumentException.class, () -> ctx.readJson(null));
assertThrows(IllegalArgumentException.class, () -> ctx.readJson(null, opts));
assertThrows(IllegalArgumentException.class, () -> ctx.readJson("/p", null));
}
}
}
SessionContextMemoryUsageTest.java 0000664 0000000 0000000 00000012137 15205164533 0034371 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Test;
class SessionContextMemoryUsageTest {
/** A query with enough rows to drive a non-trivial sort, so peak memory is measurable. */
private static final String HEAVY_SQL =
"SELECT a, b FROM "
+ " (SELECT i AS a, i * 7 AS b FROM "
+ " (SELECT * FROM "
+ " generate_series(1, 200000) AS t(i))) "
+ "ORDER BY b DESC";
@Test
void snapshotIsZeroBeforeAnyQuery() {
try (SessionContext ctx = new SessionContext()) {
MemoryUsage usage = ctx.memoryUsage();
assertEquals(0L, usage.currentBytes());
assertEquals(0L, usage.peakBytes());
}
}
@Test
void peakIncreasesAfterMemoryHeavyQuery() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
try (DataFrame df = ctx.sql(HEAVY_SQL);
ArrowReader reader = df.collect(allocator)) {
// Drain the result so the sort actually runs and allocates buffers.
while (reader.loadNextBatch()) {
assertNotNull(reader.getVectorSchemaRoot());
}
}
MemoryUsage usage = ctx.memoryUsage();
assertTrue(usage.peakBytes() > 0L, () -> "peakBytes was " + usage.peakBytes());
assertTrue(
usage.peakBytes() >= usage.currentBytes(),
() -> "peak " + usage.peakBytes() + " < current " + usage.currentBytes());
}
}
@Test
void peakStaysAfterCurrentDrops() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
try (DataFrame df = ctx.sql(HEAVY_SQL);
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
assertNotNull(reader.getVectorSchemaRoot());
}
}
MemoryUsage afterRun = ctx.memoryUsage();
// After the query is fully drained and closed, current should be ~0
// (DataFusion shrinks reservations on drop) but peak should remain.
assertEquals(0L, afterRun.currentBytes());
assertTrue(afterRun.peakBytes() > 0L);
}
}
@Test
void pollableFromAnotherThread() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
AtomicBoolean done = new AtomicBoolean(false);
AtomicLong observedPeak = new AtomicLong(0L);
AtomicReference error = new AtomicReference<>();
Thread poller =
new Thread(
() -> {
try {
while (!done.get()) {
long p = ctx.memoryUsage().peakBytes();
long prev = observedPeak.get();
if (p > prev) {
observedPeak.compareAndSet(prev, p);
}
Thread.yield();
}
} catch (Throwable t) {
error.set(t);
}
});
poller.start();
try {
try (DataFrame df = ctx.sql(HEAVY_SQL);
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
assertNotNull(reader.getVectorSchemaRoot());
}
}
} finally {
done.set(true);
poller.join();
}
assertNotNull(observedPeak);
// The poller should have observed at least one non-zero peak.
assertTrue(
observedPeak.get() > 0L,
() -> "concurrent poller never saw non-zero peak; error=" + error.get());
// And no exceptions on the polling thread.
assertEquals(null, error.get());
}
}
@Test
void closedContextThrows() {
SessionContext ctx = new SessionContext();
ctx.close();
assertThrows(IllegalStateException.class, ctx::memoryUsage);
}
}
SessionContextObjectStoreTest.java 0000664 0000000 0000000 00000027270 15205164533 0034363 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Test;
class SessionContextObjectStoreTest {
// ---- happy path ---------------------------------------------------------
@Test
void s3RegistrationSucceedsAndContextIsUsable() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.s3()
.bucket("my-bucket")
.region("us-east-1")
.accessKeyId("AKIAEXAMPLE")
.secretAccessKey("EXAMPLE")
.build())
.build();
DataFrame df = ctx.sql("SELECT 1");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
assertEquals(1, reader.getVectorSchemaRoot().getRowCount());
}
}
@Test
void gcsRegistrationSucceedsAndContextIsUsable() {
// GoogleCloudStorageBuilder eagerly parses the embedded PEM private key
// when serviceAccountKey is supplied. The `disable_oauth` flag is the
// upstream-test-fixture knob that lets us hand it a placeholder key
// without standing up a real Google credential — see object_store
// gcp/builder.rs FAKE_KEY. We're verifying the registration plumbing,
// not GCS itself.
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.gcs()
.bucket("data")
.serviceAccountKey(
"{\"private_key\":\"k\",\"private_key_id\":\"id\","
+ "\"client_email\":\"x@y\",\"disable_oauth\":true}")
.build())
.build()) {
// Context is alive — the registration didn't reject anything.
assertNotNull(ctx);
}
}
@Test
void httpRegistrationSucceedsAtHostRoot() {
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.http("https://example.com/").allowHttp(false).build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void httpRegistrationRejectsPathfulUrl() {
// Codex regression: if HttpStore is built with a pathful base URL while
// DataFusion's registry strips the path from the lookup key, every read
// double-prepends the path component (e.g. /data/data/file.parquet).
// Reject pathful registration URLs at build time with an error that
// points at the right shape (host root, SQL paths carry the rest).
SessionContextBuilder b =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.http("https://example.com/data/").allowHttp(false).build());
RuntimeException thrown = assertThrows(RuntimeException.class, b::build);
assertTrue(
thrown.getMessage() != null && thrown.getMessage().contains("host root"),
"expected error to mention 'host root', got: " + thrown.getMessage());
}
@Test
void multipleS3RegistrationsCanCoexist() {
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.s3()
.bucket("a")
.region("us-east-1")
.accessKeyId("AKIAA")
.secretAccessKey("S")
.build())
.registerObjectStore(
ObjectStoreOptions.s3()
.bucket("b")
.region("eu-west-1")
.endpoint("https://minio.internal:9000")
.accessKeyId("AKIAB")
.secretAccessKey("S")
.allowHttp(true)
.build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void s3UrlOverrideTakesEffect() {
// The override path is exercised end-to-end (registration must succeed
// with the alternate scheme). The registry would reject a malformed URL.
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.s3()
.bucket("my-bucket")
.url("s3a://my-bucket")
.region("us-east-1")
.accessKeyId("k")
.secretAccessKey("s")
.build())
.build()) {
assertNotNull(ctx);
}
}
// ---- error handling -----------------------------------------------------
@Test
void httpWithoutUrlFailsAtJavaLevel() {
// Caught before any byte hits the JNI boundary.
assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.http(null));
}
@Test
void s3ExplicitEndpointBuildsAlongsideCredentialEnvSeeding() {
// Codex regression: AmazonS3Builder::build picks `s3_endpoint` over
// `endpoint` (object_store 0.13 aws/builder.rs:1209). If we'd seeded the
// builder via from_env(), a stray AWS_ENDPOINT_URL_S3 in the JVM env
// would silently override Java's endpoint(...) -- breaking MinIO/R2
// registrations. The Rust path now walks env explicitly and skips the
// endpoint keys when Java sets endpoint, so the explicit MinIO endpoint
// wins. End-to-end JVM env-injection is impractical from a Java unit
// test, so this test exercises the registration path with an explicit
// endpoint and confirms the build succeeds; the precedence behaviour
// itself is enforced by the env-walk loop in native/src/object_store.rs
// (covered by `cargo check` and a code-level review of the loop body).
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.s3()
.bucket("local-bucket")
.region("us-east-1")
.endpoint("https://minio.internal:9000")
.accessKeyId("k")
.secretAccessKey("s")
.allowHttp(true)
.build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void s3ExplicitImdsv1FallbackFalseOverridesEnv() {
// Codex regression: AmazonS3Builder::with_imdsv1_fallback() only sets to
// `true` (no boolean arg), so a previous version of the binding silently
// dropped explicit `imdsv1Fallback(false)`, leaving a less-secure
// env-seeded `AWS_IMDSV1_FALLBACK=true` in place. The Rust path now
// writes through with_config(...) so explicit false also takes effect.
// Same caveat as the endpoint test: this exercises the registration
// path; the override is enforced by the with_config call in
// native/src/object_store.rs.
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.s3()
.bucket("b")
.region("us-east-1")
.imdsv1Fallback(false)
.build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void gcsExplicitServiceAccountPathOverridesEnvKey() {
// Codex regression: when the JVM env has GOOGLE_SERVICE_ACCOUNT_KEY and
// the Java caller supplies serviceAccountPath(...), the previous
// GoogleCloudStorageBuilder::from_env() seed left both populated, and
// GCS build() rejects that combo with ServiceAccountPathAndKeyProvided
// (object_store 0.13 gcp/builder.rs:525). The Rust path now walks env
// explicitly and skips GOOGLE_SERVICE_ACCOUNT* / GOOGLE_APPLICATION_CREDENTIALS
// when Java provides any of the credential fields. Use the upstream
// FAKE_KEY (disable_oauth) to dodge real PEM parsing.
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.gcs()
.bucket("data")
.serviceAccountKey(
"{\"private_key\":\"k\",\"private_key_id\":\"id\","
+ "\"client_email\":\"x@y\",\"disable_oauth\":true}")
.build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void s3JavaBucketWinsOverEnvBucket() {
// Codex regression: AWS_BUCKET / AWS_BUCKET_NAME parse to
// AmazonS3ConfigKey::Bucket on the env-walking path (object_store 0.13
// aws/builder.rs:497). If the loop ran with the Java bucket already set,
// an env bucket would silently overwrite it -- the registry URL still
// says s3:// but the built store would target the env
// bucket, silently routing reads to the wrong place. The Rust path now
// skips AWS_BUCKET / AWS_BUCKET_NAME during env walking and re-applies
// opts.bucket *after* the loop. Exercise the registration path so any
// regression surfaces.
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.s3()
.bucket("java-explicit-bucket")
.region("us-east-1")
.accessKeyId("k")
.secretAccessKey("s")
.build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void gcsJavaBucketWinsOverEnvBucket() {
// Codex regression: GOOGLE_BUCKET / GOOGLE_BUCKET_NAME parse to
// GoogleConfigKey::Bucket. Same shape as the S3 case above; the env
// walk now skips both keys and re-applies opts.bucket after the loop.
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(
ObjectStoreOptions.gcs()
.bucket("java-explicit-bucket")
.serviceAccountKey(
"{\"private_key\":\"k\",\"private_key_id\":\"id\","
+ "\"client_email\":\"x@y\",\"disable_oauth\":true}")
.build())
.build()) {
assertNotNull(ctx);
}
}
@Test
void s3MissingCredentialsStillBuildsAndDefersToCredChain() {
// The AWS SDK's default credential chain is allowed to find creds at
// request time; building the store does not require credentials up
// front. This keeps parity with object_store::aws::AmazonS3Builder and
// avoids forcing every embedder to thread credentials through Java when
// the JVM is already running on a credentialed host (IAM role, etc.).
try (SessionContext ctx =
SessionContext.builder()
.registerObjectStore(ObjectStoreOptions.s3().bucket("b").region("us-east-1").build())
.build()) {
assertNotNull(ctx);
}
}
}
SessionContextParquetOptionsTest.java 0000664 0000000 0000000 00000012322 15205164533 0035125 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class SessionContextParquetOptionsTest {
@Test
void readParquetWithDefaultOptionsCountsAllRows() throws Exception {
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
Assumptions.assumeTrue(
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.readParquet(lineitem.toAbsolutePath().toString());
ArrowReader reader = df.collect(allocator)) {
long total = 0;
while (reader.loadNextBatch()) {
VectorSchemaRoot root = reader.getVectorSchemaRoot();
total += root.getRowCount();
}
assertEquals(6_001_215L, total);
assertTrue(total > 0);
}
}
@Test
void registerParquetWithOptionsRespectsCustomExtension(@TempDir Path tempDir) throws Exception {
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
Assumptions.assumeTrue(
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
Path renamed = tempDir.resolve("lineitem.parq");
Files.copy(lineitem, renamed, StandardCopyOption.REPLACE_EXISTING);
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerParquet(
"t",
renamed.toAbsolutePath().toString(),
new ParquetReadOptions().fileExtension(".parq"));
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM t");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
BigIntVector count = (BigIntVector) root.getVector(0);
assertEquals(6_001_215L, count.get(0));
}
}
}
@Test
void readParquetWithExplicitSchemaUsesProvidedSchema() throws Exception {
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
Assumptions.assumeTrue(
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
Schema custom =
new Schema(
List.of(
new Field("l_orderkey", FieldType.notNullable(new ArrowType.Int(64, true)), null)));
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df =
ctx.readParquet(
lineitem.toAbsolutePath().toString(), new ParquetReadOptions().schema(custom));
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
Schema observed = reader.getVectorSchemaRoot().getSchema();
assertEquals(1, observed.getFields().size());
assertEquals("l_orderkey", observed.getFields().get(0).getName());
}
}
@Test
void registerParquetWithMetadataSizeHintIsAccepted() throws Exception {
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
Assumptions.assumeTrue(
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerParquet(
"t",
lineitem.toAbsolutePath().toString(),
new ParquetReadOptions().metadataSizeHint(1L << 20));
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM t");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
BigIntVector count = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
assertEquals(6_001_215L, count.get(0));
}
}
}
}
SessionContextRuntimeStatsTest.java 0000664 0000000 0000000 00000012132 15205164533 0034571 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
/**
* Tests for {@link SessionContext#runtimeStats()}. The {@code runtime-metrics} Cargo feature is off
* by default in {@code native/Cargo.toml}; if the native crate was built without it (and without
* {@code RUSTFLAGS="--cfg tokio_unstable"}), every test here is skipped via {@link
* #checkFeatureEnabled}. Run
*
*
*
* before {@code ./mvnw test} to exercise this class.
*/
class SessionContextRuntimeStatsTest {
@BeforeAll
static void checkFeatureEnabled() {
try (SessionContext ctx = new SessionContext()) {
ctx.runtimeStats();
} catch (RuntimeException e) {
String msg = e.getMessage() == null ? "" : e.getMessage();
Assumptions.assumeFalse(
msg.contains("runtime-metrics` Cargo feature"),
"datafusion-jni built without the `runtime-metrics` feature; skipping runtime-stats"
+ " tests");
}
}
@Test
void fieldsAreNonNegative() {
try (SessionContext ctx = new SessionContext()) {
RuntimeStats s = ctx.runtimeStats();
assertTrue(s.numWorkers() > 0, () -> "numWorkers=" + s.numWorkers());
assertTrue(s.liveTasksCount() >= 0, () -> "liveTasksCount=" + s.liveTasksCount());
assertTrue(s.globalQueueDepth() >= 0, () -> "globalQueueDepth=" + s.globalQueueDepth());
assertTrue(s.elapsedNanos() >= 0L, () -> "elapsedNanos=" + s.elapsedNanos());
assertTrue(s.totalBusyNanos() >= 0L, () -> "totalBusyNanos=" + s.totalBusyNanos());
assertTrue(s.totalParkCount() >= 0L);
assertTrue(s.totalPollsCount() >= 0L);
assertTrue(s.totalNoopCount() >= 0L);
assertTrue(s.totalStealCount() >= 0L);
assertTrue(s.totalLocalScheduleCount() >= 0L);
assertTrue(s.totalOverflowCount() >= 0L);
}
}
@Test
void numWorkersMatchesRuntimeShape() {
try (SessionContext ctx = new SessionContext()) {
// The shared multi-threaded Tokio runtime defaults to one worker per CPU.
// We don't assert the exact count -- runners differ -- but it must be >= 1.
assertTrue(ctx.runtimeStats().numWorkers() >= 1);
}
}
@Test
void elapsedIncreasesAcrossSnapshots() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
// The strict guarantee from `tokio_metrics::RuntimeMonitor::intervals()`
// is that each delta covers the wall-clock since the previous probe.
// `total_busy_duration` and `total_polls_count` only advance if work
// *actually runs on a Tokio worker thread* during the interval -- and
// for small in-memory DataFusion queries the future may complete
// synchronously on the JNI thread (which is not a worker), so neither
// metric is a reliable monotonic signal in tests. `elapsed` is, since
// it's pure wall-clock between probes.
RuntimeStats before = ctx.runtimeStats();
for (int i = 0; i < 5; i++) {
try (DataFrame df =
ctx.sql(
"SELECT i FROM "
+ " (SELECT * FROM generate_series(1, 50000) AS t(i)) "
+ "ORDER BY i DESC");
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
assertNotNull(reader.getVectorSchemaRoot());
}
}
}
RuntimeStats after = ctx.runtimeStats();
assertTrue(
after.elapsedNanos() > before.elapsedNanos(),
() ->
"after.elapsedNanos="
+ after.elapsedNanos()
+ " not greater than before="
+ before.elapsedNanos());
}
}
@Test
void closedContextThrows() {
SessionContext ctx = new SessionContext();
ctx.close();
assertThrows(IllegalStateException.class, ctx::runtimeStats);
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/SessionContextTest.java 0000664 0000000 0000000 00000006561 15205164533 0032276 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
class SessionContextTest {
@Test
void canExecuteSelect1() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
assertEquals(1, reader.getVectorSchemaRoot().getRowCount());
}
}
@Test
void selectCountStarLineitem() throws Exception {
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
Assumptions.assumeTrue(
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerParquet("lineitem", lineitem.toAbsolutePath().toString());
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM lineitem");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
BigIntVector count = (BigIntVector) root.getVector(0);
assertEquals(6_001_215L, count.get(0));
}
}
}
@Test
void dataFrameCloseWithoutCollectReleasesNativeHandle() {
try (SessionContext ctx = new SessionContext()) {
// Planning succeeds but we never execute. close() must release the native plan.
DataFrame df = ctx.sql("SELECT 1");
df.close();
// Double-close must be a no-op.
df.close();
}
}
@Test
void dataFrameCollectTwiceFails() {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.sql("SELECT 1")) {
try (ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
} catch (Exception e) {
throw new RuntimeException(e);
}
assertThrows(IllegalStateException.class, () -> df.collect(allocator));
}
}
}
SessionContextTypedErrorTest.java 0000664 0000000 0000000 00000011433 15205164533 0034231 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertThrows;
import org.junit.jupiter.api.Test;
/**
* End-to-end checks that real native call sites surface as the right typed Java exception. The
* assertions are on exception type, not message text — the whole point of the typed
* hierarchy is to free callers from message-string sniffing.
*/
class SessionContextTypedErrorTest {
@Test
void unknownTableInSqlIsPlanException() {
try (SessionContext ctx = new SessionContext()) {
DataFusionException e =
assertThrows(
DataFusionException.class, () -> ctx.sql("SELECT * FROM nonexistent_table_xyz_42"));
assertInstanceOf(PlanException.class, e);
}
}
@Test
void malformedParquetFileIsIoException(@org.junit.jupiter.api.io.TempDir java.nio.file.Path dir)
throws java.io.IOException {
// Triggering a clean DataFusionError::IoError / ParquetError requires a
// file that exists but isn't a valid parquet -- a missing path is
// classified as "no table paths" (Execution) rather than IoError.
java.nio.file.Path bogus = dir.resolve("not-actually-parquet.parquet");
java.nio.file.Files.writeString(bogus, "this is not a parquet file");
try (SessionContext ctx = new SessionContext()) {
DataFusionException e =
assertThrows(
DataFusionException.class,
() -> ctx.registerParquet("t", bogus.toAbsolutePath().toString()));
assertInstanceOf(IoException.class, e);
}
}
@Test
void unknownConfigKeyIsConfigurationException() {
// SessionContextBuilder.setOption defers config-key validation to build()
// time, where DataFusion's `ConfigOptions::set` returns
// DataFusionError::Configuration for unrecognised keys.
SessionContextBuilder b =
SessionContext.builder().setOption("datafusion.bogus_made_up_key_for_typed_test", "x");
DataFusionException e = assertThrows(DataFusionException.class, b::build);
assertInstanceOf(ConfigurationException.class, e);
}
@Test
void divideByZeroIsExecutionException() throws java.io.IOException {
// SELECT 1/0 surfaces as DataFusionError::ArrowError(ArrowError::DivideByZero)
// -- the case Codex flagged that previously routed to IoException because
// the outer arm matched *any* ArrowError. ExecutionException is the right
// typed class for an arithmetic failure produced during query execution.
try (SessionContext ctx = new SessionContext();
org.apache.arrow.memory.RootAllocator allocator =
new org.apache.arrow.memory.RootAllocator()) {
DataFusionException e =
assertThrows(
DataFusionException.class,
() -> {
try (DataFrame df = ctx.sql("SELECT 1/0")) {
df.collect(allocator).close();
}
});
assertInstanceOf(ExecutionException.class, e);
}
}
@Test
void runtimeKeyRejectionIsParentDataFusionException() {
// The native-side guard against `datafusion.runtime.*` keys uses a
// stringly-typed `Err("...")?` (not a DataFusionError variant), so it
// surfaces as the parent DataFusionException — the catch-all path that
// protects callers from a wrong typed subclass when the underlying error
// doesn't carry a clean variant signal.
SessionContextBuilder b =
SessionContext.builder().setOption("datafusion.runtime.memory_limit", "2G");
DataFusionException e = assertThrows(DataFusionException.class, b::build);
// Strictly the parent class, not any subclass.
if (e instanceof PlanException
|| e instanceof IoException
|| e instanceof ConfigurationException
|| e instanceof ExecutionException
|| e instanceof ResourcesExhaustedException
|| e instanceof NotImplementedException) {
throw new AssertionError(
"expected parent DataFusionException, got " + e.getClass().getName());
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/SortExprTest.java 0000664 0000000 0000000 00000003714 15205164533 0031071 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
class SortExprTest {
@Test
void ascDefaultsToNullsLast() {
SortExpr s = SortExpr.asc("a");
assertTrue(s.ascending());
assertFalse(s.nullsFirst());
}
@Test
void descDefaultsToNullsFirst() {
SortExpr s = SortExpr.desc("a");
assertFalse(s.ascending());
assertTrue(s.nullsFirst());
}
@Test
void nullsFirstSetterRoundTrips() {
SortExpr s = SortExpr.asc("a").nullsFirst(true);
assertTrue(s.nullsFirst());
s.nullsFirst(false);
assertFalse(s.nullsFirst());
}
@Test
void factoryRejectsNullColumn() {
assertThrows(IllegalArgumentException.class, () -> SortExpr.asc(null));
assertThrows(IllegalArgumentException.class, () -> SortExpr.desc(null));
}
@Test
void accessorReturnsColumnName() {
assertEquals("foo", SortExpr.asc("foo").column());
assertEquals("bar", SortExpr.desc("bar").column());
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/TableProviderTest.java 0000664 0000000 0000000 00000037461 15205164533 0032053 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.channels.Channels;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.ipc.ArrowStreamReader;
import org.apache.arrow.vector.ipc.ArrowStreamWriter;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.Test;
class TableProviderTest {
private static final ArrowType INT32 = new ArrowType.Int(32, true);
private static final ArrowType UTF8 = new ArrowType.Utf8();
/**
* In-memory {@link TableProvider} fixture. The batches are serialised to Arrow IPC bytes once at
* construction (using a private allocator); each {@link #scan(BufferAllocator)} call returns a
* fresh {@link ArrowStreamReader} backed by those bytes, using the framework-supplied allocator.
*/
static final class InMemoryTableProvider implements TableProvider {
private final Schema schema;
private final byte[] ipcBytes;
private final AtomicInteger scanCount = new AtomicInteger();
InMemoryTableProvider(Schema schema, byte[] ipcBytes) {
this.schema = schema;
this.ipcBytes = ipcBytes;
}
/**
* Build a fixture from one or more vector-schema-root batches. The caller's allocator may be a
* temporary RootAllocator; this constructor reads all data into IPC bytes immediately.
*/
static InMemoryTableProvider fromBatches(Schema schema, List batches) {
return new InMemoryTableProvider(schema, serializeBatches(schema, batches));
}
static byte[] serializeBatches(Schema schema, List batches) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (BufferAllocator tmp = new RootAllocator();
VectorSchemaRoot stagingRoot = VectorSchemaRoot.create(schema, tmp);
ArrowStreamWriter writer =
new ArrowStreamWriter(stagingRoot, null, Channels.newChannel(baos))) {
writer.start();
for (VectorSchemaRoot batch : batches) {
stagingRoot.allocateNew();
int rowCount = batch.getRowCount();
stagingRoot.setRowCount(rowCount);
for (int i = 0; i < batch.getFieldVectors().size(); i++) {
org.apache.arrow.vector.FieldVector src = batch.getFieldVectors().get(i);
org.apache.arrow.vector.FieldVector dst = stagingRoot.getFieldVectors().get(i);
for (int r = 0; r < rowCount; r++) {
dst.copyFromSafe(r, r, src);
}
dst.setValueCount(rowCount);
}
writer.writeBatch();
}
writer.end();
} catch (IOException e) {
throw new RuntimeException("failed to serialize batches", e);
}
return baos.toByteArray();
}
@Override
public Schema schema() {
return schema;
}
@Override
public ArrowReader scan(BufferAllocator allocator) {
scanCount.incrementAndGet();
return new ArrowStreamReader(new ByteArrayInputStream(ipcBytes), allocator);
}
int scanCount() {
return scanCount.get();
}
}
/** Build a one-batch in-memory fixture of (id INT, name UTF8) with the given rows. */
private static InMemoryTableProvider buildTwoColumnTable(int[] ids, String[] names) {
Schema schema =
new Schema(
List.of(
new Field("id", FieldType.nullable(INT32), null),
new Field("name", FieldType.nullable(UTF8), null)));
try (BufferAllocator tmp = new RootAllocator();
VectorSchemaRoot root = VectorSchemaRoot.create(schema, tmp)) {
IntVector idVec = (IntVector) root.getVector("id");
VarCharVector nameVec = (VarCharVector) root.getVector("name");
int n = ids.length;
idVec.allocateNew(n);
nameVec.allocateNew(n);
for (int i = 0; i < n; i++) {
idVec.set(i, ids[i]);
nameVec.setSafe(i, names[i].getBytes());
}
idVec.setValueCount(n);
nameVec.setValueCount(n);
root.setRowCount(n);
return InMemoryTableProvider.fromBatches(schema, List.of(root));
}
}
@Test
void registerTable_selectStar_returnsAllRows() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
InMemoryTableProvider src =
buildTwoColumnTable(new int[] {1, 2, 3}, new String[] {"a", "b", "c"});
ctx.registerTable("t", src);
try (DataFrame df = ctx.sql("SELECT id, name FROM t ORDER BY id");
ArrowReader r = df.collect(allocator)) {
assertTrue(r.loadNextBatch());
VectorSchemaRoot out = r.getVectorSchemaRoot();
IntVector id = (IntVector) out.getVector("id");
VarCharVector name = (VarCharVector) out.getVector("name");
assertEquals(3, id.getValueCount());
assertEquals(1, id.get(0));
assertEquals(2, id.get(1));
assertEquals(3, id.get(2));
assertEquals("a", new String(name.get(0)));
assertEquals("b", new String(name.get(1)));
assertEquals("c", new String(name.get(2)));
while (r.loadNextBatch()) {}
}
assertEquals(1, src.scanCount());
}
}
@Test
void registerTable_unionAllSelf_callsScanTwice() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
InMemoryTableProvider src = buildTwoColumnTable(new int[] {1, 2}, new String[] {"a", "b"});
ctx.registerTable("t", src);
try (DataFrame df = ctx.sql("SELECT id FROM t UNION ALL SELECT id FROM t");
ArrowReader r = df.collect(allocator)) {
long total = 0;
while (r.loadNextBatch()) {
IntVector id = (IntVector) r.getVectorSchemaRoot().getVector("id");
total += id.getValueCount();
}
assertEquals(4, total);
}
assertEquals(2, src.scanCount());
}
}
@Test
void registerTable_emptyStream_yieldsNoRows() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
Schema schema = new Schema(List.of(new Field("id", FieldType.nullable(INT32), null)));
InMemoryTableProvider src = InMemoryTableProvider.fromBatches(schema, List.of());
ctx.registerTable("t", src);
try (DataFrame df = ctx.sql("SELECT id FROM t");
ArrowReader r = df.collect(allocator)) {
long total = 0;
while (r.loadNextBatch()) {
IntVector id = (IntVector) r.getVectorSchemaRoot().getVector("id");
total += id.getValueCount();
}
assertEquals(0, total);
}
}
}
@Test
void registerTable_projectSingleColumn_returnsOnlyThatColumn() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
InMemoryTableProvider src =
buildTwoColumnTable(new int[] {10, 20, 30}, new String[] {"x", "y", "z"});
ctx.registerTable("t", src);
try (DataFrame df = ctx.sql("SELECT name FROM t ORDER BY name");
ArrowReader r = df.collect(allocator)) {
assertTrue(r.loadNextBatch());
VectorSchemaRoot out = r.getVectorSchemaRoot();
assertEquals(1, out.getSchema().getFields().size());
VarCharVector name = (VarCharVector) out.getVector("name");
assertEquals(3, name.getValueCount());
assertEquals("x", new String(name.get(0)));
assertEquals("y", new String(name.get(1)));
assertEquals("z", new String(name.get(2)));
while (r.loadNextBatch()) {}
}
}
}
static final class ThrowingTableProvider implements TableProvider {
@Override
public Schema schema() {
return new Schema(List.of(new Field("id", FieldType.nullable(INT32), null)));
}
@Override
public ArrowReader scan(BufferAllocator allocator) {
throw new IllegalArgumentException("custom boom from TableProvider");
}
}
@Test
void registerTable_scanThrows_propagatesClassAndMessage() {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerTable("t", new ThrowingTableProvider());
RuntimeException ex =
org.junit.jupiter.api.Assertions.assertThrows(
RuntimeException.class,
() -> {
try (DataFrame df = ctx.sql("SELECT id FROM t");
ArrowReader r = df.collect(allocator)) {
while (r.loadNextBatch()) {}
}
});
String msg = ex.getMessage();
assertTrue(
msg.contains("IllegalArgumentException"),
"expected exception class in error, got: " + msg);
assertTrue(
msg.contains("custom boom from TableProvider"),
"expected user message in error, got: " + msg);
}
}
static final class NullReturningTableProvider implements TableProvider {
@Override
public Schema schema() {
return new Schema(List.of(new Field("id", FieldType.nullable(INT32), null)));
}
@Override
public ArrowReader scan(BufferAllocator allocator) {
return null;
}
}
@Test
void registerTable_scanReturnsNull_failsWithIllegalStateException() {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerTable("t", new NullReturningTableProvider());
RuntimeException ex =
org.junit.jupiter.api.Assertions.assertThrows(
RuntimeException.class,
() -> {
try (DataFrame df = ctx.sql("SELECT id FROM t");
ArrowReader r = df.collect(allocator)) {
while (r.loadNextBatch()) {}
}
});
String msg = ex.getMessage();
assertTrue(
msg.contains("IllegalStateException"),
"expected IllegalStateException in error, got: " + msg);
assertTrue(msg.contains("returned null"), "expected 'returned null' wording, got: " + msg);
}
}
/** Declares (id INT) but scan() returns (id INT, extra UTF8). */
static final class SchemaLyingTableProvider implements TableProvider {
@Override
public Schema schema() {
return new Schema(List.of(new Field("id", FieldType.nullable(INT32), null)));
}
@Override
public ArrowReader scan(BufferAllocator allocator) {
Schema actualSchema =
new Schema(
List.of(
new Field("id", FieldType.nullable(INT32), null),
new Field("extra", FieldType.nullable(UTF8), null)));
try (BufferAllocator tmp = new RootAllocator();
VectorSchemaRoot root = VectorSchemaRoot.create(actualSchema, tmp)) {
root.setRowCount(0);
byte[] ipc = InMemoryTableProvider.serializeBatches(actualSchema, List.of(root));
return new ArrowStreamReader(new ByteArrayInputStream(ipc), allocator);
}
}
}
@Test
void registerTable_schemaMismatch_failsQueryWithReadableError() {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerTable("t", new SchemaLyingTableProvider());
RuntimeException ex =
org.junit.jupiter.api.Assertions.assertThrows(
RuntimeException.class,
() -> {
try (DataFrame df = ctx.sql("SELECT id FROM t");
ArrowReader r = df.collect(allocator)) {
while (r.loadNextBatch()) {}
}
});
String msg = ex.getMessage();
assertTrue(
msg.contains("registered schema") || msg.contains("returned schema"),
"expected schema-mismatch wording, got: " + msg);
}
}
@Test
void registerTable_twoTables_joinable() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
InMemoryTableProvider left = buildTwoColumnTable(new int[] {1, 2}, new String[] {"a", "b"});
InMemoryTableProvider right = buildTwoColumnTable(new int[] {2, 3}, new String[] {"B", "C"});
ctx.registerTable("l", left);
ctx.registerTable("r", right);
int totalRows = 0;
int lidVal = -1;
int ridVal = -1;
String lnameVal = null;
String rnameVal = null;
try (DataFrame df =
ctx.sql(
"SELECT l.id AS lid, r.id AS rid, l.name AS lname, r.name AS rname"
+ " FROM l JOIN r ON l.id = r.id");
ArrowReader rd = df.collect(allocator)) {
while (rd.loadNextBatch()) {
VectorSchemaRoot out = rd.getVectorSchemaRoot();
IntVector lid = (IntVector) out.getVector("lid");
IntVector rid = (IntVector) out.getVector("rid");
VarCharVector lname = (VarCharVector) out.getVector("lname");
VarCharVector rname = (VarCharVector) out.getVector("rname");
int n = lid.getValueCount();
for (int i = 0; i < n; i++) {
if (totalRows == 0) {
lidVal = lid.get(i);
ridVal = rid.get(i);
lnameVal = new String(lname.get(i));
rnameVal = new String(rname.get(i));
}
totalRows++;
}
}
}
assertEquals(1, totalRows);
assertEquals(2, lidVal);
assertEquals(2, ridVal);
assertEquals("b", lnameVal);
assertEquals("B", rnameVal);
assertEquals(1, left.scanCount());
assertEquals(1, right.scanCount());
}
}
@Test
void simpleTableProvider_registerAndQuery_returnsRows() throws Exception {
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
InMemoryTableProvider backing =
buildTwoColumnTable(new int[] {1, 2}, new String[] {"a", "b"});
TableProvider provider = new SimpleTableProvider(backing.schema(), backing::scan);
ctx.registerTable("t", provider);
try (DataFrame df = ctx.sql("SELECT id, name FROM t ORDER BY id");
ArrowReader r = df.collect(allocator)) {
assertTrue(r.loadNextBatch());
VectorSchemaRoot out = r.getVectorSchemaRoot();
IntVector id = (IntVector) out.getVector("id");
VarCharVector name = (VarCharVector) out.getVector("name");
assertEquals(2, id.getValueCount());
assertEquals(1, id.get(0));
assertEquals(2, id.get(1));
assertEquals("a", new String(name.get(0)));
assertEquals("b", new String(name.get(1)));
while (r.loadNextBatch()) {}
}
assertEquals(1, backing.scanCount());
}
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/UnnestOptionsTest.java 0000664 0000000 0000000 00000002515 15205164533 0032131 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.Test;
class UnnestOptionsTest {
@Test
void defaultPreserveNullsIsTrue() {
assertTrue(new UnnestOptions().preserveNulls());
}
@Test
void preserveNullsSetterRoundTrips() {
UnnestOptions opts = new UnnestOptions().preserveNulls(false);
assertFalse(opts.preserveNulls());
opts.preserveNulls(true);
assertTrue(opts.preserveNulls());
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/proto/ 0000775 0000000 0000000 00000000000 15205164533 0026736 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/proto/ProtoGenerationTest.java0000664 0000000 0000000 00000002743 15205164533 0033566 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.proto;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import org.apache.datafusion.protobuf.LogicalPlanNode;
import org.junit.jupiter.api.Test;
/**
* Smoke test: confirms the datafusion-proto schema was downloaded, protoc generated Java sources,
* those sources landed on the compile classpath, and the {@code protobuf-java} runtime resolves.
*
*
Does not exercise any DataFusion plan semantics — those tests arrive with JVM-side plan
* construction.
*/
class ProtoGenerationTest {
@Test
void generatedClassIsLoadableAndConstructible() {
LogicalPlanNode node = LogicalPlanNode.newBuilder().build();
assertNotNull(node);
}
}
apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/proto/SchemaConverterTest.java0000664 0000000 0000000 00000010056 15205164533 0033533 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.proto;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.arrow.vector.types.DateUnit;
import org.apache.arrow.vector.types.FloatingPointPrecision;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.jupiter.api.Test;
import datafusion_common.DatafusionCommon;
class SchemaConverterTest {
@Test
void roundTripsPrimitivesAndMetadata() {
Map schemaMeta = new HashMap<>();
schemaMeta.put("origin", "test");
Map fieldMeta = new HashMap<>();
fieldMeta.put("ns", "demo");
List fields =
Arrays.asList(
new Field("a_bool", FieldType.nullable(ArrowType.Bool.INSTANCE), null),
new Field("a_i32", FieldType.notNullable(new ArrowType.Int(32, true)), null),
new Field("a_i64", FieldType.nullable(new ArrowType.Int(64, true)), null),
new Field("a_u32", FieldType.nullable(new ArrowType.Int(32, false)), null),
new Field(
"a_f64",
FieldType.nullable(new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)),
null),
new Field("a_str", FieldType.nullable(ArrowType.Utf8.INSTANCE), null),
new Field("a_date", FieldType.nullable(new ArrowType.Date(DateUnit.DAY)), null),
new Field(
"a_with_meta",
new FieldType(true, new ArrowType.Int(8, true), null, fieldMeta),
null));
Schema original = new Schema(fields, schemaMeta);
DatafusionCommon.Schema proto = SchemaConverter.toProto(original);
Schema roundTripped = SchemaConverter.fromProto(proto);
assertEquals(original, roundTripped);
}
@Test
void decimalPreservesPrecisionAndScale() {
Schema original =
new Schema(
List.of(
new Field("amount", FieldType.nullable(new ArrowType.Decimal(18, 5, 128)), null)));
DatafusionCommon.Schema proto = SchemaConverter.toProto(original);
Schema roundTripped = SchemaConverter.fromProto(proto);
assertEquals(original, roundTripped);
ArrowType.Decimal d = (ArrowType.Decimal) roundTripped.getFields().get(0).getType();
assertEquals(18, d.getPrecision());
assertEquals(5, d.getScale());
assertEquals(128, d.getBitWidth());
}
@Test
void unsupportedTypeRaisesUnsupportedOperationException() {
Field listField =
new Field(
"nested",
FieldType.nullable(new ArrowType.List()),
List.of(new Field("item", FieldType.nullable(new ArrowType.Int(32, true)), null)));
Schema original = new Schema(List.of(listField));
UnsupportedOperationException ex =
assertThrows(UnsupportedOperationException.class, () -> SchemaConverter.toProto(original));
assertTrue(
ex.getMessage().contains("List"),
"exception message should name the unsupported type, was: " + ex.getMessage());
}
}
SessionContextProtoTest.java 0000664 0000000 0000000 00000017547 15205164533 0034414 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/core/src/test/java/org/apache/datafusion/proto /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.proto;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.DateUnit;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.SessionContext;
import org.apache.datafusion.protobuf.BareTableReference;
import org.apache.datafusion.protobuf.EmptyRelationNode;
import org.apache.datafusion.protobuf.ListingTableScanNode;
import org.apache.datafusion.protobuf.LogicalExprNode;
import org.apache.datafusion.protobuf.LogicalPlanNode;
import org.apache.datafusion.protobuf.ProjectionColumns;
import org.apache.datafusion.protobuf.ProjectionNode;
import org.apache.datafusion.protobuf.SortExprNode;
import org.apache.datafusion.protobuf.SortNode;
import org.apache.datafusion.protobuf.TableReference;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
import datafusion_common.DatafusionCommon;
class SessionContextProtoTest {
@Test
void fromProtoExecutesProjectionOverEmptyRelation() throws Exception {
LogicalPlanNode plan =
LogicalPlanNode.newBuilder()
.setProjection(
ProjectionNode.newBuilder()
.setInput(
LogicalPlanNode.newBuilder()
.setEmptyRelation(
EmptyRelationNode.newBuilder().setProduceOneRow(true).build())
.build())
.addExpr(
LogicalExprNode.newBuilder()
.setLiteral(
DatafusionCommon.ScalarValue.newBuilder().setInt32Value(1).build())
.build())
.build())
.build();
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.fromProto(plan.toByteArray());
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
IntVector col = (IntVector) root.getVector(0);
assertEquals(1, col.get(0));
}
}
@Test
void tableSchemaReturnsExpectedLineitemFields() {
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
Assumptions.assumeTrue(
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
try (SessionContext ctx = new SessionContext()) {
ctx.registerParquet("lineitem", lineitem.toAbsolutePath().toString());
Schema schema = ctx.tableSchema("lineitem");
assertEquals(16, schema.getFields().size());
assertEquals("l_orderkey", schema.getFields().get(0).getName());
assertTrue(schema.getFields().get(0).getType() instanceof ArrowType.Int);
// l_extendedprice = Decimal128(15, 2)
ArrowType.Decimal price = (ArrowType.Decimal) schema.findField("l_extendedprice").getType();
assertEquals(15, price.getPrecision());
assertEquals(2, price.getScale());
// l_shipdate = Date(DAY)
ArrowType.Date ship = (ArrowType.Date) schema.findField("l_shipdate").getType();
assertEquals(DateUnit.DAY, ship.getUnit());
}
}
@Test
void fromProtoListingScanMatchesSql() throws Exception {
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
Assumptions.assumeTrue(
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
String absPath = lineitem.toAbsolutePath().toString();
try (BufferAllocator allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerParquet("lineitem", absPath);
Schema arrow = ctx.tableSchema("lineitem");
DatafusionCommon.Schema schemaProto = SchemaConverter.toProto(arrow);
LogicalExprNode orderKeyExpr =
LogicalExprNode.newBuilder()
.setColumn(DatafusionCommon.Column.newBuilder().setName("l_orderkey").build())
.build();
LogicalPlanNode plan =
LogicalPlanNode.newBuilder()
.setSort(
SortNode.newBuilder()
.setInput(
LogicalPlanNode.newBuilder()
.setListingScan(
ListingTableScanNode.newBuilder()
.setTableName(
TableReference.newBuilder()
.setBare(
BareTableReference.newBuilder()
.setTable("lineitem")
.build())
.build())
.addPaths(absPath)
.setFileExtension(".parquet")
.setSchema(schemaProto)
.setProjection(
ProjectionColumns.newBuilder()
.addColumns("l_orderkey")
.build())
.setParquet(
DatafusionCommon.ParquetFormat.getDefaultInstance())
.setTargetPartitions(1)
.build())
.build())
.addExpr(
SortExprNode.newBuilder()
.setExpr(orderKeyExpr)
.setAsc(true)
.setNullsFirst(false)
.build())
.setFetch(1)
.build())
.build();
long protoValue;
try (DataFrame df = ctx.fromProto(plan.toByteArray());
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
protoValue = ((BigIntVector) root.getVector(0)).get(0);
}
long sqlValue;
try (DataFrame df = ctx.sql("SELECT l_orderkey FROM lineitem ORDER BY l_orderkey LIMIT 1");
ArrowReader reader = df.collect(allocator)) {
assertTrue(reader.loadNextBatch());
VectorSchemaRoot root = reader.getVectorSchemaRoot();
assertEquals(1, root.getRowCount());
sqlValue = ((BigIntVector) root.getVector(0)).get(0);
}
assertEquals(sqlValue, protoValue);
}
}
}
apache-datafusion-java-0.1.0/dev/ 0000775 0000000 0000000 00000000000 15205164533 0016565 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/dev/changelog/ 0000775 0000000 0000000 00000000000 15205164533 0020514 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/dev/changelog/0.1.0.md 0000664 0000000 0000000 00000015473 15205164533 0021504 0 ustar 00root root 0000000 0000000
# DataFusion Java 0.1.0 Changelog
This release consists of 48 commits from 2 contributors. See credits at the end of this changelog for more information.
**Breaking changes:**
- feat(udf)!: switch ScalarFunction.evaluate to ColumnarValue API (closes #62) [#64](https://github.com/apache/datafusion-java/pull/64) (andygrove)
**Fixed bugs:**
- fix(udf): declare scalar UDF signatures with Field, not ArrowType [#59](https://github.com/apache/datafusion-java/pull/59) (LantaoJin)
**Implemented enhancements:**
- feat: initial seed of Apache DataFusion Java bindings [#1](https://github.com/apache/datafusion-java/pull/1) (andygrove)
- feat(proto): generate datafusion-proto Java classes at build time [#9](https://github.com/apache/datafusion-java/pull/9) (andygrove)
- feat(parquet): expose ParquetReadOptions via registerParquet and readParquet [#18](https://github.com/apache/datafusion-java/pull/18) (andygrove)
- feat(dataframe): add select/filter/count/show methods [#19](https://github.com/apache/datafusion-java/pull/19) (andygrove)
- feat(proto): execute Java-built protobuf plans via SessionContext [#13](https://github.com/apache/datafusion-java/pull/13) (andygrove)
- feat(csv): expose CsvReadOptions via registerCsv and readCsv [#21](https://github.com/apache/datafusion-java/pull/21) (LantaoJin)
- feat(write): add DataFrame.writeParquet with ParquetWriteOptions [#27](https://github.com/apache/datafusion-java/pull/27) (andygrove)
- feat: configure SessionContext and RuntimeEnv via builder [#28](https://github.com/apache/datafusion-java/pull/28) (andygrove)
- feat(dataframe): add limit, distinct, dropColumns, withColumnRenamed [#30](https://github.com/apache/datafusion-java/pull/30) (andygrove)
- feat(builder): expose ConfigOptions.set/get as setOption / setOptions / getOption [#49](https://github.com/apache/datafusion-java/pull/49) (LantaoJin)
- feat(json): expose NdJsonReadOptions via registerJson and readJson [#47](https://github.com/apache/datafusion-java/pull/47) (LantaoJin)
- feat(dataframe): expose withColumn and unnestColumns [#54](https://github.com/apache/datafusion-java/pull/54) (LantaoJin)
- feat: add Java scalar UDF support [#46](https://github.com/apache/datafusion-java/pull/46) (andygrove)
- feat(arrow): expose Arrow IPC reader via registerArrow and readArrow [#52](https://github.com/apache/datafusion-java/pull/52) (LantaoJin)
- feat(dataframe): add writeCsv with CsvWriteOptions [#53](https://github.com/apache/datafusion-java/pull/53) (LantaoJin)
- feat(dataframe): add executeStream(allocator) for incremental batch iteration [#51](https://github.com/apache/datafusion-java/pull/51) (LantaoJin)
- feat(avro): expose Avro reader via registerAvro and readAvro [#60](https://github.com/apache/datafusion-java/pull/60) (LantaoJin)
- feat(dataframe): add writeJson with JsonWriteOptions [#61](https://github.com/apache/datafusion-java/pull/61) (LantaoJin)
- feat(datasource): add Java-implemented data sources [#65](https://github.com/apache/datafusion-java/pull/65) (andygrove)
- feat(dataframe): add schema, explain, cache, and describe introspection methods [#71](https://github.com/apache/datafusion-java/pull/71) (LantaoJin)
- feat(session): register object_store backends on SessionContextBuilder [#73](https://github.com/apache/datafusion-java/pull/73) (LantaoJin)
- feat(dataframe): add join, joinOn, and JoinType [#72](https://github.com/apache/datafusion-java/pull/72) (LantaoJin)
- feat(dataframe): expose sort and repartition [#66](https://github.com/apache/datafusion-java/pull/66) (LantaoJin)
- feat: typed DataFusionException hierarchy across the JNI boundary [#81](https://github.com/apache/datafusion-java/pull/81) (LantaoJin)
- feat(session): configure DataFusion's built-in CacheManager from Java [#78](https://github.com/apache/datafusion-java/pull/78) (LantaoJin)
- feat(metrics): add SessionContext.memoryUsage and runtimeStats [#85](https://github.com/apache/datafusion-java/pull/85) (LantaoJin)
- feat(loader): bundle native library inside the JAR [#77](https://github.com/apache/datafusion-java/pull/77) (andygrove)
**Documentation updates:**
- docs: replace roadmap with project-status checklist [#20](https://github.com/apache/datafusion-java/pull/20) (andygrove)
- docs: add GitHub issue and PR templates [#23](https://github.com/apache/datafusion-java/pull/23) (andygrove)
- docs: add Sphinx documentation site with user and contributor guides [#24](https://github.com/apache/datafusion-java/pull/24) (andygrove)
- docs: publish Javadoc as part of the User Guide [#31](https://github.com/apache/datafusion-java/pull/31) (andygrove)
- docs: remove project-status checklist [#34](https://github.com/apache/datafusion-java/pull/34) (andygrove)
**Other:**
- chore: remove protected_branches block and format workflow [#7](https://github.com/apache/datafusion-java/pull/7) (andygrove)
- chore(.asf.yaml): null out main branch protection to clear stale rules [#11](https://github.com/apache/datafusion-java/pull/11) (andygrove)
- ci: add build workflow that runs make test [#15](https://github.com/apache/datafusion-java/pull/15) (andygrove)
- ci: replace disallowed third-party actions in build workflow [#16](https://github.com/apache/datafusion-java/pull/16) (andygrove)
- ci: publish docs site to datafusion.apache.org/java/ [#26](https://github.com/apache/datafusion-java/pull/26) (andygrove)
- refactor: pass csv and parquet read options via protobuf [#29](https://github.com/apache/datafusion-java/pull/29) (andygrove)
- build: add examples module on a multi-module Maven build [#32](https://github.com/apache/datafusion-java/pull/32) (andygrove)
- chore(release): add ASF release scripts for source tarballs [#76](https://github.com/apache/datafusion-java/pull/76) (andygrove)
- chore(release): add multi-platform release JAR build pipeline [#87](https://github.com/apache/datafusion-java/pull/87) (andygrove)
## Credits
Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor.
```
31 Andy Grove
17 Lantao Jin
```
Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release.
apache-datafusion-java-0.1.0/dev/release/ 0000775 0000000 0000000 00000000000 15205164533 0020205 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/dev/release/README.md 0000664 0000000 0000000 00000014744 15205164533 0021476 0 ustar 00root root 0000000 0000000
# Release Process
This guide is for maintainers creating release candidates of Apache
DataFusion Java and running the release process.
## Release Prerequisites
### Add a git remote for the `apache` repo
The instructions below assume the upstream git repo
`git@github.com:apache/datafusion-java.git` is configured as the remote
named `apache`:
```shell
git remote add apache git@github.com:apache/datafusion-java.git
```
### Add your GPG public key to the SVN `KEYS` file
If you will be releasing the final tarball, your GPG public key must be
present in:
- https://dist.apache.org/repos/dist/dev/datafusion/KEYS
- https://dist.apache.org/repos/dist/release/datafusion/KEYS
See https://infra.apache.org/release-signing.html#generate for generating
keys.
Committers can add signing keys via Subversion using their ASF account:
```shell
svn co https://dist.apache.org/repos/dist/dev/datafusion
cd datafusion
(gpg --list-sigs "Your Name" && gpg --armor --export "Your Name") >> KEYS
svn commit KEYS -m "Add key for Your Name"
```
## Release Process: Step by Step
As part of the Apache governance model, official releases consist of
signed source tarballs approved by the PMC.
### 1. Pick a Release Candidate (RC) number
Pick numbers in sequential order, with `1` for `rc1`, `2` for `rc2`, and
so on.
### 2. Create the Git tag for the Release Candidate
Release tags look like `0.1.0`, and release candidate tags look like
`0.1.0-rc1`.
```shell
git fetch apache
git tag 0.1.0-rc1 apache/main
git push apache 0.1.0-rc1
```
### 3. Create, sign, and upload the tarball
Run `create-tarball.sh` with the version and RC number:
```shell
./dev/release/create-tarball.sh 0.1.0 1
```
The script:
1. Creates and uploads the release candidate artifacts to the
[`dist/dev/datafusion`](https://dist.apache.org/repos/dist/dev/datafusion)
location on the Apache SVN distribution server, under
`apache-datafusion-java--rc/`.
2. Prints an email template to send to `dev@datafusion.apache.org` for
the release vote.
### 4. Vote on the Release Candidate
Send the email printed by the script to `dev@datafusion.apache.org`.
The vote must remain open for at least 72 hours and requires at least
three PMC `+1` votes and no `-1` votes to pass.
### Verifying Release Candidates
`dev/release/verify-release-candidate.sh` downloads the candidate
tarball, verifies its signature and checksums, and builds and tests the
source distribution:
```shell
./dev/release/verify-release-candidate.sh 0.1.0 1
```
The script requires `curl`, `git`, `gpg`, a C toolchain, `java`, and
either `shasum` or `sha256sum`/`sha512sum` to be on the `PATH`. It will
install a temporary Rust toolchain into the sandbox directory.
### 5. If the Vote Passes: Promote the Tarball
Move artifacts from the `dev` SVN area to the `release` area using
`release-tarball.sh`:
```shell
./dev/release/release-tarball.sh 0.1.0 1
```
### 6. Create the Release Git Tag
Tag the same release candidate commit with the final release tag:
```shell
git checkout 0.1.0-rc1
git tag 0.1.0
git push apache 0.1.0
```
### 7. Add the Release to Apache Reporter
Add the release at https://reporter.apache.org/addrelease.html?datafusion
so it appears in the project's board report.
### 8. Delete Old RCs and Releases
See the ASF documentation on [when to
archive](https://www.apache.org/legal/release-policy.html#when-to-archive).
Release candidates should be deleted once the release is published:
```shell
svn ls https://dist.apache.org/repos/dist/dev/datafusion | grep datafusion-java
svn delete -m "delete old DataFusion Java RC" \
https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-java-0.1.0-rc1/
```
Only the latest release should remain in the `release` SVN area. Delete
older releases after publishing a new one:
```shell
svn ls https://dist.apache.org/repos/dist/release/datafusion | grep datafusion-java
svn delete -m "delete old DataFusion Java release" \
https://dist.apache.org/repos/dist/release/datafusion/datafusion-java-0.1.0
```
## Binary Release: Multi-Platform JAR
Source tarballs are the official Apache release artifact, but consumers
also expect a published JAR on Maven Central that bundles native libs
for the common platforms. This section covers building that JAR.
### Prerequisites (release manager machine)
- macOS host (Apple Silicon or Intel)
- Docker Desktop running with BuildKit enabled
- Java 17+
- Rust toolchain via rustup
- `gpg` configured with a key listed in the ASF KEYS file
- `xmllint` on `PATH` (pre-installed on macOS; `libxml2-utils` on Debian/Ubuntu)
### Build the multi-platform JAR
`build-release.sh` clones the repo into two Linux Docker containers
(one for `linux/amd64`, one for `linux/arm64`), builds the native
`.so` libraries inside each, then builds the two macOS `.dylib`
libraries directly on the host. All four libraries are placed in the
JAR's resource tree at
`org/apache/datafusion///lib.`, and the
JAR is installed into a temporary local Maven repository.
```shell
./dev/release/build-release.sh
```
The script prints the local Maven repo path at the end. Inspect the JAR
to verify all four native libraries are bundled:
```shell
unzip -l "$JAR" | grep org/apache/datafusion/
```
### Publish to Apache Nexus staging
Once the local Maven repo from `build-release.sh` looks correct, sign
and upload to Apache Nexus staging using `publish-to-maven.sh`:
```shell
./dev/release/publish-to-maven.sh -u -r
```
The script prompts for the ASF password and GPG passphrase, creates a
staging repository on `repository.apache.org`, signs every artifact,
uploads it, and closes the staging repository. Verify in the Nexus UI
that the staged artifacts look correct before promoting to release.
apache-datafusion-java-0.1.0/dev/release/build-release.sh 0000775 0000000 0000000 00000013661 15205164533 0023270 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Build a multi-platform datafusion-java JAR bundling native libs for
# linux/amd64, linux/aarch64, darwin/x86_64, and darwin/aarch64. The
# resulting JAR is installed into a temporary local Maven repository
# whose path is printed at the end. This script must run on a macOS host.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
PROJECT_HOME="$(cd "${SCRIPT_DIR}/../.." >/dev/null && pwd)"
REPO="https://github.com/apache/datafusion-java.git"
BRANCH="main"
IMGTAG="latest"
function usage {
cat <&2
exit 1
fi
JAVA_VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F '.' '{print $1}')
if [ -z "$JAVA_VERSION" ] || [ "$JAVA_VERSION" -lt 17 ]; then
echo "Java 17+ is required. Found: $(java -version 2>&1 | head -n 1)" >&2
exit 1
fi
HOST_ARCH="$(uname -m)" # arm64 (Apple Silicon) or x86_64 (Intel)
case "$HOST_ARCH" in
arm64)
HOST_DARWIN_DIR="aarch64"
OTHER_DARWIN_TARGET="x86_64-apple-darwin"
OTHER_DARWIN_DIR="x86_64"
;;
x86_64)
HOST_DARWIN_DIR="x86_64"
OTHER_DARWIN_TARGET="aarch64-apple-darwin"
OTHER_DARWIN_DIR="aarch64"
;;
*)
echo "Unsupported macOS arch: $HOST_ARCH" >&2
exit 1
;;
esac
CONTAINER_AMD64="datafusion-java-amd64-builder-container"
CONTAINER_ARM64="datafusion-java-arm64-builder-container"
IMAGE_AMD64="datafusion-java-rm-amd64:${IMGTAG}"
IMAGE_ARM64="datafusion-java-rm-arm64:${IMGTAG}"
CLEANUP=1
cleanup() {
[ "$CLEANUP" != "0" ] || return 0
echo "Cleaning up build containers..."
docker rm -f "$CONTAINER_AMD64" "$CONTAINER_ARM64" >/dev/null 2>&1 || true
CLEANUP=0
}
trap cleanup SIGINT SIGTERM EXIT
echo "Cleaning leftover builder containers from any prior interrupted run"
docker rm -f "$CONTAINER_AMD64" "$CONTAINER_ARM64" >/dev/null 2>&1 || true
echo "Cleaning previous Java and native build output"
(cd "$PROJECT_HOME" && ./mvnw -q clean)
(cd "$PROJECT_HOME/native" && cargo clean)
echo "Building amd64 builder image"
docker build --no-cache \
--platform=linux/amd64 \
-t "$IMAGE_AMD64" \
"$SCRIPT_DIR/datafusion-java-rm"
echo "Building arm64 builder image"
docker build --no-cache \
--platform=linux/arm64 \
-t "$IMAGE_ARM64" \
"$SCRIPT_DIR/datafusion-java-rm"
echo "Building linux/amd64 native lib"
docker run --name "$CONTAINER_AMD64" \
--platform=linux/amd64 \
"$IMAGE_AMD64" "$REPO" "$BRANCH"
echo "Building linux/aarch64 native lib"
docker run --name "$CONTAINER_ARM64" \
--platform=linux/arm64 \
"$IMAGE_ARM64" "$REPO" "$BRANCH"
JVM_TARGET_DIR="$PROJECT_HOME/core/target/classes/org/apache/datafusion"
mkdir -p "$JVM_TARGET_DIR/linux/amd64"
docker cp \
"$CONTAINER_AMD64:/opt/datafusion-java-rm/datafusion-java/native/target/release/libdatafusion_jni.so" \
"$JVM_TARGET_DIR/linux/amd64/"
mkdir -p "$JVM_TARGET_DIR/linux/aarch64"
docker cp \
"$CONTAINER_ARM64:/opt/datafusion-java-rm/datafusion-java/native/target/release/libdatafusion_jni.so" \
"$JVM_TARGET_DIR/linux/aarch64/"
echo "Building macOS native libs on the host (host=$HOST_ARCH)"
rustup target add "$OTHER_DARWIN_TARGET"
(cd "$PROJECT_HOME/native" && cargo build --release)
(cd "$PROJECT_HOME/native" && cargo build --release --target "$OTHER_DARWIN_TARGET")
mkdir -p "$JVM_TARGET_DIR/darwin/$HOST_DARWIN_DIR"
cp "$PROJECT_HOME/native/target/release/libdatafusion_jni.dylib" \
"$JVM_TARGET_DIR/darwin/$HOST_DARWIN_DIR/"
mkdir -p "$JVM_TARGET_DIR/darwin/$OTHER_DARWIN_DIR"
cp "$PROJECT_HOME/native/target/$OTHER_DARWIN_TARGET/release/libdatafusion_jni.dylib" \
"$JVM_TARGET_DIR/darwin/$OTHER_DARWIN_DIR/"
echo "Installing JAR into local Maven repo"
LOCAL_REPO=$(mktemp -d /tmp/datafusion-java-staging-repo-XXXXXX)
(cd "$PROJECT_HOME" && ./mvnw \
"-Dmaven.repo.local=$LOCAL_REPO" \
"-Ddatafusion.native.profile=release" \
-DskipTests install)
echo ""
echo "===================================================================="
echo "Multi-platform JAR installed to local Maven repo: $LOCAL_REPO"
JAR_PATH=$(find "$LOCAL_REPO/org/apache/datafusion/datafusion-java" -name 'datafusion-java-*.jar' \
-not -name '*-sources.jar' -not -name '*-javadoc.jar' | head -n 1)
echo "JAR: $JAR_PATH"
echo "Bundled native libraries:"
unzip -l "$JAR_PATH" | grep -E 'libdatafusion_jni\.(so|dylib)$' || true
echo "===================================================================="
apache-datafusion-java-0.1.0/dev/release/check-rat-report.py 0000775 0000000 0000000 00000003440 15205164533 0023735 0 ustar 00root root 0000000 0000000 #!/usr/bin/python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import fnmatch
import re
import sys
import xml.etree.ElementTree as ET
if len(sys.argv) != 3:
sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" %
sys.argv[0])
sys.exit(1)
exclude_globs_filename = sys.argv[1]
xml_filename = sys.argv[2]
globs = [line.strip() for line in open(exclude_globs_filename, "r")]
tree = ET.parse(xml_filename)
root = tree.getroot()
resources = root.findall('resource')
all_ok = True
for r in resources:
approvals = r.findall('license-approval')
if not approvals or approvals[0].attrib['name'] == 'true':
continue
clean_name = re.sub('^[^/]+/', '', r.attrib['name'])
excluded = False
for g in globs:
if fnmatch.fnmatch(clean_name, g):
excluded = True
break
if not excluded:
sys.stdout.write("NOT APPROVED: %s (%s): %s\n" % (
clean_name, r.attrib['name'], approvals[0].attrib['name']))
all_ok = False
if not all_ok:
sys.exit(1)
print('OK')
sys.exit(0)
apache-datafusion-java-0.1.0/dev/release/create-tarball.sh 0000775 0000000 0000000 00000010611 15205164533 0023425 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Adapted from https://github.com/apache/datafusion/blob/main/dev/release/create-tarball.sh
# This script creates a signed tarball in
# dev/dist/apache-datafusion-java--rc/apache-datafusion-java-.tar.gz
# and uploads it to the "dev" area of the dist.apache.datafusion repository,
# then prints an email template for the dev@datafusion.apache.org list for a
# formal vote.
#
# See dev/release/README.md for full release instructions.
#
# Requirements:
#
# 1. gpg setup for signing and have uploaded your public
# signature to https://pgp.mit.edu/ and the project KEYS file.
#
# 2. Logged into the Apache SVN server with the appropriate
# credentials.
#
# Based in part on 02-source.sh from apache/arrow.
set -e
SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)"
if [ "$#" -ne 2 ]; then
echo "Usage: $0 "
echo "ex. $0 0.1.0 1"
exit
fi
version=$1
rc=$2
tag="${version}-rc${rc}"
release=apache-datafusion-java-${version}
distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc}
tarname=${release}.tar.gz
tarball=${distdir}/${tarname}
url="https://dist.apache.org/repos/dist/dev/datafusion/${release}-rc${rc}"
echo "Attempting to create ${tarball} from tag ${tag}"
release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag})
if [ -z "$release_hash" ]; then
echo "Cannot continue: unknown git tag: ${tag}"
exit 1
fi
echo "Draft email for dev@datafusion.apache.org mailing list"
echo ""
echo "---------------------------------------------------------"
cat < containing the files in git at $release_hash
# the files in the tarball are prefixed with apache-datafusion-java-/
mkdir -p ${distdir}
(cd "${SOURCE_TOP_DIR}" && git archive ${release_hash} --prefix ${release}/ | gzip > ${tarball})
echo "Signing tarball and creating checksums"
gpg --armor --output ${tarball}.asc --detach-sig ${tarball}
# create checksums with relative path of tarball so that they can be verified
# with a command such as
# shasum --check apache-datafusion-java-0.1.0.tar.gz.sha512
(cd ${distdir} && shasum -a 256 ${tarname}) > ${tarball}.sha256
(cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512
echo "Uploading to datafusion dist/dev to ${url}"
svn co --depth=empty https://dist.apache.org/repos/dist/dev/datafusion ${SOURCE_TOP_DIR}/dev/dist
svn add ${distdir}
svn ci -m "Apache DataFusion Java ${version} RC${rc}" ${distdir}
apache-datafusion-java-0.1.0/dev/release/datafusion-java-rm/ 0000775 0000000 0000000 00000000000 15205164533 0023675 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/dev/release/datafusion-java-rm/Dockerfile 0000664 0000000 0000000 00000004307 15205164533 0025673 0 ustar 00root root 0000000 0000000 #
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM ubuntu:20.04
USER root
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NONINTERACTIVE_SEEN=true
ENV LC_ALL=C
RUN export LC_ALL=C \
&& apt-get update \
&& apt-get install --no-install-recommends -y \
ca-certificates \
build-essential \
curl \
wget \
git \
llvm \
clang \
libssl-dev \
cmake \
cpio \
libxml2-dev \
patch \
bzip2 \
libbz2-dev \
zlib1g-dev \
default-jdk \
unzip \
gcc-10 \
g++-10 \
cpp-10
ENV CC="gcc-10"
ENV CXX="g++-10"
# protoc — picks the host arch automatically by reading $(uname -m)
RUN PB_REL="https://github.com/protocolbuffers/protobuf/releases" \
&& ARCH="$(uname -m)" \
&& case "$ARCH" in \
x86_64) PB_ARCH="x86_64" ;; \
aarch64) PB_ARCH="aarch_64" ;; \
*) echo "Unsupported arch: $ARCH" >&2; exit 1 ;; \
esac \
&& curl -LO "$PB_REL/download/v30.2/protoc-30.2-linux-${PB_ARCH}.zip" \
&& unzip "protoc-30.2-linux-${PB_ARCH}.zip" -d /root/.local \
&& rm "protoc-30.2-linux-${PB_ARCH}.zip"
ENV PATH="$PATH:/root/.local/bin"
# Rust
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
COPY build-native-libs.sh /opt/datafusion-java-rm/build-native-libs.sh
WORKDIR /opt/datafusion-java-rm
ENTRYPOINT ["/opt/datafusion-java-rm/build-native-libs.sh"]
apache-datafusion-java-0.1.0/dev/release/datafusion-java-rm/build-native-libs.sh 0000775 0000000 0000000 00000002602 15205164533 0027546 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Build the datafusion_jni release native lib inside a Linux container.
# The container's --platform determines the target arch; we just build.
set -euo pipefail
REPO=${1:-}
BRANCH=${2:-}
if [ -z "$REPO" ] || [ -z "$BRANCH" ]; then
echo "Usage: $0 " >&2
exit 1
fi
echo "Building datafusion_jni for $(uname -m) from ${REPO}/${BRANCH}"
rm -rf datafusion-java
git clone "$REPO" datafusion-java
cd datafusion-java
git checkout "$BRANCH"
cd native
cargo build --release
echo "Built $(pwd)/target/release/libdatafusion_jni.so"
ls -l target/release/libdatafusion_jni.so
apache-datafusion-java-0.1.0/dev/release/publish-to-maven.sh 0000775 0000000 0000000 00000011262 15205164533 0023740 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###
# Based on Apache DataFusion Comet's publish-to-maven.sh (itself adapted
# from Spark's release-build.sh).
###
function usage {
local NAME=$(basename "$0")
cat << EOF
usage: $NAME options
Publish signed artifacts to Apache Nexus staging.
Options:
-u ASF_USERNAME Username of ASF committer account
-r LOCAL_REPO Path to the local Maven repo created by build-release.sh
The following will be prompted for:
ASF_PASSWORD Password of ASF committer account
GPG_KEY Optional: specific GPG key id to sign with
GPG_PASSPHRASE Passphrase for the GPG signing key
EOF
exit 1
}
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
PROJECT_HOME="$(cd "${SCRIPT_DIR}/../.." >/dev/null && pwd)"
ASF_USERNAME=""
LOCAL_REPO=""
NEXUS_ROOT=https://repository.apache.org/service/local/staging
NEXUS_PROFILE=789e15c00fd47
while getopts "u:r:h" opt; do
case "$opt" in
u) ASF_USERNAME="$OPTARG" ;;
r) LOCAL_REPO="$OPTARG" ;;
h|*) usage ;;
esac
done
if [ -z "$LOCAL_REPO" ]; then
echo "Please provide the local Maven repo path (-r), as produced by build-release.sh." >&2
usage
fi
if [ -z "$ASF_USERNAME" ]; then
read -p "ASF Username : " ASF_USERNAME && echo ""
fi
read -s -p "ASF Password : " ASF_PASSWORD && echo ""
read -s -p "GPG Key (optional) : " GPG_KEY && echo ""
read -s -p "GPG Passphrase : " GPG_PASSPHRASE && echo ""
if [ -z "$ASF_USERNAME" ] || [ -z "$ASF_PASSWORD" ] || [ -z "$GPG_PASSPHRASE" ]; then
echo "Missing credentials" >&2
exit 1
fi
GPG="gpg --pinentry-mode loopback"
if [ -n "$GPG_KEY" ]; then
GPG="$GPG -u $GPG_KEY"
fi
SHA1SUM=$(which sha1sum || which shasum)
GIT_HASH=$(cd "$PROJECT_HOME" && git rev-parse --short HEAD)
echo "Creating Nexus staging repository"
REPO_REQUEST="Apache DataFusion Java (commit $GIT_HASH)"
REPO_REQUEST_RESPONSE=$(curl -X POST -d "$REPO_REQUEST" -u "$ASF_USERNAME:$ASF_PASSWORD" \
-H "Content-Type:application/xml" \
"$NEXUS_ROOT/profiles/$NEXUS_PROFILE/start")
STAGED_REPO_ID=$(echo "$REPO_REQUEST_RESPONSE" | xmllint --xpath "//stagedRepositoryId/text()" -)
echo "Created Nexus staging repository: $STAGED_REPO_ID"
if [ -z "$STAGED_REPO_ID" ]; then
echo "Error creating staged repository" >&2
echo "$REPO_REQUEST_RESPONSE" >&2
exit 1
fi
echo "Deploying artifacts from $LOCAL_REPO"
pushd "$LOCAL_REPO/org/apache/datafusion" >/dev/null
# Remove any extra files that mvn install might have written alongside
# the jar/pom (e.g. -lastUpdated metadata files).
find . -type f | grep -v '\.jar$' | grep -v '\.pom$' | xargs rm -f || true
echo "Creating hash and signature files"
for file in $(find . -type f); do
echo "$GPG_PASSPHRASE" | $GPG --passphrase-fd 0 --output "$file.asc" \
--detach-sig --armour "$file"
if command -v md5 >/dev/null; then
md5 -q "$file" > "$file.md5"
else
md5sum "$file" | cut -f1 -d' ' > "$file.md5"
fi
$SHA1SUM "$file" | cut -f1 -d' ' > "$file.sha1"
done
NEXUS_UPLOAD=$NEXUS_ROOT/deployByRepositoryId/$STAGED_REPO_ID
echo "Uploading files to $NEXUS_UPLOAD"
for file in $(find . -type f); do
FILE_SHORT=$(echo "$file" | sed -e "s/\.\///")
DEST_URL="$NEXUS_UPLOAD/org/apache/datafusion/$FILE_SHORT"
echo " Uploading $FILE_SHORT"
curl --fail-with-body \
-u "$ASF_USERNAME:$ASF_PASSWORD" \
--upload-file "$FILE_SHORT" "$DEST_URL"
done
echo "Closing nexus staging repository"
REPO_REQUEST="$STAGED_REPO_IDApache DataFusion Java (commit $GIT_HASH)"
curl --fail-with-body -X POST -d "$REPO_REQUEST" -u "$ASF_USERNAME:$ASF_PASSWORD" \
-H "Content-Type:application/xml" \
"$NEXUS_ROOT/profiles/$NEXUS_PROFILE/finish"
echo "Closed Nexus staging repository: $STAGED_REPO_ID"
popd >/dev/null
apache-datafusion-java-0.1.0/dev/release/rat_exclude_files.txt 0000664 0000000 0000000 00000000534 15205164533 0024431 0 ustar 00root root 0000000 0000000 *.gitignore
*.dockerignore
.github/pull_request_template.md
.github/ISSUE_TEMPLATE/bug_report.yml
.github/ISSUE_TEMPLATE/feature_request.yml
.gitmodules
.mvn/wrapper/maven-wrapper.properties
mvnw
mvnw.cmd
native/Cargo.lock
dev/release/rat_exclude_files.txt
docs/source/_static/**
docs/source/conf.py
rust-toolchain
target/*
**/target/*
tpch-data/*
apache-datafusion-java-0.1.0/dev/release/release-tarball.sh 0000775 0000000 0000000 00000004622 15205164533 0023607 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Adapted from https://github.com/apache/datafusion/blob/main/dev/release/release-tarball.sh
# This script copies a tarball from the "dev" area of the
# dist.apache.datafusion repository to the "release" area.
#
# It should only be run after the release has been approved by the Apache
# DataFusion PMC.
#
# See dev/release/README.md for full release instructions.
#
# Based in part on post-01-upload.sh from apache/arrow.
set -e
set -u
if [ "$#" -ne 2 ]; then
echo "Usage: $0 "
echo "ex. $0 0.1.0 1"
exit
fi
version=$1
rc=$2
read -r -p "Proceed to release tarball for ${version}-rc${rc}? [y/N]: " answer
answer=${answer:-no}
if [ "${answer}" != "y" ]; then
echo "Cancelled tarball release!"
exit 1
fi
tmp_dir=tmp-apache-datafusion-java-dist
echo "Recreate temporary directory: ${tmp_dir}"
rm -rf ${tmp_dir}
mkdir -p ${tmp_dir}
echo "Clone dev dist repository"
svn \
co \
https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-java-${version}-rc${rc} \
${tmp_dir}/dev
echo "Clone release dist repository"
svn co https://dist.apache.org/repos/dist/release/datafusion ${tmp_dir}/release
echo "Copy ${version}-rc${rc} to release working copy"
release_version=datafusion-java-${version}
mkdir -p ${tmp_dir}/release/${release_version}
cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/
svn add ${tmp_dir}/release/${release_version}
echo "Commit release"
svn ci -m "Apache DataFusion Java ${version}" ${tmp_dir}/release
echo "Clean up"
rm -rf ${tmp_dir}
echo "Success! The release is available here:"
echo " https://dist.apache.org/repos/dist/release/datafusion/${release_version}"
apache-datafusion-java-0.1.0/dev/release/verify-release-candidate.sh 0000775 0000000 0000000 00000011044 15205164533 0025400 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Adapted from https://github.com/apache/datafusion/blob/main/dev/release/verify-release-candidate.sh
# Check that required dependencies are installed
check_dependencies() {
local missing_deps=0
local required_deps=("curl" "git" "gpg" "cc" "java")
# Either shasum or sha256sum/sha512sum are required
local has_sha_tools=0
for dep in "${required_deps[@]}"; do
if ! command -v $dep &> /dev/null; then
echo "Error: $dep is not installed or not in PATH"
missing_deps=1
fi
done
if command -v shasum &> /dev/null; then
has_sha_tools=1
elif command -v sha256sum &> /dev/null && command -v sha512sum &> /dev/null; then
has_sha_tools=1
else
echo "Error: Neither shasum nor sha256sum/sha512sum are installed or in PATH"
missing_deps=1
fi
if [ $missing_deps -ne 0 ]; then
echo "Please install missing dependencies and try again"
exit 1
fi
}
case $# in
2) VERSION="$1"
RC_NUMBER="$2"
;;
*) echo "Usage: $0 X.Y.Z RC_NUMBER"
exit 1
;;
esac
set -e
set -x
set -o pipefail
check_dependencies
SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
DATAFUSION_JAVA_DIST_URL='https://dist.apache.org/repos/dist/dev/datafusion'
download_dist_file() {
curl \
--silent \
--show-error \
--fail \
--location \
--remote-name $DATAFUSION_JAVA_DIST_URL/$1
}
download_rc_file() {
download_dist_file apache-datafusion-java-${VERSION}-rc${RC_NUMBER}/$1
}
import_gpg_keys() {
download_dist_file KEYS
gpg --import KEYS
}
if type shasum >/dev/null 2>&1; then
sha256_verify="shasum -a 256 -c"
sha512_verify="shasum -a 512 -c"
else
sha256_verify="sha256sum -c"
sha512_verify="sha512sum -c"
fi
fetch_archive() {
local dist_name=$1
download_rc_file ${dist_name}.tar.gz
download_rc_file ${dist_name}.tar.gz.asc
download_rc_file ${dist_name}.tar.gz.sha256
download_rc_file ${dist_name}.tar.gz.sha512
verify_dir_artifact_signatures
}
verify_dir_artifact_signatures() {
# verify the signature and the checksums of each artifact
find . -name '*.asc' | while read sigfile; do
artifact=${sigfile/.asc/}
gpg --verify $sigfile $artifact || exit 1
# go into the directory because the checksum files contain only the
# basename of the artifact
pushd $(dirname $artifact)
base_artifact=$(basename $artifact)
${sha256_verify} $base_artifact.sha256 || exit 1
${sha512_verify} $base_artifact.sha512 || exit 1
popd
done
}
setup_tempdir() {
cleanup() {
if [ "${TEST_SUCCESS}" = "yes" ]; then
rm -fr "${DATAFUSION_JAVA_TMPDIR}"
else
echo "Failed to verify release candidate. See ${DATAFUSION_JAVA_TMPDIR} for details."
fi
}
if [ -z "${DATAFUSION_JAVA_TMPDIR:-}" ]; then
# clean up automatically if DATAFUSION_JAVA_TMPDIR is not defined
DATAFUSION_JAVA_TMPDIR=$(mktemp -d -t "$1.XXXXX")
trap cleanup EXIT
else
# don't clean up automatically
mkdir -p "${DATAFUSION_JAVA_TMPDIR}"
fi
}
test_source_distribution() {
# install rust toolchain in a similar fashion like test-miniconda
export RUSTUP_HOME=$PWD/test-rustup
export CARGO_HOME=$PWD/test-rustup
curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
export PATH=$RUSTUP_HOME/bin:$PATH
source $RUSTUP_HOME/env
# raises on any formatting errors
rustup component add rustfmt
(cd native && cargo fmt --all -- --check)
# build native + JVM and run the full test suite
make test
}
TEST_SUCCESS=no
setup_tempdir "datafusion-java-${VERSION}"
echo "Working in sandbox ${DATAFUSION_JAVA_TMPDIR}"
cd ${DATAFUSION_JAVA_TMPDIR}
dist_name="apache-datafusion-java-${VERSION}"
import_gpg_keys
fetch_archive ${dist_name}
tar xf ${dist_name}.tar.gz
pushd ${dist_name}
test_source_distribution
popd
TEST_SUCCESS=yes
echo 'Release candidate looks good!'
exit 0
apache-datafusion-java-0.1.0/docs/ 0000775 0000000 0000000 00000000000 15205164533 0016737 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/docs/Makefile 0000664 0000000 0000000 00000002126 15205164533 0020400 0 ustar 00root root 0000000 0000000 # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Minimal makefile for Sphinx documentation
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source
BUILDDIR = build
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
apache-datafusion-java-0.1.0/docs/README.md 0000664 0000000 0000000 00000003103 15205164533 0020213 0 ustar 00root root 0000000 0000000
# Apache DataFusion Java Documentation
This directory contains the Sphinx source for the Apache DataFusion Java
documentation site.
## Build
Building the docs requires Python 3.9 or newer and a working `./mvnw` at the
repo root (the build runs `./mvnw javadoc:javadoc` to generate the API
reference, which also needs JDK 17 and one-time network access to fetch
upstream `.proto` files). A virtual environment under `docs/venv/` is the
recommended workflow.
```sh
cd docs
python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt
./build.sh
```
The generated site is written to `docs/build/html/`. Open
`docs/build/html/index.html` in a browser to preview.
Subsequent builds need only:
```sh
cd docs
source venv/bin/activate
./build.sh
```
`./build.sh` runs `sphinx-build` with `-W` so warnings fail the build.
apache-datafusion-java-0.1.0/docs/build.sh 0000775 0000000 0000000 00000002400 15205164533 0020371 0 ustar 00root root 0000000 0000000 #!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -e
cd "$(dirname "$0")"
rm -rf build
rm -rf source/_extra
# Generate Javadoc for the library module. javadoc:javadoc triggers
# generate-sources, which fetches upstream protos via wget.
(cd .. && ./mvnw -q -DskipTests -pl :datafusion-java javadoc:javadoc)
mkdir -p source/_extra
cp -R ../core/target/reports/apidocs source/_extra/api
if [ -d venv ]; then
# shellcheck disable=SC1091
source venv/bin/activate
fi
sphinx-build -b html -W --keep-going source build/html
apache-datafusion-java-0.1.0/docs/make.bat 0000664 0000000 0000000 00000003103 15205164533 0020341 0 ustar 00root root 0000000 0000000 @ECHO OFF
@rem Licensed to the Apache Software Foundation (ASF) under one
@rem or more contributor license agreements. See the NOTICE file
@rem distributed with this work for additional information
@rem regarding copyright ownership. The ASF licenses this file
@rem to you under the Apache License, Version 2.0 (the
@rem "License"); you may not use this file except in compliance
@rem with the License. You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing,
@rem software distributed under the License is distributed on an
@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@rem KIND, either express or implied. See the License for the
@rem specific language governing permissions and limitations
@rem under the License.
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=source
set BUILDDIR=build
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.https://www.sphinx-doc.org/
exit /b 1
)
if "%1" == "" goto help
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
:end
popd
apache-datafusion-java-0.1.0/docs/requirements.txt 0000664 0000000 0000000 00000001535 15205164533 0022227 0 ustar 00root root 0000000 0000000 # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
sphinx>=7.0,<8.0
myst-parser>=2.0,<4.0
pydata-sphinx-theme>=0.16.1,<0.17.0
apache-datafusion-java-0.1.0/docs/source/ 0000775 0000000 0000000 00000000000 15205164533 0020237 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/docs/source/_static/ 0000775 0000000 0000000 00000000000 15205164533 0021665 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/docs/source/_static/.gitkeep 0000664 0000000 0000000 00000000000 15205164533 0023304 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/docs/source/_templates/ 0000775 0000000 0000000 00000000000 15205164533 0022374 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/docs/source/_templates/.gitkeep 0000664 0000000 0000000 00000000000 15205164533 0024013 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/docs/source/conf.py 0000664 0000000 0000000 00000003216 15205164533 0021540 0 ustar 00root root 0000000 0000000 # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Sphinx configuration for the Apache DataFusion Java documentation."""
project = "Apache DataFusion Java"
copyright = "2026, Apache Software Foundation"
author = "Apache Software Foundation"
extensions = [
"sphinx.ext.mathjax",
"sphinx.ext.napoleon",
"myst_parser",
]
source_suffix = {
".md": "markdown",
}
templates_path = ["_templates"]
exclude_patterns = ["_extra"]
html_theme = "pydata_sphinx_theme"
html_theme_options = {
"use_edit_page_button": False,
"show_toc_level": 2,
}
html_context = {
"github_user": "apache",
"github_repo": "datafusion-java",
"github_version": "main",
"doc_path": "docs/source",
}
html_static_path = ["_static"]
html_extra_path = ["_extra"]
# Auto-generate anchor links for headings h1, h2, h3.
myst_heading_anchors = 3
# Enable nice rendering of GitHub-style task lists.
myst_enable_extensions = ["tasklist"]
apache-datafusion-java-0.1.0/docs/source/contributor-guide/ 0000775 0000000 0000000 00000000000 15205164533 0023704 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/docs/source/contributor-guide/code-style.md 0000664 0000000 0000000 00000002354 15205164533 0026302 0 ustar 00root root 0000000 0000000
# Code style
## Java
Run the Spotless formatter before committing. CI fails the build if
formatting drifts:
```sh
./mvnw spotless:apply
```
## Rust
Run inside `native/`:
```sh
cargo fmt
cargo clippy --all-targets -- -D warnings
```
`-D warnings` turns clippy warnings into build failures, matching CI.
## License headers
New source files need the Apache 2.0 license header. Apache RAT enforces
this during `verify` — `./mvnw verify` will fail if a tracked file is
missing the header.
apache-datafusion-java-0.1.0/docs/source/contributor-guide/development.md 0000664 0000000 0000000 00000012522 15205164533 0026552 0 ustar 00root root 0000000 0000000
# Development
## Build prerequisites
- JDK 17 or newer.
- Rust toolchain (stable, installed via [rustup]).
- [`tpchgen-cli`] — only needed to generate test data for the Parquet
integration test (`cargo install tpchgen-cli`).
Maven is bundled via the `./mvnw` wrapper; no separate Maven install is
required.
[rustup]: https://rustup.rs/
[`tpchgen-cli`]: https://github.com/clflushopt/tpchgen-rs
## Build and test
```sh
make test
```
This builds the native Rust crate and runs the JUnit tests. The steps can
be run individually:
```sh
cd native && cargo build
./mvnw test
```
The native library must be built before running JVM tests.
The first build in a fresh checkout reaches out to
`raw.githubusercontent.com` to fetch the DataFusion `.proto` files used
to generate the `datafusion-proto` Java classes. Subsequent builds are
offline; the `download-maven-plugin` cache under
`~/.m2/repository/.cache/` satisfies them.
## Test data
The Parquet integration test reads TPC-H SF1 data (~345 MB across 8 tables
in Snappy-compressed Parquet). Generate it once with:
```sh
make tpch-data
```
Tests that need this data skip cleanly if it is missing. `make clean`
does **not** remove `tpch-data/` — delete it manually to reclaim the
disk space.
## Repository layout
The repository is a multi-module Maven build:
- `pom.xml` — parent POM declaring the `core` and `examples` modules and
shared plugin/dependency versions.
- `core/` — `datafusion-java` library module (Java sources, tests, and
generated protobuf classes).
- `examples/` — `datafusion-java-examples` module containing runnable
examples that depend on the library; built alongside the library so they
cannot fall out of sync with the API.
- `native/` — Rust crate (JNI + Arrow C Data Interface).
- `proto/` — Protobuf definitions shared between Java and Rust.
- `Makefile` — top-level build orchestration (`make test`, `make tpch-data`).
- `mvnw`, `mvnw.cmd` — bundled Maven wrapper.
- `docs/` — Sphinx documentation source and build scripts.
## Running an example
The examples module wires `exec-maven-plugin` with the right
`java.library.path` and `--add-opens` flags. Install the library to the
local Maven repository once, then run any example by main class:
```sh
./mvnw install -DskipTests
./mvnw -pl :datafusion-java-examples exec:exec \
-Dexec.mainClass=org.apache.datafusion.examples.SqlQueryExample
```
The bundled examples (under
`examples/src/main/java/org/apache/datafusion/examples/`):
- `SqlQueryExample` — register a CSV and run a SQL aggregation.
- `DataFrameExample` — read CSV → filter / select / rename / distinct →
write Parquet → read back.
- `ProtoPlanExample` — build a DataFusion `LogicalPlanNode` directly via
the generated protobuf classes and execute it through
`SessionContext.fromProto`.
Re-run `mvnw install -DskipTests` whenever you change the library.
## Passing structured options across the JNI boundary
When a JNI call needs to carry more than a handful of scalar arguments —
for example, a struct of nullable knobs like `CsvReadOptions` or
`SessionOptions` — encode the call's configuration as a protobuf message
rather than expanding the JNI signature with one parameter per field.
Add a `.proto` file under `proto/`, declare `package datafusion_java;`,
and follow the conventions already in use:
- One message per logical bundle of options.
- Use `optional` for fields whose unset-ness must survive the boundary
(so the Rust side can leave a DataFusion default in place).
- Use proto enums for closed sets of choices instead of strings.
Prefix every value with the enum's name (e.g.
`FILE_COMPRESSION_TYPE_GZIP`, not bare `GZIP`) because proto3 enum
values are scoped at the package level, and the zero value must be a
`_UNSPECIFIED` sentinel — the Rust side should reject `UNSPECIFIED`
rather than silently default it.
- Keep large opaque payloads (Arrow IPC schemas, plan nodes) as separate
`byte[]` JNI arguments next to the options proto, not inside it.
- Suffix the message name with `Proto` if a sibling Java class would
otherwise shadow it (e.g. `CsvReadOptionsProto` vs the public
`CsvReadOptions`).
The proto is compiled by both `prost-build` (Rust, via `native/build.rs`)
and the Maven `protobuf-maven-plugin` (Java). The Java side builds the
message, serializes to bytes, and passes the byte array through JNI; the
Rust side decodes once and folds the fields into the corresponding
DataFusion struct.
This pattern keeps JNI signatures short, makes nullable and enum fields
explicit in a single typed schema, and lets new fields be added without
touching the signature on either side.
apache-datafusion-java-0.1.0/docs/source/contributor-guide/index.md 0000664 0000000 0000000 00000003142 15205164533 0025335 0 ustar 00root root 0000000 0000000
# Contributor Guide
Bug reports, design discussion, and patches are welcome. This project follows
the Apache DataFusion contribution model.
## Filing issues and discussing changes
- File bugs and feature requests on
[GitHub issues](https://github.com/apache/datafusion-java/issues).
- For larger or design-level discussion, the mailing list is
[dev@datafusion.apache.org](mailto:dev@datafusion.apache.org).
- Please open an issue before sending a PR for any significant change so
the approach can be agreed on first.
## Development workflow
Branch from `main`, write changes with
[conventional commit](https://www.conventionalcommits.org/) messages in
the imperative mood (e.g. `feat: add foo`, `fix(native): handle bar`),
and open a pull request targeting `main`.
```{toctree}
:maxdepth: 1
development
code-style
releasing
updating-datafusion-version
```
apache-datafusion-java-0.1.0/docs/source/contributor-guide/releasing.md 0000664 0000000 0000000 00000002146 15205164533 0026202 0 ustar 00root root 0000000 0000000
# Releasing
There are no releases of Apache DataFusion Java yet. Once the first
release approaches, this page will document the Apache release process
the project follows.
In the meantime, refer to the
[Apache DataFusion release process](https://datafusion.apache.org/contributor-guide/release/index.html)
for the broader pattern used by sibling subprojects.
apache-datafusion-java-0.1.0/docs/source/contributor-guide/updating-datafusion-version.md 0000664 0000000 0000000 00000005135 15205164533 0031663 0 ustar 00root root 0000000 0000000
# Updating the DataFusion / protobuf schema version
Three things must move together when bumping DataFusion:
1. `native/Cargo.toml` — the `datafusion` crate dependency.
2. `pom.xml` — the `` Maven property. **Must equal
the Cargo version**; a mismatch means JVM-built protobuf plans won't
deserialize on the native side.
3. `pom.xml` — the `` checksums on the two `download-maven-plugin`
executions. These pin the downloaded `.proto` files; the build fails
if upstream silently re-tags them, which is the desired behavior.
## Recipe
```sh
# 1. Bump the Cargo dep
$EDITOR native/Cargo.toml # set datafusion = ""
(cd native && cargo update -p datafusion)
# 2. Bump the Maven property to match
$EDITOR pom.xml # set
# 3. Compute the new SHA-512 hashes for both `.proto` files from the
# upstream tag you just set in step 2, then paste them into the two
# elements in pom.xml.
NEW=$(grep -m1 -oE '[^<]+' pom.xml | cut -d'>' -f2)
curl -sL "https://raw.githubusercontent.com/apache/datafusion/$NEW/datafusion/proto-common/proto/datafusion_common.proto" | shasum -a 512 | awk '{print $1}'
curl -sL "https://raw.githubusercontent.com/apache/datafusion/$NEW/datafusion/proto/proto/datafusion.proto" | shasum -a 512 | awk '{print $1}'
$EDITOR pom.xml # paste the two hashes into the elements
# Drop the local download cache so the next build re-downloads against
# the new hashes.
rm -rf ~/.m2/repository/.cache/download-maven-plugin target/proto
# 4. Verify
make && make test
```
## Why the protobuf runtime version is separate
The protobuf runtime version (`` in `pom.xml`) tracks
the Java ecosystem (security and JDK compatibility), not DataFusion.
Bump it independently when there is a reason.
apache-datafusion-java-0.1.0/docs/source/index.md 0000664 0000000 0000000 00000004664 15205164533 0021702 0 ustar 00root root 0000000 0000000
# Apache DataFusion Java
Java bindings for [Apache DataFusion]. Queries run in native Rust and results
return to the JVM as [Apache Arrow] batches via the Arrow C Data Interface.
[Apache DataFusion]: https://datafusion.apache.org/
[Apache Arrow]: https://arrow.apache.org/
> Early development: no releases yet, API will change. Bug reports and
> contributions welcome.
## Quickstart
```java
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.SessionContext;
try (var allocator = new RootAllocator();
var ctx = new SessionContext()) {
ctx.registerParquet("orders", "/path/to/orders.parquet");
try (DataFrame df = ctx.sql(
"SELECT o_orderpriority, COUNT(*) AS n " +
"FROM orders GROUP BY o_orderpriority");
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
var batch = reader.getVectorSchemaRoot();
// ...
}
}
}
```
See the [User Guide](user-guide/index.md) for installation, the DataFrame and
SQL APIs, and Parquet ingestion. See the [Contributor Guide](contributor-guide/index.md)
for build, test, and release workflows.
```{toctree}
:maxdepth: 1
:caption: Links
:hidden:
GitHub Repository
Issue Tracker
Apache DataFusion
Code of Conduct
```
```{toctree}
:maxdepth: 2
:caption: Documentation
:hidden:
User Guide
Contributor Guide
```
apache-datafusion-java-0.1.0/docs/source/user-guide/ 0000775 0000000 0000000 00000000000 15205164533 0022310 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/docs/source/user-guide/api-reference.md 0000664 0000000 0000000 00000002667 15205164533 0025352 0 ustar 00root root 0000000 0000000
# API Reference
Generated Javadoc for the public Java API lives at
api/index.html.
The reference covers the `org.apache.datafusion` package, including
`SessionContext`, `SessionContextBuilder`, `DataFrame`, `CsvReadOptions`,
`ParquetReadOptions`, and `NativeLibraryLoader`, plus the
`org.apache.datafusion.proto` helpers. Auto-generated protobuf classes are
intentionally excluded from the Javadoc — they are an implementation detail.
apache-datafusion-java-0.1.0/docs/source/user-guide/dataframe.md 0000664 0000000 0000000 00000005261 15205164533 0024562 0 ustar 00root root 0000000 0000000
# DataFrame and SQL
DataFusion Java supports two query interfaces: SQL strings via
`SessionContext.sql(String)`, and a programmatic DataFrame API.
## SQL
```java
try (DataFrame df = ctx.sql("SELECT a, b FROM t WHERE a > 10")) {
df.show();
}
```
`sql(String)` plans the query and returns a `DataFrame`. Execution does
not start until you pull results.
## DataFrame transformations
The DataFrame API exposes `select`, `filter`, `limit`, `distinct`,
`dropColumns`, and `withColumnRenamed`.
```java
try (DataFrame df = ctx.readParquet("/path/to/orders.parquet")) {
try (DataFrame filtered = df.filter("o_orderpriority = '1-URGENT'")) {
filtered.show();
}
}
```
Each transformation returns a new `DataFrame` that must be closed.
## Pulling results
Three patterns are available:
**Stream as Arrow batches.** Use `collect(allocator)` to pull the result
set as Arrow record batches via the [Arrow C Data Interface]:
```java
try (DataFrame df = ctx.sql("SELECT ...");
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
var batch = reader.getVectorSchemaRoot();
// process batch...
}
}
```
[Arrow C Data Interface]: https://arrow.apache.org/docs/format/CDataInterface.html
**Count rows.** `df.count()` returns the row count without materializing
the rows in the JVM.
**Print for inspection.** `df.show()` and `df.show(int n)` print results
to standard output. Useful for exploration; not appropriate for
production code paths.
## Schema introspection
To get the schema of a registered table without running a query:
```java
org.apache.arrow.vector.types.pojo.Schema schema = ctx.tableSchema("orders");
```
## Plan input
A DataFusion logical plan can be deserialized from `datafusion-proto`
bytes via `SessionContext.fromProto(byte[])`. The `datafusion-proto` Java
classes are generated by the Maven build. This is useful for accepting
plans produced by other DataFusion-aware tooling.
apache-datafusion-java-0.1.0/docs/source/user-guide/index.md 0000664 0000000 0000000 00000002576 15205164533 0023753 0 ustar 00root root 0000000 0000000
# User Guide
Apache DataFusion Java is a thin Java binding over the
[Apache DataFusion](https://datafusion.apache.org/) query engine. SQL and
DataFrame queries execute in native Rust; results return to the JVM as
[Apache Arrow](https://arrow.apache.org/) record batches over the Arrow C
Data Interface.
This guide covers installation, the `SessionContext` and `DataFrame` APIs,
and Parquet ingestion.
```{toctree}
:maxdepth: 1
installation
quickstart
sessioncontext
dataframe
parquet
proto-plans
scalar-udf
table-provider
api-reference
```
> Early development: no releases yet, API will change. Bug reports and
> contributions welcome.
apache-datafusion-java-0.1.0/docs/source/user-guide/installation.md 0000664 0000000 0000000 00000003501 15205164533 0025332 0 ustar 00root root 0000000 0000000
# Installation
Apache DataFusion Java has not yet published a release. Until the first
release, the only way to use the library is to build from source.
## Requirements
- **JDK 17 or newer.** Set `JAVA_HOME` to point at it.
- **Rust toolchain (stable).** Install via [rustup].
[rustup]: https://rustup.rs/
## Build from source
```sh
git clone https://github.com/apache/datafusion-java.git
cd datafusion-java
make test
```
`make test` compiles the native Rust crate, then runs the JUnit tests
against it. The native library must be built before the JVM tests can
run.
The first build in a fresh checkout reaches out to
`raw.githubusercontent.com` to fetch the DataFusion `.proto` files used to
generate the `datafusion-proto` Java classes. Subsequent builds are
offline; the `download-maven-plugin` cache under
`~/.m2/repository/.cache/` satisfies them.
For development workflow details — running individual tests, the TPC-H
integration test data, code style, and how to update the underlying
DataFusion version — see the [Contributor Guide](../contributor-guide/development.md).
apache-datafusion-java-0.1.0/docs/source/user-guide/parquet.md 0000664 0000000 0000000 00000003776 15205164533 0024330 0 ustar 00root root 0000000 0000000
# Parquet
DataFusion Java reads Parquet through two entry points on `SessionContext`:
`registerParquet` to expose a file as a named table, and `readParquet` to
get a `DataFrame` directly.
## Register a table
```java
ctx.registerParquet("orders", "/path/to/orders.parquet");
try (DataFrame df = ctx.sql("SELECT * FROM orders LIMIT 10")) {
df.show();
}
```
The file's footer is read at registration time. The table remains in the
catalog for the lifetime of the `SessionContext`.
## Read a DataFrame directly
```java
try (DataFrame df = ctx.readParquet("/path/to/orders.parquet")) {
df.show();
}
```
`readParquet` skips the catalog and hands back a `DataFrame` straight
away.
## ParquetReadOptions
Both entry points accept a `ParquetReadOptions` to tune the underlying
read. Construct one directly and chain setters:
```java
ParquetReadOptions opts = new ParquetReadOptions()
.fileExtension(".parquet");
ctx.registerParquet("orders", "/path/to/orders.parquet", opts);
// or
try (DataFrame df = ctx.readParquet("/path/to/orders.parquet", opts)) {
df.show();
}
```
The supported setters track what DataFusion exposes on its Rust
`ParquetReadOptions` builder. Inspect the class on the Java side for the
exact setters available in the version you are using.
apache-datafusion-java-0.1.0/docs/source/user-guide/proto-plans.md 0000664 0000000 0000000 00000020301 15205164533 0025104 0 ustar 00root root 0000000 0000000
# Logical plans via datafusion-proto
`SessionContext.fromProto(byte[])` accepts a serialized DataFusion
`LogicalPlanNode` and returns a lazy `DataFrame`. This is useful when you
already have a plan produced by another DataFusion-aware tool, or when
you want to construct the plan programmatically with finer-grained
control than the `sql` or DataFrame APIs.
The protobuf Java classes are generated by the build into the
`org.apache.datafusion.protobuf` (plan and expression nodes) and
`datafusion_common` (scalar values, schema, column references, file
formats) packages. The Maven build downloads pinned `.proto` files from
the matching upstream DataFusion tag on first build, then generates the
Java classes locally — see the
[Contributor Guide](../contributor-guide/updating-datafusion-version.md)
for how to bump the version.
## A minimal plan
The smallest interesting plan is a projection of a literal over an
empty input. It is useful as a sanity check and exercises serialization
end-to-end without touching any storage.
```java
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.SessionContext;
import org.apache.datafusion.protobuf.EmptyRelationNode;
import org.apache.datafusion.protobuf.LogicalExprNode;
import org.apache.datafusion.protobuf.LogicalPlanNode;
import org.apache.datafusion.protobuf.ProjectionNode;
import datafusion_common.DatafusionCommon;
LogicalPlanNode plan =
LogicalPlanNode.newBuilder()
.setProjection(
ProjectionNode.newBuilder()
.setInput(
LogicalPlanNode.newBuilder()
.setEmptyRelation(
EmptyRelationNode.newBuilder().setProduceOneRow(true).build())
.build())
.addExpr(
LogicalExprNode.newBuilder()
.setLiteral(
DatafusionCommon.ScalarValue.newBuilder().setInt32Value(1).build())
.build())
.build())
.build();
try (var allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.fromProto(plan.toByteArray());
ArrowReader reader = df.collect(allocator)) {
reader.loadNextBatch();
VectorSchemaRoot batch = reader.getVectorSchemaRoot();
IntVector col = (IntVector) batch.getVector(0);
System.out.println(col.get(0)); // 1
}
```
`fromProto` performs the same logical-planning, optimization, and
physical-planning pipeline as `sql`; the result is a lazy
[`DataFrame`](dataframe.md) that only executes when you pull results.
## Scanning a Parquet file via ListingTableScanNode
A `ListingTableScanNode` reads one or more files of the same format
from disk. Unlike `registerParquet`, it does not require the table to
be in the catalog — the scan node carries everything DataFusion needs:
the file paths, the schema, the projection, the file format, and the
target partition count.
The scan node's `schema` field is a `datafusion_common.Schema`, not an
Arrow `Schema`. Convert between the two with the helper in
`org.apache.datafusion.proto.SchemaConverter`:
```java
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.proto.SchemaConverter;
Schema arrow = ctx.tableSchema("lineitem");
DatafusionCommon.Schema schemaProto = SchemaConverter.toProto(arrow);
```
The full example: register the file once to introspect its schema,
then build a plan that scans the same file, sorts by `l_orderkey`,
and fetches the first row. This is equivalent to
`SELECT l_orderkey FROM lineitem ORDER BY l_orderkey LIMIT 1`.
```java
import org.apache.datafusion.protobuf.BareTableReference;
import org.apache.datafusion.protobuf.ListingTableScanNode;
import org.apache.datafusion.protobuf.ProjectionColumns;
import org.apache.datafusion.protobuf.SortExprNode;
import org.apache.datafusion.protobuf.SortNode;
import org.apache.datafusion.protobuf.TableReference;
String path = "/path/to/lineitem.parquet";
try (var allocator = new RootAllocator();
SessionContext ctx = new SessionContext()) {
ctx.registerParquet("lineitem", path);
DatafusionCommon.Schema schemaProto =
SchemaConverter.toProto(ctx.tableSchema("lineitem"));
LogicalExprNode orderKeyCol =
LogicalExprNode.newBuilder()
.setColumn(DatafusionCommon.Column.newBuilder().setName("l_orderkey").build())
.build();
LogicalPlanNode plan =
LogicalPlanNode.newBuilder()
.setSort(
SortNode.newBuilder()
.setInput(
LogicalPlanNode.newBuilder()
.setListingScan(
ListingTableScanNode.newBuilder()
.setTableName(
TableReference.newBuilder()
.setBare(
BareTableReference.newBuilder()
.setTable("lineitem")
.build())
.build())
.addPaths(path)
.setFileExtension(".parquet")
.setSchema(schemaProto)
.setProjection(
ProjectionColumns.newBuilder()
.addColumns("l_orderkey")
.build())
.setParquet(
DatafusionCommon.ParquetFormat.getDefaultInstance())
.setTargetPartitions(1)
.build())
.build())
.addExpr(
SortExprNode.newBuilder()
.setExpr(orderKeyCol)
.setAsc(true)
.setNullsFirst(false)
.build())
.setFetch(1)
.build())
.build();
try (DataFrame df = ctx.fromProto(plan.toByteArray());
ArrowReader reader = df.collect(allocator)) {
reader.loadNextBatch();
// ...
}
}
```
## When to use proto plans
The `sql` and DataFrame APIs are the right choice for most workloads.
Reach for `fromProto` when you need one of:
- **Cross-tool interop.** Accept plans produced by another
DataFusion-based system (a planner, a scheduler, a query frontend).
- **Programmatic plan construction.** Build the plan node tree directly
instead of going through SQL parsing, useful for tools that compile
their own surface language to DataFusion.
- **Plan persistence.** Serialize a plan to bytes, store or transmit
it, and execute it later — possibly in a different process or on a
different machine.
## Schema conversion support
`SchemaConverter.toProto` and `SchemaConverter.fromProto` support the
primitive Arrow types this project's tests exercise: `Bool`, signed and
unsigned integer types 8 through 64 bits, `Float32`, `Float64`, `Utf8`,
`Utf8View`, `LargeUtf8`, `Date32`, and `Decimal128`. Anything else
raises `UnsupportedOperationException` naming the offending type.
apache-datafusion-java-0.1.0/docs/source/user-guide/quickstart.md 0000664 0000000 0000000 00000005242 15205164533 0025027 0 ustar 00root root 0000000 0000000
# Quickstart
This page walks through a complete query end-to-end.
## The full example
```java
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.SessionContext;
try (var allocator = new RootAllocator();
var ctx = new SessionContext()) {
ctx.registerParquet("orders", "/path/to/orders.parquet");
try (DataFrame df = ctx.sql(
"SELECT o_orderpriority, COUNT(*) AS n " +
"FROM orders GROUP BY o_orderpriority");
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
var batch = reader.getVectorSchemaRoot();
// ...
}
}
}
```
## Walkthrough
**Allocator.** `RootAllocator` is the Arrow off-heap memory allocator. Every
JVM-side Arrow buffer is tracked under an allocator; when the allocator is
closed, leaked buffers are reported. Use one allocator per query (or one
per application) and close it in a `try`-with-resources.
**Session context.** `SessionContext` is the entry point into DataFusion. It
holds the catalog of registered tables and the query planner. It is
`AutoCloseable` and **not thread-safe** — use one per thread, or guard
access externally.
**Registering data.** `registerParquet(name, path)` reads the file's footer
on call and exposes it under the given table name. See
[Parquet](parquet.md) for the options form.
**SQL.** `ctx.sql("...")` plans the query and returns a `DataFrame`. The
query is not executed until results are pulled.
**Collecting results.** `df.collect(allocator)` starts native execution and
returns an `ArrowReader`. Each `loadNextBatch()` call pulls the next
`VectorSchemaRoot`; iterate until it returns `false`.
**Cleanup.** Both `SessionContext` and `DataFrame` are `AutoCloseable`. Use
`try`-with-resources so native resources and Arrow buffers are released
even on exception.
apache-datafusion-java-0.1.0/docs/source/user-guide/scalar-udf.md 0000664 0000000 0000000 00000016236 15205164533 0024663 0 ustar 00root root 0000000 0000000
# Scalar UDFs
A scalar UDF is a Java-implemented SQL function that operates one row at a time,
expressed in vectorised form: each invocation receives a batch of input columns
and returns either a per-row output column of the same length (`Array`) or a
single value broadcast to every row (`Scalar`).
## Implement
Implement the `ScalarFunction` interface. The implementation declares its own
SQL name, argument fields, return field, and volatility, and supplies the
per-batch `evaluate` body:
```java
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.datafusion.ColumnarValue;
import org.apache.datafusion.ScalarFunction;
import org.apache.datafusion.ScalarFunctionArgs;
import org.apache.datafusion.Volatility;
public final class AddOne implements ScalarFunction {
private static final ArrowType INT32 = new ArrowType.Int(32, true);
@Override public String name() { return "add_one"; }
@Override public List argFields() { return List.of(Field.nullable("x", INT32)); }
@Override public Field returnField() { return Field.nullable("y", INT32); }
@Override public Volatility volatility() { return Volatility.IMMUTABLE; }
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
IntVector in = (IntVector) args.args().get(0).vector();
IntVector out = new IntVector("add_one", allocator);
out.allocateNew(in.getValueCount());
for (int i = 0; i < in.getValueCount(); i++) {
if (in.isNull(i)) out.setNull(i);
else out.set(i, in.get(i) + 1);
}
out.setValueCount(in.getValueCount());
return ColumnarValue.array(out);
}
}
```
Each entry in `args.args()` is a `ColumnarValue` — either `ColumnarValue.Array`
(a per-row vector of length `args.rowCount()`) or `ColumnarValue.Scalar` (a
length-1 vector representing a single literal or folded constant). Access the
underlying Arrow vector with `.vector()`.
Allocate any new vectors — including the result — from the supplied
`BufferAllocator`. The input vectors are read-only views; do not close them.
Ownership of the returned vector transfers to the framework on return.
## Declaring argument and return fields
Each argument and the return value are described as Arrow `Field`s. A `Field`
carries a name, a `FieldType` (Arrow type plus nullability and metadata), and a
list of child fields used by nested types.
For primitive types the `Field.nullable(name, arrowType)` and
`Field.notNullable(name, arrowType)` factories are the shortest form:
```java
Field x = Field.nullable("x", new ArrowType.Int(32, true));
Field y = Field.notNullable("y", new ArrowType.Int(32, true));
```
Nested Arrow types — `List`, `Struct`, `Map`, `Union` — must declare their
element / member / key / value fields as children, because that information
does not live on the `ArrowType` itself. Use the
`new Field(name, FieldType, children)` constructor:
```java
// List
ArrowType INT32 = new ArrowType.Int(32, true);
Field listOfInt =
new Field(
"vals",
FieldType.nullable(new ArrowType.List()),
List.of(Field.nullable("item", INT32)));
// Struct
Field structAB =
new Field(
"ab",
FieldType.nullable(new ArrowType.Struct()),
List.of(Field.nullable("a", INT32), Field.nullable("b", INT32)));
```
A UDF declared with `Field.nullable(...)` arguments is registered with an exact
nullability-bearing signature; calls whose argument types do not match exactly
are rejected. The declared `returnField`'s nullability is preserved end-to-end:
a non-nullable return field stays non-nullable in the result schema.
## Returning a Scalar
Functions that yield a single value (nullary constants like `pi()`, or any
function that wants the framework to broadcast a result across the batch) can
return `ColumnarValue.scalar(...)` over a length-1 vector:
```java
public final class JavaPi implements ScalarFunction {
private static final ArrowType FLOAT64 =
new ArrowType.FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE);
@Override public String name() { return "java_pi"; }
@Override public List argFields() { return List.of(); }
@Override public Field returnField() { return Field.nullable("p", FLOAT64); }
@Override public Volatility volatility() { return Volatility.VOLATILE; }
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
org.apache.arrow.vector.Float8Vector out =
new org.apache.arrow.vector.Float8Vector("pi", allocator);
out.allocateNew(1);
out.set(0, Math.PI);
out.setValueCount(1);
return ColumnarValue.scalar(out);
}
}
```
The framework expands the scalar across `args.rowCount()` rows automatically.
## Register
Wrap the implementation in a `ScalarUdf` and pass it to
`SessionContext.registerUdf`:
```java
try (SessionContext ctx = new SessionContext()) {
ctx.registerUdf(new ScalarUdf(new AddOne()));
try (DataFrame df = ctx.sql("SELECT add_one(x) FROM t");
ArrowReader r = df.collect(allocator)) {
// ...
}
}
```
`ScalarUdf` mirrors DataFusion's `ScalarUDF` struct; `ScalarFunction` mirrors
`ScalarUDFImpl`. Use `Volatility.IMMUTABLE` for pure functions, `STABLE` for
functions that are deterministic within a single query, and `VOLATILE` for
non-deterministic functions.
## Errors
If the UDF throws, the exception class and message surface in the
`RuntimeException` raised from `collect()`. If the returned `ColumnarValue` is
`null`, an Array result's vector length does not equal `args.rowCount()`, or
the result's Arrow type differs from the declared return field, the runtime
raises a `RuntimeException` with a descriptive message. A Scalar result whose
vector is not length-1 is rejected at the `ColumnarValue.scalar` factory.
## Threading
DataFusion may invoke a UDF concurrently from multiple worker threads. If the
implementation carries mutable state, the implementation must synchronize it.
## Limitations (v1)
- Scalar UDFs only — no aggregates, window functions, or table functions.
- Exact-signature only — no variadic or polymorphic argument lists.
- No nullable-argument short-circuiting; null inputs are passed through to the
UDF as nulls in the input vector.
apache-datafusion-java-0.1.0/docs/source/user-guide/sessioncontext.md 0000664 0000000 0000000 00000004053 15205164533 0025724 0 ustar 00root root 0000000 0000000
# SessionContext
`SessionContext` is the entry point into DataFusion from Java. It owns the
catalog of registered tables and the query planner.
## Lifecycle
```java
try (SessionContext ctx = new SessionContext()) {
// register tables, build queries...
}
```
`SessionContext` is `AutoCloseable`. Closing it releases the underlying
native context. Use `try`-with-resources so the native side is freed even
on exception.
## Threading
A `SessionContext` is **not thread-safe**. Do not share one across threads
without external synchronization. The simplest pattern is one context per
thread.
## Configuration
`SessionContext.builder()` exposes a fluent builder for overriding
DataFusion defaults — batch size, target partitions, statistics
collection, information schema, memory pool size, and the spill
directory. See the
SessionContextBuilder
Javadoc for the full list.
```java
try (SessionContext ctx = SessionContext.builder()
.batchSize(4096)
.targetPartitions(8)
.build()) {
// ...
}
```
apache-datafusion-java-0.1.0/docs/source/user-guide/table-provider.md 0000664 0000000 0000000 00000010576 15205164533 0025562 0 ustar 00root root 0000000 0000000
# Java table providers
`SessionContext.registerTable(name, provider)` registers a Java-implemented
table. SQL queries that reference `name` call back into your `TableProvider`
to fetch batches. Data flows from Java to native code via the Arrow C Data
Interface, so there are no extra copies in the hot path. This is the Java
counterpart to DataFusion's Rust `SessionContext::register_table`.
## Implement
```java
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.TableProvider;
public final class MyTable implements TableProvider {
private final Schema schema;
public MyTable(Schema schema) {
this.schema = schema;
}
@Override public Schema schema() { return schema; }
@Override
public ArrowReader scan(BufferAllocator allocator) {
// Return a fresh ArrowReader. The reader must allocate its buffers
// from `allocator` (or a child of it) — the framework needs the
// allocator hierarchy to share a root.
return openMyReader(allocator);
}
}
```
For the common case of "I have a schema and a function that returns an
`ArrowReader`," `SimpleTableProvider` packages those two into a ready-made
`TableProvider` without having to subclass:
```java
TableProvider t = new SimpleTableProvider(mySchema(), allocator -> openMyReader(allocator));
ctx.registerTable("t", t);
```
## Register and query
```java
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerTable("t", new MyTable(mySchema()));
try (DataFrame df = ctx.sql("SELECT * FROM t WHERE x > 10");
ArrowReader r = df.collect(allocator)) {
while (r.loadNextBatch()) {
// ...
}
}
}
```
## Contract
- `schema()` is called exactly once, on the caller's thread, at registration
time. Throwing from it aborts registration with the original exception.
- `scan(allocator)` is called once per SQL query that touches the table, on a
worker thread. It must return a fresh, independent `ArrowReader` on every
call — this is what makes self-joins and `UNION ALL` over the same table
work.
- The reader returned by `scan` must allocate its buffers from the supplied
`allocator` (or a child of it). Arrow Java's `Data.exportArrayStream`
requires the reader's allocator and the export allocator to share a root.
- The returned reader's schema must equal the schema returned by `schema()`.
A mismatch fails the query.
- You do not need to close the returned reader yourself. The framework
installs a release callback that closes it when the underlying FFI stream
is dropped.
## Errors
Exceptions thrown from `scan()` or from the returned reader surface in the
`RuntimeException` raised by `collect()`. The error message includes the Java
exception class and `getMessage()`, in the same format used for scalar UDF
errors.
## Threading
`SessionContext` is single-threaded, but `scan(allocator)` may be invoked from
any DataFusion worker thread. If your implementation maintains mutable state
across scans, synchronise it.
## Limitations (v1)
- Single-partition scans only. DataFusion sees the table as one partition;
multi-partition parallelism is a follow-up.
- No projection or filter pushdown. DataFusion applies projection and
filters on top of the batches you return; the Java side always sees the
full schema. The interface is intentionally minimal so it can grow these
capabilities (as default methods) without breaking existing implementations.
- No `deregisterTable`. Tables live until the `SessionContext` is closed.
apache-datafusion-java-0.1.0/examples/ 0000775 0000000 0000000 00000000000 15205164533 0017625 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/pom.xml 0000664 0000000 0000000 00000006055 15205164533 0021150 0 ustar 00root root 0000000 0000000
4.0.0org.apache.datafusiondatafusion-java-parent0.1.0datafusion-java-examplesjarApache DataFusion Java Examplestruetrueorg.apache.datafusiondatafusion-javaorg.apache.arrowarrow-vectororg.apache.arrowarrow-memory-nettyruntimeorg.apache.arrowarrow-jdbc${arrow.version}com.h2databaseh22.3.232org.codehaus.mojoexec-maven-plugin3.5.0java--add-opens=java.base/java.nio=ALL-UNNAMED-classpath${exec.mainClass}
apache-datafusion-java-0.1.0/examples/src/ 0000775 0000000 0000000 00000000000 15205164533 0020414 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/ 0000775 0000000 0000000 00000000000 15205164533 0021340 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/ 0000775 0000000 0000000 00000000000 15205164533 0022261 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/org/ 0000775 0000000 0000000 00000000000 15205164533 0023050 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/org/apache/ 0000775 0000000 0000000 00000000000 15205164533 0024271 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/org/apache/datafusion/ 0000775 0000000 0000000 00000000000 15205164533 0026426 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/org/apache/datafusion/examples/ 0000775 0000000 0000000 00000000000 15205164533 0030244 5 ustar 00root root 0000000 0000000 AddOneExample.java 0000664 0000000 0000000 00000006644 15205164533 0033510 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/org/apache/datafusion/examples /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.examples;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.datafusion.ColumnarValue;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.ScalarFunction;
import org.apache.datafusion.ScalarFunctionArgs;
import org.apache.datafusion.ScalarUdf;
import org.apache.datafusion.SessionContext;
import org.apache.datafusion.Volatility;
/** Demonstrates registering a Java scalar UDF and invoking it from SQL. */
public final class AddOneExample {
/** Adds 1 to each value of an Int32 column. */
public static final class AddOne implements ScalarFunction {
private static final ArrowType INT32 = new ArrowType.Int(32, true);
@Override
public String name() {
return "add_one";
}
@Override
public List argFields() {
return List.of(Field.nullable("x", INT32));
}
@Override
public Field returnField() {
return Field.nullable("y", INT32);
}
@Override
public Volatility volatility() {
return Volatility.IMMUTABLE;
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
IntVector in = (IntVector) args.args().get(0).vector();
IntVector out = new IntVector("add_one_out", allocator);
int n = in.getValueCount();
out.allocateNew(n);
for (int i = 0; i < n; i++) {
if (in.isNull(i)) {
out.setNull(i);
} else {
out.set(i, in.get(i) + 1);
}
}
out.setValueCount(n);
return ColumnarValue.array(out);
}
}
public static void main(String[] args) throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new AddOne()));
try (DataFrame df =
ctx.sql(
"SELECT add_one(x) AS y FROM (VALUES (CAST(1 AS INT)),(CAST(2 AS INT)),(CAST(3 AS"
+ " INT))) AS t(x)");
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
VectorSchemaRoot root = reader.getVectorSchemaRoot();
IntVector y = (IntVector) root.getVector("y");
for (int i = 0; i < y.getValueCount(); i++) {
System.out.println("y = " + y.get(i));
}
}
}
}
}
}
DataFrameExample.java 0000664 0000000 0000000 00000005650 15205164533 0034176 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/org/apache/datafusion/examples /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.examples;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Comparator;
import java.util.stream.Stream;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.ParquetWriteOptions;
import org.apache.datafusion.SessionContext;
/**
* Demonstrates the DataFrame transformation API plus a Parquet round-trip: read CSV → filter /
* select / rename / distinct → write Parquet → read back.
*/
public final class DataFrameExample {
private DataFrameExample() {}
public static void main(String[] args) throws Exception {
Path workDir = Files.createTempDirectory("datafusion-example");
Path csv = workDir.resolve("orders.csv");
Path parquet = workDir.resolve("orders.parquet");
Files.writeString(
csv,
"""
priority,amount,region
HIGH,100,US
LOW,25,EU
HIGH,40,US
MEDIUM,60,US
HIGH,75,EU
LOW,25,EU
""");
try (SessionContext ctx = new SessionContext()) {
try (DataFrame source = ctx.readCsv(csv.toAbsolutePath().toString());
DataFrame highPriority = source.filter("priority = 'HIGH'");
DataFrame projected = highPriority.select("region", "amount");
DataFrame renamed = projected.withColumnRenamed("amount", "value");
DataFrame deduped = renamed.distinct()) {
System.out.println("HIGH-priority orders, distinct (region, value):");
deduped.show();
deduped.writeParquet(
parquet.toAbsolutePath().toString(),
new ParquetWriteOptions().singleFileOutput(true).compression("snappy"));
}
try (DataFrame readBack = ctx.readParquet(parquet.toAbsolutePath().toString())) {
System.out.println("Round-tripped row count: " + readBack.count());
}
} finally {
try (Stream tree = Files.walk(workDir)) {
tree.sorted(Comparator.reverseOrder()).forEach(DataFrameExample::deleteSilently);
}
}
}
private static void deleteSilently(Path p) {
try {
Files.deleteIfExists(p);
} catch (Exception ignored) {
// best-effort temp cleanup
}
}
}
apache-datafusion-java-0.1.0/examples/src/main/java/org/apache/datafusion/examples/JdbcExample.java 0000664 0000000 0000000 00000021775 15205164533 0033301 0 ustar 00root root 0000000 0000000 /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.examples;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Calendar;
import java.util.TimeZone;
import org.apache.arrow.adapter.jdbc.ArrowVectorIterator;
import org.apache.arrow.adapter.jdbc.JdbcToArrow;
import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig;
import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder;
import org.apache.arrow.adapter.jdbc.JdbcToArrowUtils;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.VarCharVector;
import org.apache.arrow.vector.VectorLoader;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.VectorUnloader;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.SessionContext;
import org.apache.datafusion.TableProvider;
/**
* Demonstrates a JDBC-backed {@link TableProvider}. Populates an H2 in-memory table, registers it
* with DataFusion via {@link SessionContext#registerTable}, and runs an aggregation query against
* it.
*
*
*/
public final class JdbcExample {
/**
* Read-only DataFusion table backed by a JDBC query. Schema is captured at construction time from
* {@link PreparedStatement#getMetaData()}; each {@link #scan} re-executes the query and streams
* the result through {@code arrow-jdbc}'s {@link ArrowVectorIterator}.
*/
public static final class JdbcTableProvider implements TableProvider {
private final String url;
private final String query;
private final Schema schema;
public JdbcTableProvider(String url, String query) {
this.url = url;
this.query = query;
this.schema = fetchSchema();
}
private Schema fetchSchema() {
try (BufferAllocator tmp = new RootAllocator();
Connection conn = DriverManager.getConnection(url);
PreparedStatement stmt = conn.prepareStatement(query)) {
JdbcToArrowConfig config = configFor(tmp);
return JdbcToArrowUtils.jdbcToArrowSchema(stmt.getMetaData(), config);
} catch (SQLException e) {
throw new RuntimeException("Failed to fetch JDBC schema for query: " + query, e);
}
}
@Override
public Schema schema() {
return schema;
}
@Override
public ArrowReader scan(BufferAllocator allocator) {
// Run the query and stream the result through arrow-jdbc's ArrowVectorIterator, which
// emits VectorSchemaRoots whose buffers are allocated from `allocator`. We wrap that
// iterator in a small ArrowReader subclass so that DataFusion can consume it. The
// JDBC Connection / ResultSet are kept open by the closure created in iteratorOf()
// and closed by JdbcArrowReader.closeReadSource().
JdbcToArrowConfig config = configFor(allocator);
OpenedQuery opened = openQuery(config);
try {
ArrowVectorIterator iter = JdbcToArrow.sqlToArrowVectorIterator(opened.rs, config);
return new JdbcArrowReader(allocator, schema, iter, opened);
} catch (SQLException | IOException e) {
opened.closeQuietly();
throw new RuntimeException("Failed to create JDBC iterator for query: " + query, e);
}
}
private OpenedQuery openQuery(JdbcToArrowConfig config) {
try {
Connection conn = DriverManager.getConnection(url);
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery(query);
return new OpenedQuery(conn, stmt, rs);
} catch (SQLException e) {
throw new RuntimeException("Failed to execute JDBC query: " + query, e);
}
}
private static JdbcToArrowConfig configFor(BufferAllocator allocator) {
return new JdbcToArrowConfigBuilder(
allocator, Calendar.getInstance(TimeZone.getTimeZone("UTC")))
.build();
}
}
/** Bundle of JDBC handles to close together when the scan finishes. */
private static final class OpenedQuery {
final Connection conn;
final Statement stmt;
final ResultSet rs;
OpenedQuery(Connection conn, Statement stmt, ResultSet rs) {
this.conn = conn;
this.stmt = stmt;
this.rs = rs;
}
void closeQuietly() {
try {
rs.close();
} catch (SQLException ignored) {
// best-effort
}
try {
stmt.close();
} catch (SQLException ignored) {
// best-effort
}
try {
conn.close();
} catch (SQLException ignored) {
// best-effort
}
}
}
/**
* {@link ArrowReader} backed by an {@link ArrowVectorIterator}. Each {@link #loadNextBatch} pulls
* the next {@link VectorSchemaRoot} from the iterator and transfers its data into the reader's
* managed root via {@link VectorUnloader}/{@link VectorLoader}.
*/
private static final class JdbcArrowReader extends ArrowReader {
private final Schema schema;
private final ArrowVectorIterator iter;
private final OpenedQuery opened;
JdbcArrowReader(
BufferAllocator allocator, Schema schema, ArrowVectorIterator iter, OpenedQuery opened) {
super(allocator);
this.schema = schema;
this.iter = iter;
this.opened = opened;
}
@Override
protected Schema readSchema() {
return schema;
}
@Override
public boolean loadNextBatch() throws IOException {
if (!iter.hasNext()) {
return false;
}
try (VectorSchemaRoot batch = iter.next()) {
VectorUnloader unloader = new VectorUnloader(batch);
try (ArrowRecordBatch rb = unloader.getRecordBatch()) {
new VectorLoader(getVectorSchemaRoot()).load(rb);
}
}
return true;
}
@Override
public long bytesRead() {
return 0;
}
@Override
protected void closeReadSource() {
iter.close();
opened.closeQuietly();
}
}
public static void main(String[] args) throws Exception {
String url = "jdbc:h2:mem:demo;DB_CLOSE_DELAY=-1";
// Populate an H2 in-memory table. Column names are double-quoted so H2 stores them in the
// exact (lowercase) case; arrow-jdbc uses ResultSetMetaData.getColumnName, which returns
// the stored case, so the resulting Arrow schema ends up with lowercase field names that
// DataFusion SQL can refer to without quoting.
try (Connection conn = DriverManager.getConnection(url);
Statement stmt = conn.createStatement()) {
stmt.execute(
"CREATE TABLE \"orders\" ("
+ "\"id\" INT PRIMARY KEY,"
+ " \"customer\" VARCHAR(64) NOT NULL,"
+ " \"total\" DOUBLE NOT NULL)");
stmt.execute(
"INSERT INTO \"orders\" VALUES"
+ " (1, 'alice', 19.99),"
+ " (2, 'bob', 7.50),"
+ " (3, 'alice', 100.00)");
}
JdbcTableProvider src =
new JdbcTableProvider(url, "SELECT \"id\", \"customer\", \"total\" FROM \"orders\"");
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerTable("orders", src);
try (DataFrame df =
ctx.sql(
"SELECT customer, SUM(total) AS spend"
+ " FROM orders GROUP BY customer ORDER BY customer");
ArrowReader reader = df.collect(allocator)) {
System.out.printf("%-10s | %s%n", "customer", "spend");
System.out.println("-----------+--------");
while (reader.loadNextBatch()) {
VectorSchemaRoot root = reader.getVectorSchemaRoot();
VarCharVector customer = (VarCharVector) root.getVector("customer");
Float8Vector spend = (Float8Vector) root.getVector("spend");
for (int i = 0; i < customer.getValueCount(); i++) {
System.out.printf("%-10s | %.2f%n", new String(customer.get(i)), spend.get(i));
}
}
}
}
}
}
NestedTypeUdfExample.java 0000664 0000000 0000000 00000011316 15205164533 0035071 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/org/apache/datafusion/examples /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.examples;
import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.datafusion.ColumnarValue;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.ScalarFunction;
import org.apache.datafusion.ScalarFunctionArgs;
import org.apache.datafusion.ScalarUdf;
import org.apache.datafusion.SessionContext;
import org.apache.datafusion.Volatility;
/**
* Demonstrates a scalar UDF over a nested Arrow type.
*
*
The function takes a {@code List} and returns the sum of its elements as Int64. The
* argument is declared with a {@link Field} carrying a child element field, which is how nested
* Arrow types — List, Struct, Map, Union — express their inner type information.
*/
public final class NestedTypeUdfExample {
/** Sums the Int32 elements of a List column. Null lists yield null; null elements skip. */
public static final class ListSum implements ScalarFunction {
private static final ArrowType INT32 = new ArrowType.Int(32, true);
private static final ArrowType INT64 = new ArrowType.Int(64, true);
@Override
public String name() {
return "java_list_sum";
}
@Override
public List argFields() {
// List needs the element type as a child Field; ArrowType.List alone does not carry
// it.
return List.of(
new Field(
"vals",
FieldType.nullable(new ArrowType.List()),
List.of(Field.nullable("item", INT32))));
}
@Override
public Field returnField() {
return Field.nullable("s", INT64);
}
@Override
public Volatility volatility() {
return Volatility.IMMUTABLE;
}
@Override
public ColumnarValue evaluate(BufferAllocator allocator, ScalarFunctionArgs args) {
ListVector in = (ListVector) args.args().get(0).vector();
IntVector elems = (IntVector) in.getDataVector();
BigIntVector out = new BigIntVector("list_sum_out", allocator);
int n = in.getValueCount();
out.allocateNew(n);
for (int i = 0; i < n; i++) {
if (in.isNull(i)) {
out.setNull(i);
continue;
}
long sum = 0L;
int start = in.getElementStartIndex(i);
int end = in.getElementEndIndex(i);
for (int j = start; j < end; j++) {
if (!elems.isNull(j)) {
sum += elems.get(j);
}
}
out.set(i, sum);
}
out.setValueCount(n);
return ColumnarValue.array(out);
}
}
public static void main(String[] args) throws Exception {
try (SessionContext ctx = new SessionContext();
BufferAllocator allocator = new RootAllocator()) {
ctx.registerUdf(new ScalarUdf(new ListSum()));
try (DataFrame df =
ctx.sql(
"SELECT java_list_sum(vals) AS s"
+ " FROM (VALUES"
+ " (make_array(CAST(1 AS INT), CAST(2 AS INT), CAST(3 AS INT))),"
+ " (make_array(CAST(10 AS INT), CAST(20 AS INT))),"
+ " (make_array(CAST(7 AS INT)))"
+ " ) AS t(vals)");
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
VectorSchemaRoot root = reader.getVectorSchemaRoot();
BigIntVector s = (BigIntVector) root.getVector("s");
for (int i = 0; i < s.getValueCount(); i++) {
System.out.println("sum = " + s.get(i));
}
}
}
}
}
}
ProtoPlanExample.java 0000664 0000000 0000000 00000005600 15205164533 0034263 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/org/apache/datafusion/examples /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.examples;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.SessionContext;
import org.apache.datafusion.protobuf.EmptyRelationNode;
import org.apache.datafusion.protobuf.LogicalExprNode;
import org.apache.datafusion.protobuf.LogicalPlanNode;
import org.apache.datafusion.protobuf.ProjectionNode;
import datafusion_common.DatafusionCommon;
/**
* Builds a DataFusion {@code LogicalPlanNode} directly via the generated protobuf classes and
* executes it through {@link SessionContext#fromProto(byte[])}. Useful as a starting point for
* tools that produce DataFusion plans externally.
*/
public final class ProtoPlanExample {
private ProtoPlanExample() {}
public static void main(String[] args) throws Exception {
// Project two literals over an empty-but-one-row relation: SELECT 42, 7
LogicalPlanNode plan =
LogicalPlanNode.newBuilder()
.setProjection(
ProjectionNode.newBuilder()
.setInput(
LogicalPlanNode.newBuilder()
.setEmptyRelation(
EmptyRelationNode.newBuilder().setProduceOneRow(true).build())
.build())
.addExpr(literalInt(42))
.addExpr(literalInt(7))
.build())
.build();
try (var allocator = new RootAllocator();
SessionContext ctx = new SessionContext();
DataFrame df = ctx.fromProto(plan.toByteArray());
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
VectorSchemaRoot batch = reader.getVectorSchemaRoot();
System.out.println(batch.contentToTSVString());
}
}
}
private static LogicalExprNode literalInt(int value) {
return LogicalExprNode.newBuilder()
.setLiteral(DatafusionCommon.ScalarValue.newBuilder().setInt32Value(value).build())
.build();
}
}
SqlQueryExample.java 0000664 0000000 0000000 00000004243 15205164533 0034134 0 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/examples/src/main/java/org/apache/datafusion/examples /*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.datafusion.examples;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.ipc.ArrowReader;
import org.apache.datafusion.DataFrame;
import org.apache.datafusion.SessionContext;
/** Registers a small CSV file and runs a SQL query against it. */
public final class SqlQueryExample {
private SqlQueryExample() {}
public static void main(String[] args) throws Exception {
Path csv = Files.createTempFile("orders", ".csv");
Files.writeString(
csv,
"""
priority,amount
HIGH,100
LOW,25
HIGH,40
MEDIUM,60
HIGH,75
""");
try (var allocator = new RootAllocator();
var ctx = new SessionContext()) {
ctx.registerCsv("orders", csv.toAbsolutePath().toString());
try (DataFrame df =
ctx.sql(
"SELECT priority, COUNT(*) AS n, SUM(amount) AS total "
+ "FROM orders GROUP BY priority ORDER BY total DESC");
ArrowReader reader = df.collect(allocator)) {
while (reader.loadNextBatch()) {
VectorSchemaRoot batch = reader.getVectorSchemaRoot();
System.out.println(batch.contentToTSVString());
}
}
} finally {
Files.deleteIfExists(csv);
}
}
}
apache-datafusion-java-0.1.0/mvnw 0000775 0000000 0000000 00000027016 15205164533 0016732 0 ustar 00root root 0000000 0000000 #!/bin/sh
# ----------------------------------------------------------------------------
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# ----------------------------------------------------------------------------
# Apache Maven Wrapper startup batch script, version 3.3.4
#
# Optional ENV vars
# -----------------
# JAVA_HOME - location of a JDK home dir, required when download maven via java source
# MVNW_REPOURL - repo url base for downloading maven distribution
# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
# ----------------------------------------------------------------------------
set -euf
[ "${MVNW_VERBOSE-}" != debug ] || set -x
# OS specific support.
native_path() { printf %s\\n "$1"; }
case "$(uname)" in
CYGWIN* | MINGW*)
[ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
native_path() { cygpath --path --windows "$1"; }
;;
esac
# set JAVACMD and JAVACCMD
set_java_home() {
# For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
if [ -n "${JAVA_HOME-}" ]; then
if [ -x "$JAVA_HOME/jre/sh/java" ]; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
JAVACCMD="$JAVA_HOME/jre/sh/javac"
else
JAVACMD="$JAVA_HOME/bin/java"
JAVACCMD="$JAVA_HOME/bin/javac"
if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
return 1
fi
fi
else
JAVACMD="$(
'set' +e
'unset' -f command 2>/dev/null
'command' -v java
)" || :
JAVACCMD="$(
'set' +e
'unset' -f command 2>/dev/null
'command' -v javac
)" || :
if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
return 1
fi
fi
}
# hash string like Java String::hashCode
hash_string() {
str="${1:-}" h=0
while [ -n "$str" ]; do
char="${str%"${str#?}"}"
h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
str="${str#?}"
done
printf %x\\n $h
}
verbose() { :; }
[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
die() {
printf %s\\n "$1" >&2
exit 1
}
trim() {
# MWRAPPER-139:
# Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
# Needed for removing poorly interpreted newline sequences when running in more
# exotic environments such as mingw bash on Windows.
printf "%s" "${1}" | tr -d '[:space:]'
}
scriptDir="$(dirname "$0")"
scriptName="$(basename "$0")"
# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
while IFS="=" read -r key value; do
case "${key-}" in
distributionUrl) distributionUrl=$(trim "${value-}") ;;
distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
esac
done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
case "${distributionUrl##*/}" in
maven-mvnd-*bin.*)
MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
*AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
:Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
:Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
:Linux*x86_64*) distributionPlatform=linux-amd64 ;;
*)
echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
distributionPlatform=linux-amd64
;;
esac
distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
;;
maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
esac
# apply MVNW_REPOURL and calculate MAVEN_HOME
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/
[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
distributionUrlName="${distributionUrl##*/}"
distributionUrlNameMain="${distributionUrlName%.*}"
distributionUrlNameMain="${distributionUrlNameMain%-bin}"
MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
exec_maven() {
unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
}
if [ -d "$MAVEN_HOME" ]; then
verbose "found existing MAVEN_HOME at $MAVEN_HOME"
exec_maven "$@"
fi
case "${distributionUrl-}" in
*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
esac
# prepare tmp dir
if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
trap clean HUP INT TERM EXIT
else
die "cannot create temp dir"
fi
mkdir -p -- "${MAVEN_HOME%/*}"
# Download and Install Apache Maven
verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
verbose "Downloading from: $distributionUrl"
verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
# select .zip or .tar.gz
if ! command -v unzip >/dev/null; then
distributionUrl="${distributionUrl%.zip}.tar.gz"
distributionUrlName="${distributionUrl##*/}"
fi
# verbose opt
__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
# normalize http auth
case "${MVNW_PASSWORD:+has-password}" in
'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
esac
if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
verbose "Found wget ... using wget"
wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
verbose "Found curl ... using curl"
curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
elif set_java_home; then
verbose "Falling back to use Java to download"
javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
cat >"$javaSource" <<-END
public class Downloader extends java.net.Authenticator
{
protected java.net.PasswordAuthentication getPasswordAuthentication()
{
return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
}
public static void main( String[] args ) throws Exception
{
setDefault( new Downloader() );
java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
}
}
END
# For Cygwin/MinGW, switch paths to Windows format before running javac and java
verbose " - Compiling Downloader.java ..."
"$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
verbose " - Running Downloader.java ..."
"$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
fi
# If specified, validate the SHA-256 sum of the Maven distribution zip file
if [ -n "${distributionSha256Sum-}" ]; then
distributionSha256Result=false
if [ "$MVN_CMD" = mvnd.sh ]; then
echo "Checksum validation is not supported for maven-mvnd." >&2
echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
exit 1
elif command -v sha256sum >/dev/null; then
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
distributionSha256Result=true
fi
elif command -v shasum >/dev/null; then
if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
distributionSha256Result=true
fi
else
echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
exit 1
fi
if [ $distributionSha256Result = false ]; then
echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
exit 1
fi
fi
# unzip and move
if command -v unzip >/dev/null; then
unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
else
tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
fi
# Find the actual extracted directory name (handles snapshots where filename != directory name)
actualDistributionDir=""
# First try the expected directory name (for regular distributions)
if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
actualDistributionDir="$distributionUrlNameMain"
fi
fi
# If not found, search for any directory with the Maven executable (for snapshots)
if [ -z "$actualDistributionDir" ]; then
# enable globbing to iterate over items
set +f
for dir in "$TMP_DOWNLOAD_DIR"/*; do
if [ -d "$dir" ]; then
if [ -f "$dir/bin/$MVN_CMD" ]; then
actualDistributionDir="$(basename "$dir")"
break
fi
fi
done
set -f
fi
if [ -z "$actualDistributionDir" ]; then
verbose "Contents of $TMP_DOWNLOAD_DIR:"
verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
die "Could not find Maven distribution directory in extracted archive"
fi
verbose "Found extracted Maven distribution directory: $actualDistributionDir"
printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
clean || :
exec_maven "$@"
apache-datafusion-java-0.1.0/mvnw.cmd 0000664 0000000 0000000 00000020441 15205164533 0017464 0 ustar 00root root 0000000 0000000 <# : batch portion
@REM ----------------------------------------------------------------------------
@REM Licensed to the Apache Software Foundation (ASF) under one
@REM or more contributor license agreements. See the NOTICE file
@REM distributed with this work for additional information
@REM regarding copyright ownership. The ASF licenses this file
@REM to you under the Apache License, Version 2.0 (the
@REM "License"); you may not use this file except in compliance
@REM with the License. You may obtain a copy of the License at
@REM
@REM http://www.apache.org/licenses/LICENSE-2.0
@REM
@REM Unless required by applicable law or agreed to in writing,
@REM software distributed under the License is distributed on an
@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@REM KIND, either express or implied. See the License for the
@REM specific language governing permissions and limitations
@REM under the License.
@REM ----------------------------------------------------------------------------
@REM ----------------------------------------------------------------------------
@REM Apache Maven Wrapper startup batch script, version 3.3.4
@REM
@REM Optional ENV vars
@REM MVNW_REPOURL - repo url base for downloading maven distribution
@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output
@REM ----------------------------------------------------------------------------
@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
@SET __MVNW_CMD__=
@SET __MVNW_ERROR__=
@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
@SET PSModulePath=
@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
)
@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
@SET __MVNW_PSMODULEP_SAVE=
@SET __MVNW_ARG0_NAME__=
@SET MVNW_USERNAME=
@SET MVNW_PASSWORD=
@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
@echo Cannot start maven from wrapper >&2 && exit /b 1
@GOTO :EOF
: end batch / begin powershell #>
$ErrorActionPreference = "Stop"
if ($env:MVNW_VERBOSE -eq "true") {
$VerbosePreference = "Continue"
}
# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
if (!$distributionUrl) {
Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
}
switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
"maven-mvnd-*" {
$USE_MVND = $true
$distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
$MVN_CMD = "mvnd.cmd"
break
}
default {
$USE_MVND = $false
$MVN_CMD = $script -replace '^mvnw','mvn'
break
}
}
# apply MVNW_REPOURL and calculate MAVEN_HOME
# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/
if ($env:MVNW_REPOURL) {
$MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
$distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
}
$distributionUrlName = $distributionUrl -replace '^.*/',''
$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
$MAVEN_M2_PATH = "$HOME/.m2"
if ($env:MAVEN_USER_HOME) {
$MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
}
if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
}
$MAVEN_WRAPPER_DISTS = $null
if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
$MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
} else {
$MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
}
$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
exit $?
}
if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
}
# prepare tmp dir
$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
trap {
if ($TMP_DOWNLOAD_DIR.Exists) {
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
}
}
New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
# Download and Install Apache Maven
Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
Write-Verbose "Downloading from: $distributionUrl"
Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
$webclient = New-Object System.Net.WebClient
if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
$webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
}
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
# If specified, validate the SHA-256 sum of the Maven distribution zip file
$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
if ($distributionSha256Sum) {
if ($USE_MVND) {
Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
}
Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
}
}
# unzip and move
Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
# Find the actual extracted directory name (handles snapshots where filename != directory name)
$actualDistributionDir = ""
# First try the expected directory name (for regular distributions)
$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
$actualDistributionDir = $distributionUrlNameMain
}
# If not found, search for any directory with the Maven executable (for snapshots)
if (!$actualDistributionDir) {
Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
$testPath = Join-Path $_.FullName "bin/$MVN_CMD"
if (Test-Path -Path $testPath -PathType Leaf) {
$actualDistributionDir = $_.Name
}
}
}
if (!$actualDistributionDir) {
Write-Error "Could not find Maven distribution directory in extracted archive"
}
Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
try {
Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
} catch {
if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
Write-Error "fail to move MAVEN_HOME"
}
} finally {
try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
}
Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
apache-datafusion-java-0.1.0/native/ 0000775 0000000 0000000 00000000000 15205164533 0017275 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/native/Cargo.lock 0000664 0000000 0000000 00000325572 15205164533 0021220 0 ustar 00root root 0000000 0000000 # This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "adler2"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
[[package]]
name = "ahash"
version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
dependencies = [
"cfg-if",
"const-random",
"getrandom 0.3.4",
"once_cell",
"version_check",
"zerocopy",
]
[[package]]
name = "aho-corasick"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
dependencies = [
"memchr",
]
[[package]]
name = "alloc-no-stdlib"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
[[package]]
name = "alloc-stdlib"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
dependencies = [
"alloc-no-stdlib",
]
[[package]]
name = "allocator-api2"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "anyhow"
version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "apache-avro"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf"
dependencies = [
"bigdecimal",
"bon",
"bzip2",
"crc32fast",
"digest",
"liblzma",
"log",
"miniz_oxide",
"num-bigint",
"quad-rand",
"rand 0.9.4",
"regex-lite",
"serde",
"serde_bytes",
"serde_json",
"snap",
"strum",
"strum_macros",
"thiserror 2.0.18",
"uuid",
"zstd",
]
[[package]]
name = "ar_archive_writer"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b"
dependencies = [
"object",
]
[[package]]
name = "arrayref"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
[[package]]
name = "arrayvec"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "607e64bb911ee4f90483e044fe78f175989148c2892e659a2cd25429e782ec54"
dependencies = [
"arrow-arith",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-csv",
"arrow-data",
"arrow-ipc",
"arrow-json",
"arrow-ord",
"arrow-row",
"arrow-schema",
"arrow-select",
"arrow-string",
]
[[package]]
name = "arrow-arith"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e754319ed8a85d817fe7adf183227e0b5308b82790a737b426c1124626b48118"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"num-traits",
]
[[package]]
name = "arrow-array"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841321891f247aa86c6112c80d83d89cb36e0addd020fa2425085b8eb6c3f579"
dependencies = [
"ahash",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"chrono-tz",
"half",
"hashbrown 0.17.1",
"num-complex",
"num-integer",
"num-traits",
]
[[package]]
name = "arrow-buffer"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f955dfb73fae000425f49c8226d2044dab60fb7ad4af1e24f961756354d996c9"
dependencies = [
"bytes",
"half",
"num-bigint",
"num-traits",
]
[[package]]
name = "arrow-cast"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca5e686972523798f76bef355145bc1ae25a84c731e650268d31ab763c701663"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-ord",
"arrow-schema",
"arrow-select",
"atoi",
"base64",
"chrono",
"comfy-table",
"half",
"lexical-core",
"num-traits",
"ryu",
]
[[package]]
name = "arrow-csv"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86c276756867fc8186ec380c72c290e6e3b23a1d4fb05df6b1d62d2e62666d48"
dependencies = [
"arrow-array",
"arrow-cast",
"arrow-schema",
"chrono",
"csv",
"csv-core",
"regex",
]
[[package]]
name = "arrow-data"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db3b5846209775b6dc8056d77ff9a032b27043383dd5488abd0b663e265b9373"
dependencies = [
"arrow-buffer",
"arrow-schema",
"half",
"num-integer",
"num-traits",
]
[[package]]
name = "arrow-ipc"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd8907ddd8f9fbabf91ec2c85c1d81fe2874e336d2443eb36373595e28b98dd5"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"flatbuffers",
"lz4_flex",
"zstd",
]
[[package]]
name = "arrow-json"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4518c59acc501f10d7dcae397fe12b8db3d81bc7de94456f8a58f9165d6f502"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-ord",
"arrow-schema",
"arrow-select",
"chrono",
"half",
"indexmap",
"itoa",
"lexical-core",
"memchr",
"num-traits",
"ryu",
"serde_core",
"serde_json",
"simdutf8",
]
[[package]]
name = "arrow-ord"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efa70d9d6b1356f1fb9f1f651b84a725b7e0abb93f188cf7d31f14abfa2f2e6f"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
]
[[package]]
name = "arrow-row"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faec88a945338192beffbbd4be0def70135422930caa244ac3cec0cd213b26b4"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"half",
]
[[package]]
name = "arrow-schema"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18aa020f6bc8e5201dcd2d4b7f98c68f8a410ef37128263243e6ff2a47a67d4f"
dependencies = [
"bitflags",
"serde_core",
"serde_json",
]
[[package]]
name = "arrow-select"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a657ab5132e9c8ca3b24eb15a823d0ced38017fe3930ff50167466b02e2d592c"
dependencies = [
"ahash",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"num-traits",
]
[[package]]
name = "arrow-string"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6de2efbbd1a9f9780ceb8d1ff5d20421b35863b361e3386b4f571f1fc69fcb8"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"memchr",
"num-traits",
"regex",
"regex-syntax",
]
[[package]]
name = "async-compression"
version = "0.4.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac"
dependencies = [
"compression-codecs",
"compression-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "async-trait"
version = "0.1.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "atoi"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
dependencies = [
"num-traits",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "autocfg"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "base64"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bigdecimal"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695"
dependencies = [
"autocfg",
"libm",
"num-bigint",
"num-integer",
"num-traits",
"serde",
]
[[package]]
name = "bitflags"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
[[package]]
name = "blake2"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
dependencies = [
"digest",
]
[[package]]
name = "blake3"
version = "1.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
"cpufeatures 0.3.0",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
dependencies = [
"generic-array",
]
[[package]]
name = "bon"
version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"
dependencies = [
"bon-macros",
"rustversion",
]
[[package]]
name = "bon-macros"
version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
dependencies = [
"darling",
"ident_case",
"prettyplease",
"proc-macro2",
"quote",
"rustversion",
"syn",
]
[[package]]
name = "brotli"
version = "8.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor",
]
[[package]]
name = "brotli-decompressor"
version = "5.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
]
[[package]]
name = "bumpalo"
version = "3.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
[[package]]
name = "byteorder"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
[[package]]
name = "bzip2"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c"
dependencies = [
"libbz2-rs-sys",
]
[[package]]
name = "cc"
version = "1.2.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
dependencies = [
"find-msvc-tools",
"jobserver",
"libc",
"shlex",
]
[[package]]
name = "cesu8"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chacha20"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
dependencies = [
"cfg-if",
"cpufeatures 0.3.0",
"rand_core 0.10.1",
]
[[package]]
name = "chrono"
version = "0.4.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
dependencies = [
"iana-time-zone",
"num-traits",
"serde",
"windows-link",
]
[[package]]
name = "chrono-tz"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
dependencies = [
"chrono",
"phf",
]
[[package]]
name = "combine"
version = "4.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
dependencies = [
"bytes",
"memchr",
]
[[package]]
name = "comfy-table"
version = "7.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47"
dependencies = [
"unicode-segmentation",
"unicode-width",
]
[[package]]
name = "compression-codecs"
version = "0.4.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf"
dependencies = [
"bzip2",
"compression-core",
"flate2",
"liblzma",
"memchr",
"zstd",
"zstd-safe",
]
[[package]]
name = "compression-core"
version = "0.4.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789"
[[package]]
name = "const-random"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359"
dependencies = [
"const-random-macro",
]
[[package]]
name = "const-random-macro"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e"
dependencies = [
"getrandom 0.2.17",
"once_cell",
"tiny-keccak",
]
[[package]]
name = "constant_time_eq"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
[[package]]
name = "core-foundation"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "cpufeatures"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
dependencies = [
"libc",
]
[[package]]
name = "cpufeatures"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
dependencies = [
"libc",
]
[[package]]
name = "crc32fast"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
dependencies = [
"generic-array",
"typenum",
]
[[package]]
name = "csv"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938"
dependencies = [
"csv-core",
"itoa",
"ryu",
"serde_core",
]
[[package]]
name = "csv-core"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782"
dependencies = [
"memchr",
]
[[package]]
name = "darling"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0"
dependencies = [
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn",
]
[[package]]
name = "darling_macro"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
"darling_core",
"quote",
"syn",
]
[[package]]
name = "dashmap"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
]
[[package]]
name = "datafusion"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93db0e623840612f7f2cd757f7e8a8922064192363732c88692e0870016e141b"
dependencies = [
"arrow",
"arrow-schema",
"async-trait",
"bytes",
"bzip2",
"chrono",
"datafusion-catalog",
"datafusion-catalog-listing",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-datasource-arrow",
"datafusion-datasource-avro",
"datafusion-datasource-csv",
"datafusion-datasource-json",
"datafusion-datasource-parquet",
"datafusion-execution",
"datafusion-expr",
"datafusion-expr-common",
"datafusion-functions",
"datafusion-functions-aggregate",
"datafusion-functions-nested",
"datafusion-functions-table",
"datafusion-functions-window",
"datafusion-optimizer",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-optimizer",
"datafusion-physical-plan",
"datafusion-session",
"datafusion-sql",
"flate2",
"futures",
"itertools",
"liblzma",
"log",
"object_store",
"parking_lot",
"parquet",
"rand 0.9.4",
"regex",
"sqlparser",
"tempfile",
"tokio",
"url",
"uuid",
"zstd",
]
[[package]]
name = "datafusion-catalog"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37cefde60b26a7f4ff61e9d2ff2833322f91df2b568d7238afe67bde5bdffb66"
dependencies = [
"arrow",
"async-trait",
"dashmap",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-expr",
"datafusion-physical-plan",
"datafusion-session",
"futures",
"itertools",
"log",
"object_store",
"parking_lot",
"tokio",
]
[[package]]
name = "datafusion-catalog-listing"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17e112307715d6a7a331111a4c2330ff54bc237183511c319e3708a4cff431fb"
dependencies = [
"arrow",
"async-trait",
"datafusion-catalog",
"datafusion-common",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"futures",
"itertools",
"log",
"object_store",
]
[[package]]
name = "datafusion-common"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d72a11ca44a95e1081870d3abb80c717496e8a7acb467a1d3e932bb636af5cc2"
dependencies = [
"ahash",
"apache-avro",
"arrow",
"arrow-ipc",
"chrono",
"half",
"hashbrown 0.16.1",
"indexmap",
"itertools",
"libc",
"log",
"object_store",
"parquet",
"paste",
"recursive",
"sqlparser",
"tokio",
"web-time",
]
[[package]]
name = "datafusion-common-runtime"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89f4afaed29670ec4fd6053643adc749fe3f4bc9d1ce1b8c5679b22c67d12def"
dependencies = [
"futures",
"log",
"tokio",
]
[[package]]
name = "datafusion-datasource"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9fb386e1691355355a96419978a0022b7947b44d4a24a6ea99f00b6b485cbb6"
dependencies = [
"arrow",
"async-compression",
"async-trait",
"bytes",
"bzip2",
"chrono",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
"flate2",
"futures",
"glob",
"itertools",
"liblzma",
"log",
"object_store",
"rand 0.9.4",
"tokio",
"tokio-util",
"url",
"zstd",
]
[[package]]
name = "datafusion-datasource-arrow"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffa6c52cfed0734c5f93754d1c0175f558175248bf686c944fb05c373e5fc096"
dependencies = [
"arrow",
"arrow-ipc",
"async-trait",
"bytes",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
"futures",
"itertools",
"object_store",
"tokio",
]
[[package]]
name = "datafusion-datasource-avro"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a579c3bd290c66ea4b269493e75e8a3ed42c9c895a651f10210a29538aee50c4"
dependencies = [
"apache-avro",
"arrow",
"async-trait",
"bytes",
"datafusion-common",
"datafusion-datasource",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
"futures",
"num-traits",
"object_store",
]
[[package]]
name = "datafusion-datasource-csv"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "503f29e0582c1fc189578d665ff57d9300da1f80c282777d7eb67bb79fb8cdca"
dependencies = [
"arrow",
"async-trait",
"bytes",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
"futures",
"object_store",
"regex",
"tokio",
]
[[package]]
name = "datafusion-datasource-json"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e33804749abc8d0c8cb7473228483cb8070e524c6f6086ee1b85a64debe2b3d2"
dependencies = [
"arrow",
"async-trait",
"bytes",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-session",
"futures",
"object_store",
"serde_json",
"tokio",
"tokio-stream",
]
[[package]]
name = "datafusion-datasource-parquet"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a8e0365e0e08e8ff94d912f0ababcf9065a1a304018ba90b1fc83c855b4997"
dependencies = [
"arrow",
"async-trait",
"bytes",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-datasource",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr",
"datafusion-physical-expr-adapter",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-pruning",
"datafusion-session",
"futures",
"itertools",
"log",
"object_store",
"parking_lot",
"parquet",
"tokio",
]
[[package]]
name = "datafusion-doc"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de6ac0df1662b9148ad3c987978b32cbec7c772f199b1d53520c8fa764a87ee"
[[package]]
name = "datafusion-execution"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c03c7fbdaefcca4ef6ffe425a5fc2325763bfb426599bb0bf4536466efabe709"
dependencies = [
"arrow",
"arrow-buffer",
"async-trait",
"chrono",
"dashmap",
"datafusion-common",
"datafusion-expr",
"datafusion-physical-expr-common",
"futures",
"log",
"object_store",
"parking_lot",
"rand 0.9.4",
"tempfile",
"url",
]
[[package]]
name = "datafusion-expr"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "574b9b6977fedbd2a611cbff12e5caf90f31640ad9dc5870f152836d94bad0dd"
dependencies = [
"arrow",
"async-trait",
"chrono",
"datafusion-common",
"datafusion-doc",
"datafusion-expr-common",
"datafusion-functions-aggregate-common",
"datafusion-functions-window-common",
"datafusion-physical-expr-common",
"indexmap",
"itertools",
"paste",
"recursive",
"serde_json",
"sqlparser",
]
[[package]]
name = "datafusion-expr-common"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d7c3adf3db8bf61e92eb90cb659c8e8b734593a8f7c8e12a843c7ddba24b87e"
dependencies = [
"arrow",
"datafusion-common",
"indexmap",
"itertools",
"paste",
]
[[package]]
name = "datafusion-functions"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f28aa4e10384e782774b10e72aca4d93ef7b31aa653095d9d4536b0a3dbc51b6"
dependencies = [
"arrow",
"arrow-buffer",
"base64",
"blake2",
"blake3",
"chrono",
"chrono-tz",
"datafusion-common",
"datafusion-doc",
"datafusion-execution",
"datafusion-expr",
"datafusion-expr-common",
"datafusion-macros",
"hex",
"itertools",
"log",
"md-5",
"memchr",
"num-traits",
"rand 0.9.4",
"regex",
"sha2",
"unicode-segmentation",
"uuid",
]
[[package]]
name = "datafusion-functions-aggregate"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00aa6217e56098ba84e0a338176fe52f0a84cca398021512c6c8c5eff806d0ad"
dependencies = [
"ahash",
"arrow",
"datafusion-common",
"datafusion-doc",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions-aggregate-common",
"datafusion-macros",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"half",
"log",
"num-traits",
"paste",
]
[[package]]
name = "datafusion-functions-aggregate-common"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b511250349407db7c43832ab2de63f5557b19a20dfd236b39ca2c04468b50d47"
dependencies = [
"ahash",
"arrow",
"datafusion-common",
"datafusion-expr-common",
"datafusion-physical-expr-common",
]
[[package]]
name = "datafusion-functions-nested"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef13a858e20d50f0a9bb5e96e7ac82b4e7597f247515bccca4fdd2992df0212a"
dependencies = [
"arrow",
"arrow-ord",
"datafusion-common",
"datafusion-doc",
"datafusion-execution",
"datafusion-expr",
"datafusion-expr-common",
"datafusion-functions",
"datafusion-functions-aggregate",
"datafusion-functions-aggregate-common",
"datafusion-macros",
"datafusion-physical-expr-common",
"hashbrown 0.16.1",
"itertools",
"itoa",
"log",
"paste",
]
[[package]]
name = "datafusion-functions-table"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b40d3f5bbb3905f9ccb1ce9485a9595c77b69758a7c24d3ba79e334ff51e7e"
dependencies = [
"arrow",
"async-trait",
"datafusion-catalog",
"datafusion-common",
"datafusion-expr",
"datafusion-physical-plan",
"parking_lot",
"paste",
]
[[package]]
name = "datafusion-functions-window"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4e88ec9d57c9b685d02f58bfee7be62d72610430ddcedb82a08e5d9925dbfb6"
dependencies = [
"arrow",
"datafusion-common",
"datafusion-doc",
"datafusion-expr",
"datafusion-functions-window-common",
"datafusion-macros",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"log",
"paste",
]
[[package]]
name = "datafusion-functions-window-common"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8307bb93519b1a91913723a1130cfafeee3f72200d870d88e91a6fc5470ede5c"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
]
[[package]]
name = "datafusion-jni"
version = "0.1.0"
dependencies = [
"arrow",
"async-trait",
"datafusion",
"datafusion-proto",
"futures",
"jni",
"object_store",
"prost",
"prost-build",
"protoc-bin-vendored",
"tokio",
"tokio-metrics",
"url",
]
[[package]]
name = "datafusion-macros"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e367e6a71051d0ebdd29b2f85d12059b38b1d1f172c6906e80016da662226bd"
dependencies = [
"datafusion-doc",
"quote",
"syn",
]
[[package]]
name = "datafusion-optimizer"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e929015451a67f77d9d8b727b2bf3a40c4445fdef6cdc53281d7d97c76888ace"
dependencies = [
"arrow",
"chrono",
"datafusion-common",
"datafusion-expr",
"datafusion-expr-common",
"datafusion-physical-expr",
"indexmap",
"itertools",
"log",
"recursive",
"regex",
"regex-syntax",
]
[[package]]
name = "datafusion-physical-expr"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b1e68aba7a4b350401cfdf25a3d6f989ad898a7410164afe9ca52080244cb59"
dependencies = [
"ahash",
"arrow",
"datafusion-common",
"datafusion-expr",
"datafusion-expr-common",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
"half",
"hashbrown 0.16.1",
"indexmap",
"itertools",
"parking_lot",
"paste",
"petgraph",
"recursive",
"tokio",
]
[[package]]
name = "datafusion-physical-expr-adapter"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea22315f33cf2e0adc104e8ec42e285f6ed93998d565c65e82fec6a9ee9f9db4"
dependencies = [
"arrow",
"datafusion-common",
"datafusion-expr",
"datafusion-functions",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"itertools",
]
[[package]]
name = "datafusion-physical-expr-common"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b04b45ea8ad3ac2d78f2ea2a76053e06591c9629c7a603eda16c10649ecf4362"
dependencies = [
"ahash",
"arrow",
"chrono",
"datafusion-common",
"datafusion-expr-common",
"hashbrown 0.16.1",
"indexmap",
"itertools",
"parking_lot",
]
[[package]]
name = "datafusion-physical-optimizer"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cb13397809a425918f608dfe8653f332015a3e330004ab191b4404187238b95"
dependencies = [
"arrow",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
"datafusion-expr-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-pruning",
"itertools",
"recursive",
]
[[package]]
name = "datafusion-physical-plan"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5edc023675791af9d5fb4cc4c24abf5f7bd3bd4dcf9e5bd90ea1eff6976dcc79"
dependencies = [
"ahash",
"arrow",
"arrow-ord",
"arrow-schema",
"async-trait",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions",
"datafusion-functions-aggregate-common",
"datafusion-functions-window-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"futures",
"half",
"hashbrown 0.16.1",
"indexmap",
"itertools",
"log",
"num-traits",
"parking_lot",
"pin-project-lite",
"tokio",
]
[[package]]
name = "datafusion-proto"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738"
dependencies = [
"arrow",
"chrono",
"datafusion-catalog",
"datafusion-catalog-listing",
"datafusion-common",
"datafusion-datasource",
"datafusion-datasource-arrow",
"datafusion-datasource-csv",
"datafusion-datasource-json",
"datafusion-datasource-parquet",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions-table",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"datafusion-proto-common",
"object_store",
"prost",
"rand 0.9.4",
]
[[package]]
name = "datafusion-proto-common"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9"
dependencies = [
"arrow",
"datafusion-common",
"prost",
]
[[package]]
name = "datafusion-pruning"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac8c76860e355616555081cab5968cec1af7a80701ff374510860bcd567e365a"
dependencies = [
"arrow",
"datafusion-common",
"datafusion-datasource",
"datafusion-expr-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
"datafusion-physical-plan",
"itertools",
"log",
]
[[package]]
name = "datafusion-session"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5412111aa48e2424ba926112e192f7a6b7e4ccb450145d25ce5ede9f19dc491e"
dependencies = [
"async-trait",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
"datafusion-physical-plan",
"parking_lot",
]
[[package]]
name = "datafusion-sql"
version = "53.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa0d133ddf8b9b3b872acac900157f783e7b879fe9a6bccf389abebbfac45ec1"
dependencies = [
"arrow",
"bigdecimal",
"chrono",
"datafusion-common",
"datafusion-expr",
"datafusion-functions-nested",
"indexmap",
"log",
"recursive",
"regex",
"sqlparser",
]
[[package]]
name = "digest"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
"crypto-common",
"subtle",
]
[[package]]
name = "displaydoc"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
]
[[package]]
name = "fastrand"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "fixedbitset"
version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
[[package]]
name = "flatbuffers"
version = "25.12.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
dependencies = [
"bitflags",
"rustc_version",
]
[[package]]
name = "flate2"
version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [
"crc32fast",
"miniz_oxide",
"zlib-rs",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foldhash"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "form_urlencoded"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
dependencies = [
"percent-encoding",
]
[[package]]
name = "futures"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
name = "futures-core"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
[[package]]
name = "futures-executor"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
[[package]]
name = "futures-macro"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures-sink"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893"
[[package]]
name = "futures-task"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
[[package]]
name = "futures-util"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"slab",
]
[[package]]
name = "generic-array"
version = "0.14.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
dependencies = [
"typenum",
"version_check",
]
[[package]]
name = "getrandom"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"wasi",
"wasm-bindgen",
]
[[package]]
name = "getrandom"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"r-efi 5.3.0",
"wasip2",
"wasm-bindgen",
]
[[package]]
name = "getrandom"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
dependencies = [
"cfg-if",
"libc",
"r-efi 6.0.0",
"rand_core 0.10.1",
"wasip2",
"wasip3",
]
[[package]]
name = "glob"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
[[package]]
name = "h2"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
dependencies = [
"atomic-waker",
"bytes",
"fnv",
"futures-core",
"futures-sink",
"http",
"indexmap",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "half"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
dependencies = [
"cfg-if",
"crunchy",
"num-traits",
"zerocopy",
]
[[package]]
name = "hashbrown"
version = "0.14.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"foldhash 0.1.5",
]
[[package]]
name = "hashbrown"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash 0.2.0",
]
[[package]]
name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "http"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
dependencies = [
"bytes",
"itoa",
]
[[package]]
name = "http-body"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
dependencies = [
"bytes",
"http",
]
[[package]]
name = "http-body-util"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
dependencies = [
"bytes",
"futures-core",
"http",
"http-body",
"pin-project-lite",
]
[[package]]
name = "httparse"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
[[package]]
name = "humantime"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
[[package]]
name = "hyper"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
dependencies = [
"atomic-waker",
"bytes",
"futures-channel",
"futures-core",
"h2",
"http",
"http-body",
"httparse",
"itoa",
"pin-project-lite",
"smallvec",
"tokio",
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.27.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
dependencies = [
"http",
"hyper",
"hyper-util",
"rustls",
"rustls-native-certs",
"tokio",
"tokio-rustls",
"tower-service",
]
[[package]]
name = "hyper-util"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
"base64",
"bytes",
"futures-channel",
"futures-util",
"http",
"http-body",
"hyper",
"ipnet",
"libc",
"percent-encoding",
"pin-project-lite",
"socket2",
"tokio",
"tower-service",
"tracing",
]
[[package]]
name = "iana-time-zone"
version = "0.1.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"log",
"wasm-bindgen",
"windows-core",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "icu_collections"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
dependencies = [
"displaydoc",
"potential_utf",
"utf8_iter",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
name = "icu_locale_core"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
dependencies = [
"displaydoc",
"litemap",
"tinystr",
"writeable",
"zerovec",
]
[[package]]
name = "icu_normalizer"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
dependencies = [
"icu_collections",
"icu_normalizer_data",
"icu_properties",
"icu_provider",
"smallvec",
"zerovec",
]
[[package]]
name = "icu_normalizer_data"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
[[package]]
name = "icu_properties"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
dependencies = [
"icu_collections",
"icu_locale_core",
"icu_properties_data",
"icu_provider",
"zerotrie",
"zerovec",
]
[[package]]
name = "icu_properties_data"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
[[package]]
name = "icu_provider"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
dependencies = [
"displaydoc",
"icu_locale_core",
"writeable",
"yoke",
"zerofrom",
"zerotrie",
"zerovec",
]
[[package]]
name = "id-arena"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "idna"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
dependencies = [
"idna_adapter",
"smallvec",
"utf8_iter",
]
[[package]]
name = "idna_adapter"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
dependencies = [
"icu_normalizer",
"icu_properties",
]
[[package]]
name = "indexmap"
version = "2.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
"hashbrown 0.17.1",
"serde",
"serde_core",
]
[[package]]
name = "integer-encoding"
version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
[[package]]
name = "ipnet"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "itertools"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jni"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"
dependencies = [
"cesu8",
"cfg-if",
"combine",
"jni-sys 0.3.1",
"log",
"thiserror 1.0.69",
"walkdir",
"windows-sys 0.45.0",
]
[[package]]
name = "jni-sys"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
dependencies = [
"jni-sys 0.4.1",
]
[[package]]
name = "jni-sys"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
dependencies = [
"jni-sys-macros",
]
[[package]]
name = "jni-sys-macros"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "jobserver"
version = "0.1.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
dependencies = [
"getrandom 0.3.4",
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
dependencies = [
"cfg-if",
"futures-util",
"once_cell",
"wasm-bindgen",
]
[[package]]
name = "leb128fmt"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "lexical-core"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594"
dependencies = [
"lexical-parse-float",
"lexical-parse-integer",
"lexical-util",
"lexical-write-float",
"lexical-write-integer",
]
[[package]]
name = "lexical-parse-float"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56"
dependencies = [
"lexical-parse-integer",
"lexical-util",
]
[[package]]
name = "lexical-parse-integer"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34"
dependencies = [
"lexical-util",
]
[[package]]
name = "lexical-util"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17"
[[package]]
name = "lexical-write-float"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361"
dependencies = [
"lexical-util",
"lexical-write-integer",
]
[[package]]
name = "lexical-write-integer"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df"
dependencies = [
"lexical-util",
]
[[package]]
name = "libbz2-rs-sys"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3a6a8c165077efc8f3a971534c50ea6a1a18b329ef4a66e897a7e3a1494565f"
[[package]]
name = "libc"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "liblzma"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6033b77c21d1f56deeae8014eb9fbe7bdf1765185a6c508b5ca82eeaed7f899"
dependencies = [
"liblzma-sys",
]
[[package]]
name = "liblzma-sys"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a60851d15cd8c5346eca4ab8babff585be2ae4bc8097c067291d3ffe2add3b6"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "libm"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "litemap"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
[[package]]
name = "lock_api"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
dependencies = [
"scopeguard",
]
[[package]]
name = "log"
version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "lru-slab"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lz4_flex"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
dependencies = [
"twox-hash",
]
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest",
]
[[package]]
name = "memchr"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "miniz_oxide"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
dependencies = [
"adler2",
"simd-adler32",
]
[[package]]
name = "mio"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
dependencies = [
"libc",
"wasi",
"windows-sys 0.61.2",
]
[[package]]
name = "multimap"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
[[package]]
name = "num-bigint"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
dependencies = [
"num-integer",
"num-traits",
"serde",
]
[[package]]
name = "num-complex"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
dependencies = [
"num-traits",
]
[[package]]
name = "num-integer"
version = "0.1.46"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
dependencies = [
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
"libm",
]
[[package]]
name = "object"
version = "0.37.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
dependencies = [
"memchr",
]
[[package]]
name = "object_store"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49"
dependencies = [
"async-trait",
"base64",
"bytes",
"chrono",
"form_urlencoded",
"futures-channel",
"futures-core",
"futures-util",
"http",
"http-body-util",
"humantime",
"hyper",
"itertools",
"md-5",
"parking_lot",
"percent-encoding",
"quick-xml",
"rand 0.10.1",
"reqwest",
"ring",
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"thiserror 2.0.18",
"tokio",
"tracing",
"url",
"walkdir",
"wasm-bindgen-futures",
"web-time",
]
[[package]]
name = "once_cell"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "openssl-probe"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
[[package]]
name = "ordered-float"
version = "2.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"
dependencies = [
"num-traits",
]
[[package]]
name = "parking_lot"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-link",
]
[[package]]
name = "parquet"
version = "58.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43d7efd3052f7d6ef601085559a246bc991e9a8cc77e02753737df6322ce35f1"
dependencies = [
"ahash",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-ipc",
"arrow-schema",
"arrow-select",
"base64",
"brotli",
"bytes",
"chrono",
"flate2",
"futures",
"half",
"hashbrown 0.17.1",
"lz4_flex",
"num-bigint",
"num-integer",
"num-traits",
"object_store",
"paste",
"seq-macro",
"simdutf8",
"snap",
"thrift",
"tokio",
"twox-hash",
"zstd",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "percent-encoding"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "petgraph"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455"
dependencies = [
"fixedbitset",
"hashbrown 0.15.5",
"indexmap",
"serde",
]
[[package]]
name = "phf"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
dependencies = [
"phf_shared",
]
[[package]]
name = "phf_shared"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981"
dependencies = [
"siphasher",
]
[[package]]
name = "pin-project-lite"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
[[package]]
name = "pkg-config"
version = "0.3.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
[[package]]
name = "potential_utf"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
dependencies = [
"zerovec",
]
[[package]]
name = "ppv-lite86"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
dependencies = [
"zerocopy",
]
[[package]]
name = "prettyplease"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "prost"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568"
dependencies = [
"bytes",
"prost-derive",
]
[[package]]
name = "prost-build"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
dependencies = [
"heck",
"itertools",
"log",
"multimap",
"petgraph",
"prettyplease",
"prost",
"prost-types",
"regex",
"syn",
"tempfile",
]
[[package]]
name = "prost-derive"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
dependencies = [
"anyhow",
"itertools",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "prost-types"
version = "0.14.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7"
dependencies = [
"prost",
]
[[package]]
name = "protoc-bin-vendored"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa"
dependencies = [
"protoc-bin-vendored-linux-aarch_64",
"protoc-bin-vendored-linux-ppcle_64",
"protoc-bin-vendored-linux-s390_64",
"protoc-bin-vendored-linux-x86_32",
"protoc-bin-vendored-linux-x86_64",
"protoc-bin-vendored-macos-aarch_64",
"protoc-bin-vendored-macos-x86_64",
"protoc-bin-vendored-win32",
]
[[package]]
name = "protoc-bin-vendored-linux-aarch_64"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c"
[[package]]
name = "protoc-bin-vendored-linux-ppcle_64"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c"
[[package]]
name = "protoc-bin-vendored-linux-s390_64"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0"
[[package]]
name = "protoc-bin-vendored-linux-x86_32"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5"
[[package]]
name = "protoc-bin-vendored-linux-x86_64"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78"
[[package]]
name = "protoc-bin-vendored-macos-aarch_64"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092"
[[package]]
name = "protoc-bin-vendored-macos-x86_64"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756"
[[package]]
name = "protoc-bin-vendored-win32"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3"
[[package]]
name = "psm"
version = "0.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea"
dependencies = [
"ar_archive_writer",
"cc",
]
[[package]]
name = "quad-rand"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
[[package]]
name = "quick-xml"
version = "0.39.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "quinn"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
dependencies = [
"bytes",
"cfg_aliases",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustls",
"socket2",
"thiserror 2.0.18",
"tokio",
"tracing",
"web-time",
]
[[package]]
name = "quinn-proto"
version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
dependencies = [
"bytes",
"getrandom 0.3.4",
"lru-slab",
"rand 0.9.4",
"ring",
"rustc-hash",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 2.0.18",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2",
"tracing",
"windows-sys 0.60.2",
]
[[package]]
name = "quote"
version = "1.0.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "r-efi"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "rand"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
dependencies = [
"rand_chacha",
"rand_core 0.9.5",
]
[[package]]
name = "rand"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
dependencies = [
"chacha20",
"getrandom 0.4.2",
"rand_core 0.10.1",
]
[[package]]
name = "rand_chacha"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
"rand_core 0.9.5",
]
[[package]]
name = "rand_core"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_core"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
[[package]]
name = "recursive"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e"
dependencies = [
"recursive-proc-macro-impl",
"stacker",
]
[[package]]
name = "recursive-proc-macro-impl"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b"
dependencies = [
"quote",
"syn",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-lite"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
[[package]]
name = "regex-syntax"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
name = "reqwest"
version = "0.12.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
"base64",
"bytes",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-rustls",
"hyper-util",
"js-sys",
"log",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls",
"rustls-native-certs",
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls",
"tokio-util",
"tower",
"tower-http",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
]
[[package]]
name = "ring"
version = "0.17.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
dependencies = [
"cc",
"cfg-if",
"getrandom 0.2.17",
"libc",
"untrusted",
"windows-sys 0.52.0",
]
[[package]]
name = "rustc-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
[[package]]
name = "rustc_version"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
dependencies = [
"semver",
]
[[package]]
name = "rustix"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
]
[[package]]
name = "rustls"
version = "0.23.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
dependencies = [
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki",
"subtle",
"zeroize",
]
[[package]]
name = "rustls-native-certs"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
dependencies = [
"openssl-probe",
"rustls-pki-types",
"schannel",
"security-framework",
]
[[package]]
name = "rustls-pki-types"
version = "1.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
dependencies = [
"web-time",
"zeroize",
]
[[package]]
name = "rustls-webpki"
version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
]
[[package]]
name = "rustversion"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
[[package]]
name = "ryu"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "schannel"
version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "security-framework"
version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
dependencies = [
"bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "semver"
version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
[[package]]
name = "seq-macro"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_bytes"
version = "0.11.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8"
dependencies = [
"serde",
"serde_core",
]
[[package]]
name = "serde_core"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
dependencies = [
"form_urlencoded",
"itoa",
"ryu",
"serde",
]
[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
"cpufeatures 0.2.17",
"digest",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "simd-adler32"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
[[package]]
name = "simdutf8"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
[[package]]
name = "siphasher"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
[[package]]
name = "slab"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
[[package]]
name = "smallvec"
version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
[[package]]
name = "snap"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
[[package]]
name = "socket2"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.61.2",
]
[[package]]
name = "sqlparser"
version = "0.61.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
dependencies = [
"log",
"recursive",
"sqlparser_derive",
]
[[package]]
name = "sqlparser_derive"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
[[package]]
name = "stacker"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190"
dependencies = [
"cc",
"cfg-if",
"libc",
"psm",
"windows-sys 0.61.2",
]
[[package]]
name = "strsim"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "strum"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
[[package]]
name = "strum_macros"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "subtle"
version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "syn"
version = "2.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
[[package]]
name = "synstructure"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tempfile"
version = "3.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.61.2",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl 1.0.69",
]
[[package]]
name = "thiserror"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4"
dependencies = [
"thiserror-impl 2.0.18",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "thiserror-impl"
version = "2.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "thrift"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09"
dependencies = [
"byteorder",
"integer-encoding",
"ordered-float",
]
[[package]]
name = "tiny-keccak"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
dependencies = [
"crunchy",
]
[[package]]
name = "tinystr"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
dependencies = [
"displaydoc",
"zerovec",
]
[[package]]
name = "tinyvec"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
dependencies = [
"bytes",
"libc",
"mio",
"pin-project-lite",
"socket2",
"tokio-macros",
"windows-sys 0.61.2",
]
[[package]]
name = "tokio-macros"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tokio-metrics"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9e81d53caf955549b1dec7af4ac2149e94cc25ed97b4a545151140281e2f528"
dependencies = [
"futures-util",
"pin-project-lite",
"tokio",
"tokio-stream",
]
[[package]]
name = "tokio-rustls"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
dependencies = [
"rustls",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
"tokio-util",
]
[[package]]
name = "tokio-util"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098"
dependencies = [
"bytes",
"futures-core",
"futures-sink",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tower"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
dependencies = [
"futures-core",
"futures-util",
"pin-project-lite",
"sync_wrapper",
"tokio",
"tower-layer",
"tower-service",
]
[[package]]
name = "tower-http"
version = "0.6.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51"
dependencies = [
"bitflags",
"bytes",
"futures-util",
"http",
"http-body",
"pin-project-lite",
"tower",
"tower-layer",
"tower-service",
"url",
]
[[package]]
name = "tower-layer"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tracing-core"
version = "0.1.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
dependencies = [
"once_cell",
]
[[package]]
name = "try-lock"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "twox-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
[[package]]
name = "typenum"
version = "1.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-segmentation"
version = "1.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9629274872b2bfaf8d66f5f15725007f635594914870f65218920345aa11aa8c"
[[package]]
name = "unicode-width"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]]
name = "unicode-xid"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
dependencies = [
"form_urlencoded",
"idna",
"percent-encoding",
"serde",
]
[[package]]
name = "utf8_iter"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "uuid"
version = "1.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"
dependencies = [
"getrandom 0.4.2",
"js-sys",
"serde_core",
"wasm-bindgen",
]
[[package]]
name = "version_check"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "want"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasip2"
version = "1.0.3+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
dependencies = [
"wit-bindgen 0.57.1",
]
[[package]]
name = "wasip3"
version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
dependencies = [
"wit-bindgen 0.51.0",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
dependencies = [
"bumpalo",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
dependencies = [
"unicode-ident",
]
[[package]]
name = "wasm-encoder"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
dependencies = [
"leb128fmt",
"wasmparser",
]
[[package]]
name = "wasm-metadata"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
dependencies = [
"anyhow",
"indexmap",
"wasm-encoder",
"wasmparser",
]
[[package]]
name = "wasm-streams"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "wasmparser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags",
"hashbrown 0.15.5",
"indexmap",
"semver",
]
[[package]]
name = "web-sys"
version = "0.3.98"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "winapi-util"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "windows-core"
version = "0.62.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
dependencies = [
"windows-implement",
"windows-interface",
"windows-link",
"windows-result",
"windows-strings",
]
[[package]]
name = "windows-implement"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "windows-interface"
version = "0.59.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-result"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-strings"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
"windows-targets 0.42.2",
]
[[package]]
name = "windows-sys"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.5",
]
[[package]]
name = "windows-sys"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
"windows-link",
]
[[package]]
name = "windows-targets"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
dependencies = [
"windows_aarch64_gnullvm 0.42.2",
"windows_aarch64_msvc 0.42.2",
"windows_i686_gnu 0.42.2",
"windows_i686_msvc 0.42.2",
"windows_x86_64_gnu 0.42.2",
"windows_x86_64_gnullvm 0.42.2",
"windows_x86_64_msvc 0.42.2",
]
[[package]]
name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.1",
"windows_aarch64_msvc 0.53.1",
"windows_i686_gnu 0.53.1",
"windows_i686_gnullvm 0.53.1",
"windows_i686_msvc 0.53.1",
"windows_x86_64_gnu 0.53.1",
"windows_x86_64_gnullvm 0.53.1",
"windows_x86_64_msvc 0.53.1",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "wit-bindgen"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
dependencies = [
"wit-bindgen-rust-macro",
]
[[package]]
name = "wit-bindgen"
version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "wit-bindgen-core"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
dependencies = [
"anyhow",
"heck",
"wit-parser",
]
[[package]]
name = "wit-bindgen-rust"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
dependencies = [
"anyhow",
"heck",
"indexmap",
"prettyplease",
"syn",
"wasm-metadata",
"wit-bindgen-core",
"wit-component",
]
[[package]]
name = "wit-bindgen-rust-macro"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
dependencies = [
"anyhow",
"prettyplease",
"proc-macro2",
"quote",
"syn",
"wit-bindgen-core",
"wit-bindgen-rust",
]
[[package]]
name = "wit-component"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags",
"indexmap",
"log",
"serde",
"serde_derive",
"serde_json",
"wasm-encoder",
"wasm-metadata",
"wasmparser",
"wit-parser",
]
[[package]]
name = "wit-parser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
dependencies = [
"anyhow",
"id-arena",
"indexmap",
"log",
"semver",
"serde",
"serde_derive",
"serde_json",
"unicode-xid",
"wasmparser",
]
[[package]]
name = "writeable"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
[[package]]
name = "yoke"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
dependencies = [
"stable_deref_trait",
"yoke-derive",
"zerofrom",
]
[[package]]
name = "yoke-derive"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
dependencies = [
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
name = "zerocopy"
version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "zerofrom"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"
dependencies = [
"zerofrom-derive",
]
[[package]]
name = "zerofrom-derive"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
dependencies = [
"proc-macro2",
"quote",
"syn",
"synstructure",
]
[[package]]
name = "zeroize"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
[[package]]
name = "zerotrie"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
dependencies = [
"displaydoc",
"yoke",
"zerofrom",
]
[[package]]
name = "zerovec"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
dependencies = [
"yoke",
"zerofrom",
"zerovec-derive",
]
[[package]]
name = "zerovec-derive"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "zlib-rs"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513"
[[package]]
name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
[[package]]
name = "zstd"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "7.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "2.0.16+zstd.1.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
dependencies = [
"cc",
"pkg-config",
]
apache-datafusion-java-0.1.0/native/Cargo.toml 0000664 0000000 0000000 00000006024 15205164533 0021227 0 ustar 00root root 0000000 0000000 # Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[package]
name = "datafusion-jni"
version = "0.1.0"
edition = "2021"
publish = false
[lib]
# `rlib` alongside `cdylib` so `cargo test` has a Rust-level harness for
# native-only invariants (e.g. error-classification routing through wrapped
# DataFusionError chains). The `cdylib` is still the artifact the JVM loads.
crate-type = ["cdylib", "rlib"]
[dependencies]
arrow = { version = "58", features = ["ffi"] }
async-trait = "0.1"
datafusion = { version = "53.1.0", features = ["avro"] }
datafusion-proto = "53.1.0"
futures = "0.3"
jni = "0.21"
# Pin to the same major as DataFusion 53.1 pulls in transitively (0.13.x)
# so we share the same `dyn ObjectStore` vtable and don't double-link.
object_store = { version = "0.13", default-features = false }
prost = "0.14"
tokio = { version = "1", features = ["rt-multi-thread"] }
# Tokio runtime metrics. Optional + cfg-gated: this crate's API surface lives
# behind `--cfg tokio_unstable`, so enabling the `runtime-metrics` feature also
# requires the caller to set `RUSTFLAGS="--cfg tokio_unstable"` at build time.
tokio-metrics = { version = "0.5", optional = true }
url = "2"
[features]
# Object-store backends are on by default so `make test` exercises them;
# downstream builds that strip a feature get a clear runtime error if a caller
# tries to register an unsupported backend. `runtime-metrics` is opt-in because
# it requires the `tokio_unstable` cfg flag at build time (see below).
default = [
"object-store-aws",
"object-store-gcp",
"object-store-http",
]
object-store-aws = ["object_store/aws"]
object-store-gcp = ["object_store/gcp"]
object-store-http = ["object_store/http"]
# Pulls in `tokio-metrics` so `SessionContext.runtimeStats()` returns real
# numbers from the underlying Tokio runtime. Off by default because
# `tokio-metrics` only compiles when `--cfg tokio_unstable` is set, and we
# don't want to force that flag onto every build. Enable explicitly:
#
# RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics
#
# The JNI handler returns a clear "feature not enabled" error from the JVM if
# invoked in a build that compiled the feature off, so the Java surface is
# unchanged either way.
runtime-metrics = ["dep:tokio-metrics"]
[build-dependencies]
prost-build = "0.14"
protoc-bin-vendored = "3"
apache-datafusion-java-0.1.0/native/build.rs 0000664 0000000 0000000 00000003170 15205164533 0020743 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
fn main() {
const PROTOS: &[&str] = &[
"../proto/session_options.proto",
"../proto/cache_manager_options.proto",
"../proto/file_compression_type.proto",
"../proto/arrow_read_options.proto",
"../proto/avro_read_options.proto",
"../proto/csv_read_options.proto",
"../proto/csv_write_options.proto",
"../proto/json_read_options.proto",
"../proto/json_write_options.proto",
"../proto/object_store_options.proto",
"../proto/parquet_read_options.proto",
];
for p in PROTOS {
println!("cargo:rerun-if-changed={p}");
}
let protoc = protoc_bin_vendored::protoc_bin_path().expect("vendored protoc not available");
std::env::set_var("PROTOC", protoc);
prost_build::compile_protos(PROTOS, &["../proto"]).expect("failed to compile protos");
}
apache-datafusion-java-0.1.0/native/src/ 0000775 0000000 0000000 00000000000 15205164533 0020064 5 ustar 00root root 0000000 0000000 apache-datafusion-java-0.1.0/native/src/arrow.rs 0000664 0000000 0000000 00000007365 15205164533 0021577 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use datafusion::error::DataFusionError;
use datafusion::execution::options::ArrowReadOptions;
use datafusion::prelude::SessionContext;
use jni::objects::{JByteArray, JClass, JString};
use jni::sys::jlong;
use jni::JNIEnv;
use prost::Message;
use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::proto_gen::ArrowReadOptionsProto;
use crate::runtime;
use crate::schema::decode_optional_schema;
fn with_arrow_options(
env: &mut JNIEnv,
options_bytes: JByteArray,
schema_ipc_bytes: JByteArray,
f: impl FnOnce(ArrowReadOptions) -> JniResult,
) -> JniResult {
let bytes: Vec = env.convert_byte_array(&options_bytes)?;
let p = ArrowReadOptionsProto::decode(bytes.as_slice())?;
let schema = decode_optional_schema(env, schema_ipc_bytes)?;
// ArrowReadOptions exposes `file_extension` as a public field (not a builder
// setter); `schema` is the only field with a fluent setter. Build via
// struct-update syntax to avoid clippy::field_reassign_with_default.
let file_ext = p.file_extension;
let mut opts = ArrowReadOptions {
file_extension: &file_ext,
..ArrowReadOptions::default()
};
if let Some(ref s) = schema {
opts = opts.schema(s);
}
f(opts)
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerArrowWithOptions<
'local,
>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
name: JString<'local>,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let name: String = env.get_string(&name)?.into();
let path: String = env.get_string(&path)?.into();
with_arrow_options(env, options_bytes, schema_ipc_bytes, |opts| {
runtime().block_on(async {
ctx.register_arrow(&name, &path, opts).await?;
Ok::<(), DataFusionError>(())
})?;
Ok(())
})
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readArrowWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let path: String = env.get_string(&path)?.into();
with_arrow_options(env, options_bytes, schema_ipc_bytes, |opts| {
let df = runtime().block_on(ctx.read_arrow(path, opts))?;
Ok(Box::into_raw(Box::new(df)) as jlong)
})
})
}
apache-datafusion-java-0.1.0/native/src/avro.rs 0000664 0000000 0000000 00000007340 15205164533 0021405 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use datafusion::error::DataFusionError;
use datafusion::execution::options::AvroReadOptions;
use datafusion::prelude::SessionContext;
use jni::objects::{JByteArray, JClass, JString};
use jni::sys::jlong;
use jni::JNIEnv;
use prost::Message;
use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::proto_gen::AvroReadOptionsProto;
use crate::runtime;
use crate::schema::decode_optional_schema;
fn with_avro_options(
env: &mut JNIEnv,
options_bytes: JByteArray,
schema_ipc_bytes: JByteArray,
f: impl FnOnce(AvroReadOptions) -> JniResult,
) -> JniResult {
let bytes: Vec = env.convert_byte_array(&options_bytes)?;
let p = AvroReadOptionsProto::decode(bytes.as_slice())?;
let schema = decode_optional_schema(env, schema_ipc_bytes)?;
// AvroReadOptions exposes `file_extension` as a public field (not a builder
// setter); `schema` is the only field with a fluent setter. Build via
// struct-update syntax to avoid clippy::field_reassign_with_default.
let file_ext = p.file_extension;
let mut opts = AvroReadOptions {
file_extension: &file_ext,
..AvroReadOptions::default()
};
if let Some(ref s) = schema {
opts = opts.schema(s);
}
f(opts)
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerAvroWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
name: JString<'local>,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let name: String = env.get_string(&name)?.into();
let path: String = env.get_string(&path)?.into();
with_avro_options(env, options_bytes, schema_ipc_bytes, |opts| {
runtime().block_on(async {
ctx.register_avro(&name, &path, opts).await?;
Ok::<(), DataFusionError>(())
})?;
Ok(())
})
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readAvroWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let path: String = env.get_string(&path)?.into();
with_avro_options(env, options_bytes, schema_ipc_bytes, |opts| {
let df = runtime().block_on(ctx.read_avro(path, opts))?;
Ok(Box::into_raw(Box::new(df)) as jlong)
})
})
}
apache-datafusion-java-0.1.0/native/src/cache_manager.rs 0000664 0000000 0000000 00000007226 15205164533 0023176 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Translate the [`CacheManagerOptionsProto`] message into a
//! [`CacheManagerConfig`] for [`RuntimeEnvBuilder::with_cache_manager`].
//!
//! Each of the three caches is independent; an unset proto field leaves the
//! corresponding upstream default in place (no cache for list-files / stats,
//! a `DefaultFilesMetadataCache` with the default limit for file metadata).
//! When a setter *is* present, the JNI layer always installs a fresh
//! `Default*Cache` impl -- the v1 contract is "configure the built-in
//! caches", not "swap in a custom one".
use std::sync::Arc;
use std::time::Duration;
use datafusion::execution::cache::cache_manager::CacheManagerConfig;
use datafusion::execution::cache::cache_unit::{
DefaultFileStatisticsCache, DefaultFilesMetadataCache,
};
use datafusion::execution::cache::DefaultListFilesCache;
use crate::errors::JniResult;
use crate::proto_gen::CacheManagerOptionsProto;
/// Build a [`CacheManagerConfig`] from the proto. Returns `Ok(None)` if the
/// caller did not set any cache-manager field, so the JNI layer can skip the
/// `with_cache_manager(...)` call entirely and let upstream's own
/// `RuntimeEnvBuilder` defaults apply.
pub(crate) fn build_config(
opts: &CacheManagerOptionsProto,
) -> JniResult> {
// Treat "all three slots unset" as "caller didn't pass a cache_manager
// message at all" -- avoids overriding upstream's cache-manager-default
// with a freshly-constructed-but-empty config that happens to be
// equivalent today but might diverge on a future DataFusion release.
if opts.file_metadata_cache_max_bytes.is_none()
&& opts.list_files_cache.is_none()
&& opts.file_statistics_cache_enabled.is_none()
{
return Ok(None);
}
let mut config = CacheManagerConfig::default();
if let Some(max_bytes) = opts.file_metadata_cache_max_bytes {
let max = max_bytes as usize;
config.file_metadata_cache = Some(Arc::new(DefaultFilesMetadataCache::new(max)));
config.metadata_cache_limit = max;
}
if let Some(lfc) = &opts.list_files_cache {
// `max_bytes` unset → use upstream's documented default
// (currently 1 MiB; pulled from CacheManagerConfig::default()
// rather than re-declared here so we track upstream automatically).
let default_limit = CacheManagerConfig::default().list_files_cache_limit;
let max = lfc.max_bytes.map(|v| v as usize).unwrap_or(default_limit);
let ttl = lfc.ttl_millis.map(Duration::from_millis);
config.list_files_cache = Some(Arc::new(DefaultListFilesCache::new(max, ttl)));
config.list_files_cache_limit = max;
config.list_files_cache_ttl = ttl;
}
if opts.file_statistics_cache_enabled.unwrap_or(false) {
config.table_files_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default()));
}
Ok(Some(config))
}
apache-datafusion-java-0.1.0/native/src/csv.rs 0000664 0000000 0000000 00000020650 15205164533 0021230 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use datafusion::common::config::CsvOptions;
use datafusion::common::parsers::CompressionTypeVariant;
use datafusion::dataframe::{DataFrame, DataFrameWriteOptions};
use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
use datafusion::error::DataFusionError;
use datafusion::prelude::{CsvReadOptions, SessionContext};
use jni::objects::{JByteArray, JClass, JString};
use jni::sys::jlong;
use jni::JNIEnv;
use prost::Message;
use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::proto_gen::{
CsvReadOptionsProto, CsvWriteOptionsProto, FileCompressionType as ProtoFileCompressionType,
};
use crate::runtime;
use crate::schema::decode_optional_schema;
fn with_csv_options(
env: &mut JNIEnv,
options_bytes: JByteArray,
schema_ipc_bytes: JByteArray,
f: impl FnOnce(CsvReadOptions) -> JniResult,
) -> JniResult {
let bytes: Vec = env.convert_byte_array(&options_bytes)?;
let p = CsvReadOptionsProto::decode(bytes.as_slice())?;
let schema = decode_optional_schema(env, schema_ipc_bytes)?;
let compression = match p.file_compression_type() {
ProtoFileCompressionType::Unspecified => {
return Err("CsvReadOptionsProto.file_compression_type is UNSPECIFIED".into());
}
ProtoFileCompressionType::Uncompressed => FileCompressionType::UNCOMPRESSED,
ProtoFileCompressionType::Gzip => FileCompressionType::GZIP,
ProtoFileCompressionType::Bzip2 => FileCompressionType::BZIP2,
ProtoFileCompressionType::Xz => FileCompressionType::XZ,
ProtoFileCompressionType::Zstd => FileCompressionType::ZSTD,
};
let file_ext = p.file_extension;
let mut opts = CsvReadOptions::new()
.has_header(p.has_header)
.delimiter(p.delimiter as u8)
.quote(p.quote as u8)
.file_extension(&file_ext)
.file_compression_type(compression);
if let Some(t) = p.terminator {
opts = opts.terminator(Some(t as u8));
}
if let Some(e) = p.escape {
opts = opts.escape(e as u8);
}
if let Some(c) = p.comment {
opts = opts.comment(c as u8);
}
if let Some(n) = p.newlines_in_values {
opts = opts.newlines_in_values(n);
}
if let Some(n) = p.schema_infer_max_records {
opts = opts.schema_infer_max_records(n as usize);
}
if let Some(ref s) = schema {
opts = opts.schema(s);
}
f(opts)
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerCsvWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
name: JString<'local>,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let name: String = env.get_string(&name)?.into();
let path: String = env.get_string(&path)?.into();
with_csv_options(env, options_bytes, schema_ipc_bytes, |opts| {
runtime().block_on(async {
ctx.register_csv(&name, &path, opts).await?;
Ok::<(), DataFusionError>(())
})?;
Ok(())
})
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readCsvWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let path: String = env.get_string(&path)?.into();
with_csv_options(env, options_bytes, schema_ipc_bytes, |opts| {
let df = runtime().block_on(ctx.read_csv(path, opts))?;
Ok(Box::into_raw(Box::new(df)) as jlong)
})
})
}
fn proto_compression_to_variant(p: ProtoFileCompressionType) -> JniResult {
match p {
ProtoFileCompressionType::Unspecified => {
Err("CsvWriteOptionsProto.file_compression_type is UNSPECIFIED".into())
}
ProtoFileCompressionType::Uncompressed => Ok(CompressionTypeVariant::UNCOMPRESSED),
ProtoFileCompressionType::Gzip => Ok(CompressionTypeVariant::GZIP),
ProtoFileCompressionType::Bzip2 => Ok(CompressionTypeVariant::BZIP2),
ProtoFileCompressionType::Xz => Ok(CompressionTypeVariant::XZ),
ProtoFileCompressionType::Zstd => Ok(CompressionTypeVariant::ZSTD),
}
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_writeCsvWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
path: JString<'local>,
options_bytes: JByteArray<'local>,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let path: String = env.get_string(&path)?.into();
let bytes: Vec = env.convert_byte_array(&options_bytes)?;
let p = CsvWriteOptionsProto::decode(bytes.as_slice())?;
// Decode the file_compression_type field eagerly so an unknown wire
// value surfaces as a clear error rather than a silent default.
let compression = if p.file_compression_type.is_some() {
Some(proto_compression_to_variant(p.file_compression_type())?)
} else {
None
};
// When the caller left `singleFileOutput` unset, force directory output (`false`)
// rather than leaving DataFusion in `Automatic` mode. Automatic mode treats paths
// with an extension (e.g. `out.csv`) as single-file targets, which would silently
// contradict the documented "directory unless overridden" default and surprise any
// caller that hands writeCsv a `.csv` path.
let mut write_opts = DataFrameWriteOptions::new()
.with_single_file_output(p.single_file_output.unwrap_or(false));
if !p.partition_cols.is_empty() {
write_opts = write_opts.with_partition_by(p.partition_cols.clone());
}
// Build CsvOptions only when at least one writer-side knob is set, so
// the DataFusion default is preserved when the caller passes
// `new CsvWriteOptions()`.
let writer_opts: Option = if p.has_header.is_some()
|| p.delimiter.is_some()
|| p.quote.is_some()
|| p.escape.is_some()
|| p.null_value.is_some()
|| compression.is_some()
{
let mut o = CsvOptions::default();
if let Some(v) = p.has_header {
o = o.with_has_header(v);
}
if let Some(v) = p.delimiter {
o = o.with_delimiter(v as u8);
}
if let Some(v) = p.quote {
o = o.with_quote(v as u8);
}
if let Some(v) = p.escape {
o = o.with_escape(Some(v as u8));
}
if let Some(v) = p.null_value {
o.null_value = Some(v);
}
if let Some(v) = compression {
o = o.with_file_compression_type(v);
}
Some(o)
} else {
None
};
runtime().block_on(df.write_csv(&path, write_opts, writer_opts))?;
Ok(())
})
}
apache-datafusion-java-0.1.0/native/src/errors.rs 0000664 0000000 0000000 00000032555 15205164533 0021760 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::any::Any;
use std::error::Error;
use std::panic::{catch_unwind, AssertUnwindSafe};
use datafusion::arrow::error::ArrowError;
use datafusion::error::DataFusionError;
use jni::JNIEnv;
pub type JniResult = Result>;
/// Run `f`, catching panics and translating its `Err` into the appropriate
/// Java exception. When the boxed error is a [`DataFusionError`], the variant
/// drives which Java exception class is thrown -- callers can `catch
/// (PlanException)` / `catch (ResourcesExhaustedException)` etc. without
/// scraping message strings. Anything else (including Rust panics) falls
/// through to the parent [`DataFusionException`] class.
pub fn try_unwrap_or_throw(env: &mut JNIEnv, default: T, f: F) -> T
where
F: FnOnce(&mut JNIEnv) -> JniResult,
{
match catch_unwind(AssertUnwindSafe(|| f(env))) {
Ok(Ok(value)) => value,
Ok(Err(boxed)) => {
// Try to recover the typed DataFusionError so we can pick a
// matching Java subclass. If the error came from elsewhere
// (jni::errors::Error, prost::DecodeError, a stringly-typed
// Err("...")?), it falls through to the parent class.
match boxed.downcast::() {
Ok(df_err) => {
// Walk the cause chain so a ResourcesExhausted / Plan /
// etc. wrapped in ArrowError::External (or any other
// upstream-mandated wrapper) is classified by its real
// root, not by the outer wrapper. Same shape as upstream's
// `DataFusionError::find_root()`. The thrown message
// preserves the outer error's full chain so wrapping
// context isn't lost; only the *class* picks the inner
// variant.
let class = classify(df_err.find_root());
throw(env, class, &df_err.to_string());
}
Err(other) => {
throw(env, PARENT, &other.to_string());
}
}
default
}
Err(panic) => {
// Rust panics are upstream-bug-shaped, not caller-actionable.
// Surface as the parent class with a `panic: ` prefix so the
// caller can grep without inspecting a typed subclass.
throw(env, PARENT, &format!("panic: {}", panic_message(&panic)));
default
}
}
}
const PARENT: &str = "org/apache/datafusion/DataFusionException";
/// Map a [`DataFusionError`] variant onto the Java exception class to throw.
/// Multiple Rust variants fold into a single Java class when they're the same
/// kind of problem from the caller's standpoint -- the proposed mapping is
/// the public Java contract; the underlying variant set is not.
///
/// Callers should pass the result of [`DataFusionError::find_root`] so a
/// `ResourcesExhausted` / `Plan` / etc. buried inside a wrapping `ArrowError`
/// or `Context(...)` is classified by its real root. Variants without a
/// clean caller-facing category fall through to the parent class.
fn classify(err: &DataFusionError) -> &'static str {
match err {
DataFusionError::Plan(_)
| DataFusionError::SQL(_, _)
| DataFusionError::SchemaError(_, _) => "org/apache/datafusion/PlanException",
DataFusionError::Execution(_)
| DataFusionError::ExecutionJoin(_)
| DataFusionError::External(_)
| DataFusionError::Ffi(_) => "org/apache/datafusion/ExecutionException",
DataFusionError::ResourcesExhausted(_) => {
"org/apache/datafusion/ResourcesExhaustedException"
}
DataFusionError::IoError(_)
| DataFusionError::ObjectStore(_)
| DataFusionError::ParquetError(_)
| DataFusionError::AvroError(_) => "org/apache/datafusion/IoException",
// ArrowError is a 21-variant grab bag -- only some of those variants
// are actually IO-shaped. DivideByZero / ArithmeticOverflow / Compute
// / Cast / InvalidArgument / Memory etc. are execution-time failures
// produced by a normal query (e.g. SELECT 1/0), not IO. Routing them
// through IoException would hide ordinary execution errors from
// callers catching ExecutionException.
DataFusionError::ArrowError(arrow_err, _) => classify_arrow(arrow_err),
DataFusionError::NotImplemented(_) => "org/apache/datafusion/NotImplementedException",
DataFusionError::Configuration(_) => "org/apache/datafusion/ConfigurationException",
// Variants with no clean caller-facing category -- Internal (upstream
// bug-shaped), Substrait, Collection (multi-error), and any new
// variants a future DataFusion bump introduces -- fall through to
// the parent. The catch-all is deliberate: "I don't know what to do
// with this" surfaces as the parent class rather than a wrong typed
// subclass. Wrapper variants like Context / Diagnostic / Shared also
// land here when find_root() can't dig past them, which is rare
// because find_root walks `source()`.
_ => PARENT,
}
}
/// Map an [`ArrowError`] variant onto the Java exception class to throw.
/// Only the genuinely IO-shaped variants (`IoError`, `IpcError`) land on
/// `IoException`; everything else is execution-time and routes through
/// `ExecutionException`. Schema/parse-shaped variants route through
/// `PlanException` so a malformed IPC schema or a parse error surfaces as a
/// query problem rather than an execution failure.
///
/// Variants without a clean caller-facing category (`CDataInterface`, the
/// various overflow/index-overflow markers) fall through to the parent.
fn classify_arrow(err: &ArrowError) -> &'static str {
match err {
ArrowError::IoError(_, _) | ArrowError::IpcError(_) => "org/apache/datafusion/IoException",
ArrowError::SchemaError(_) | ArrowError::ParseError(_) => {
"org/apache/datafusion/PlanException"
}
ArrowError::DivideByZero
| ArrowError::ArithmeticOverflow(_)
| ArrowError::ComputeError(_)
| ArrowError::CastError(_)
| ArrowError::InvalidArgumentError(_)
| ArrowError::MemoryError(_)
| ArrowError::CsvError(_)
| ArrowError::JsonError(_)
| ArrowError::AvroError(_)
| ArrowError::ParquetError(_)
| ArrowError::ExternalError(_) => "org/apache/datafusion/ExecutionException",
ArrowError::NotYetImplemented(_) => "org/apache/datafusion/NotImplementedException",
// CDataInterface, DictionaryKeyOverflowError, RunEndIndexOverflowError,
// OffsetOverflowError, and any new variants in future Arrow bumps fall
// through to the parent -- "I don't know what to do with this" stays
// at the parent rather than getting routed to the wrong subclass.
_ => PARENT,
}
}
fn throw(env: &mut JNIEnv, class: &str, message: &str) {
if env.exception_check().unwrap_or(false) {
return;
}
let _ = env.throw_new(class, message);
}
fn panic_message(panic: &Box) -> String {
if let Some(s) = panic.downcast_ref::() {
s.clone()
} else if let Some(s) = panic.downcast_ref::<&str>() {
(*s).to_string()
} else {
"rust panic with non-string payload".to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
use datafusion::arrow::error::ArrowError;
#[test]
fn classify_unwrapped_variants_picks_typed_class() {
// Sanity check: the simple unwrapped cases land on the right classes.
assert_eq!(
classify(&DataFusionError::Plan("x".into())),
"org/apache/datafusion/PlanException"
);
assert_eq!(
classify(&DataFusionError::ResourcesExhausted("x".into())),
"org/apache/datafusion/ResourcesExhaustedException"
);
assert_eq!(
classify(&DataFusionError::Configuration("x".into())),
"org/apache/datafusion/ConfigurationException"
);
}
#[test]
fn classify_unrecognised_variant_falls_through_to_parent() {
// `Internal` has no clean caller-facing category; must surface as
// the parent class rather than getting routed to a wrong subclass.
assert_eq!(classify(&DataFusionError::Internal("x".into())), PARENT);
}
#[test]
fn classify_via_find_root_unwraps_nested_resources_exhausted() {
// Codex regression: a DataFusionError::ResourcesExhausted buried
// inside an ArrowError::ExternalError(Box)
// wrapper would otherwise land on the outer ArrowError arm and be
// classified as IoException -- the wrong typed exception. Confirm
// that calling classify on `find_root` recovers the inner variant
// (the same shape upstream's own find_root() doctest pins).
let inner = DataFusionError::ResourcesExhausted("memory pool full".into());
let contexted = DataFusionError::Context("aggregate stream".into(), Box::new(inner));
let arrow_wrapped = ArrowError::ExternalError(Box::new(contexted));
let outer = DataFusionError::ArrowError(Box::new(arrow_wrapped), None);
// Without find_root, the outer would route to IoException; with it,
// the real root (ResourcesExhausted) wins.
assert_eq!(
classify(outer.find_root()),
"org/apache/datafusion/ResourcesExhaustedException"
);
}
#[test]
fn classify_via_find_root_unwraps_plan_through_context() {
// A Plan error wrapped only in DataFusionError::Context still has to
// reach PlanException, not the parent class.
let inner = DataFusionError::Plan("table 't' not found".into());
let contexted = DataFusionError::Context("logical planning".into(), Box::new(inner));
assert_eq!(
classify(contexted.find_root()),
"org/apache/datafusion/PlanException"
);
}
#[test]
fn arrow_divide_by_zero_routes_to_execution_not_io() {
// Codex regression: DataFusionError::ArrowError(ArrowError::DivideByZero)
// is the actual shape produced by `SELECT 1/0`. The 21-variant
// ArrowError grab bag means routing the whole ArrowError arm to
// IoException would put ordinary divide-by-zero / overflow / cast
// errors on the wrong typed exception. classify_arrow inspects the
// inner variant.
let outer = DataFusionError::ArrowError(Box::new(ArrowError::DivideByZero), None);
assert_eq!(
classify(outer.find_root()),
"org/apache/datafusion/ExecutionException"
);
let outer = DataFusionError::ArrowError(
Box::new(ArrowError::ArithmeticOverflow("i32 overflow".into())),
None,
);
assert_eq!(
classify(outer.find_root()),
"org/apache/datafusion/ExecutionException"
);
let outer = DataFusionError::ArrowError(
Box::new(ArrowError::CastError("Int32 -> Date32".into())),
None,
);
assert_eq!(
classify(outer.find_root()),
"org/apache/datafusion/ExecutionException"
);
}
#[test]
fn arrow_io_variants_still_route_to_io() {
// Genuinely IO-shaped ArrowError variants stay on IoException.
let io = std::io::Error::other("disk full");
let outer = DataFusionError::ArrowError(
Box::new(ArrowError::IoError("write failed".into(), io)),
None,
);
assert_eq!(
classify(outer.find_root()),
"org/apache/datafusion/IoException"
);
let outer =
DataFusionError::ArrowError(Box::new(ArrowError::IpcError("bad framing".into())), None);
assert_eq!(
classify(outer.find_root()),
"org/apache/datafusion/IoException"
);
}
#[test]
fn arrow_schema_and_parse_errors_route_to_plan() {
// SchemaError and ParseError are query-shaped problems (you wrote a
// bad query / supplied a bad schema), not execution failures.
let outer = DataFusionError::ArrowError(
Box::new(ArrowError::SchemaError("column 'foo' not found".into())),
None,
);
assert_eq!(
classify(outer.find_root()),
"org/apache/datafusion/PlanException"
);
let outer = DataFusionError::ArrowError(
Box::new(ArrowError::ParseError("expected int".into())),
None,
);
assert_eq!(
classify(outer.find_root()),
"org/apache/datafusion/PlanException"
);
}
}
apache-datafusion-java-0.1.0/native/src/jni_util.rs 0000664 0000000 0000000 00000005024 15205164533 0022250 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Small shared helpers for JNI call sites.
use jni::objects::JThrowable;
use jni::JNIEnv;
/// Best-effort: extract class name and `getMessage()` from a Java throwable.
/// Anything that goes wrong collapses to a generic message so we don't
/// double-throw inside an error path.
///
/// `kind` and `name` are used to build the surfaced error message
/// (e.g., `kind="UDF" name="add_one"` -> `"Java UDF 'add_one' threw ..."`).
pub(crate) fn jthrowable_to_string(
env: &mut JNIEnv,
throwable: &JThrowable,
kind: &str,
name: &str,
) -> String {
let class_name_result = (|| -> jni::errors::Result {
let class = env.call_method(throwable, "getClass", "()Ljava/lang/Class;", &[])?;
let class_obj = class.l()?;
let n = env.call_method(&class_obj, "getName", "()Ljava/lang/String;", &[])?;
let n_obj = n.l()?;
let n_str: String = env.get_string(&n_obj.into())?.into();
Ok(n_str)
})();
let class_name = match class_name_result {
Ok(s) => s,
Err(_) => {
env.exception_clear().ok();
"".to_string()
}
};
let message_result = (|| -> jni::errors::Result {
let msg = env.call_method(throwable, "getMessage", "()Ljava/lang/String;", &[])?;
let msg_obj = msg.l()?;
if msg_obj.is_null() {
return Ok("".to_string());
}
let s: String = env.get_string(&msg_obj.into())?.into();
Ok(s)
})();
let message = match message_result {
Ok(s) => s,
Err(_) => {
env.exception_clear().ok();
"".to_string()
}
};
format!("Java {} '{}' threw {}: {}", kind, name, class_name, message)
}
apache-datafusion-java-0.1.0/native/src/json.rs 0000664 0000000 0000000 00000016621 15205164533 0021411 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use datafusion::common::config::JsonOptions;
use datafusion::common::parsers::CompressionTypeVariant;
use datafusion::dataframe::{DataFrame, DataFrameWriteOptions};
use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
use datafusion::error::DataFusionError;
use datafusion::prelude::JsonReadOptions;
use datafusion::prelude::SessionContext;
use jni::objects::{JByteArray, JClass, JString};
use jni::sys::jlong;
use jni::JNIEnv;
use prost::Message;
use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::proto_gen::{
FileCompressionType as ProtoFileCompressionType, JsonWriteOptionsProto, NdJsonReadOptionsProto,
};
use crate::runtime;
use crate::schema::decode_optional_schema;
fn with_json_options(
env: &mut JNIEnv,
options_bytes: JByteArray,
schema_ipc_bytes: JByteArray,
f: impl FnOnce(JsonReadOptions) -> JniResult,
) -> JniResult {
let bytes: Vec = env.convert_byte_array(&options_bytes)?;
let p = NdJsonReadOptionsProto::decode(bytes.as_slice())?;
let schema = decode_optional_schema(env, schema_ipc_bytes)?;
let compression = match p.file_compression_type() {
ProtoFileCompressionType::Unspecified => {
return Err("NdJsonReadOptionsProto.file_compression_type is UNSPECIFIED".into());
}
ProtoFileCompressionType::Uncompressed => FileCompressionType::UNCOMPRESSED,
ProtoFileCompressionType::Gzip => FileCompressionType::GZIP,
ProtoFileCompressionType::Bzip2 => FileCompressionType::BZIP2,
ProtoFileCompressionType::Xz => FileCompressionType::XZ,
ProtoFileCompressionType::Zstd => FileCompressionType::ZSTD,
};
let file_ext = p.file_extension;
let mut opts = JsonReadOptions::default()
.file_extension(&file_ext)
.file_compression_type(compression);
if let Some(n) = p.schema_infer_max_records {
opts = opts.schema_infer_max_records(n as usize);
}
if let Some(ref s) = schema {
opts = opts.schema(s);
}
f(opts)
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerJsonWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
name: JString<'local>,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let name: String = env.get_string(&name)?.into();
let path: String = env.get_string(&path)?.into();
with_json_options(env, options_bytes, schema_ipc_bytes, |opts| {
runtime().block_on(async {
ctx.register_json(&name, &path, opts).await?;
Ok::<(), DataFusionError>(())
})?;
Ok(())
})
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readJsonWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let path: String = env.get_string(&path)?.into();
with_json_options(env, options_bytes, schema_ipc_bytes, |opts| {
let df = runtime().block_on(ctx.read_json(path, opts))?;
Ok(Box::into_raw(Box::new(df)) as jlong)
})
})
}
fn proto_compression_to_variant(p: ProtoFileCompressionType) -> JniResult {
match p {
ProtoFileCompressionType::Unspecified => {
Err("JsonWriteOptionsProto.file_compression_type is UNSPECIFIED".into())
}
ProtoFileCompressionType::Uncompressed => Ok(CompressionTypeVariant::UNCOMPRESSED),
ProtoFileCompressionType::Gzip => Ok(CompressionTypeVariant::GZIP),
ProtoFileCompressionType::Bzip2 => Ok(CompressionTypeVariant::BZIP2),
ProtoFileCompressionType::Xz => Ok(CompressionTypeVariant::XZ),
ProtoFileCompressionType::Zstd => Ok(CompressionTypeVariant::ZSTD),
}
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_writeJsonWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
path: JString<'local>,
options_bytes: JByteArray<'local>,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let path: String = env.get_string(&path)?.into();
let bytes: Vec = env.convert_byte_array(&options_bytes)?;
let p = JsonWriteOptionsProto::decode(bytes.as_slice())?;
// Decode the file_compression_type field eagerly so an unknown wire
// value surfaces as a clear error rather than a silent default.
let compression = if p.file_compression_type.is_some() {
Some(proto_compression_to_variant(p.file_compression_type())?)
} else {
None
};
// When the caller left `singleFileOutput` unset, force directory output (`false`)
// rather than leaving DataFusion in `Automatic` mode. Automatic mode treats paths
// with an extension (e.g. `out.json`) as single-file targets, which would silently
// contradict the documented "directory unless overridden" default and surprise any
// caller that hands writeJson a `.json` path.
let mut write_opts = DataFrameWriteOptions::new()
.with_single_file_output(p.single_file_output.unwrap_or(false));
if !p.partition_cols.is_empty() {
write_opts = write_opts.with_partition_by(p.partition_cols.clone());
}
// Build JsonOptions only when a writer-side knob is set, so the
// DataFusion default is preserved when the caller passes
// `new JsonWriteOptions()`. JsonOptions has no fluent setters --
// the fields are public, so use struct-update syntax (same
// idiom we use for ArrowReadOptions / AvroReadOptions).
let writer_opts: Option = compression.map(|c| JsonOptions {
compression: c,
..JsonOptions::default()
});
runtime().block_on(df.write_json(&path, write_opts, writer_opts))?;
Ok(())
})
}
apache-datafusion-java-0.1.0/native/src/lib.rs 0000664 0000000 0000000 00000131727 15205164533 0021213 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
mod arrow;
mod avro;
mod cache_manager;
mod csv;
mod errors;
mod jni_util;
mod json;
mod memory;
mod object_store;
mod proto;
mod runtime_metrics;
mod schema;
mod table_provider;
mod udf;
pub(crate) mod proto_gen {
include!(concat!(env!("OUT_DIR"), "/datafusion_java.rs"));
}
use std::panic::{catch_unwind, AssertUnwindSafe};
use std::path::PathBuf;
use std::sync::{Arc, OnceLock};
use datafusion::arrow::array::RecordBatch;
use datafusion::arrow::datatypes::SchemaRef;
use datafusion::arrow::error::ArrowError;
use datafusion::arrow::ffi_stream::FFI_ArrowArrayStream;
use datafusion::arrow::ipc::writer::StreamWriter;
use datafusion::arrow::record_batch::{RecordBatchIterator, RecordBatchReader};
use datafusion::common::{JoinType, UnnestOptions};
use datafusion::config::TableParquetOptions;
use datafusion::dataframe::DataFrame;
use datafusion::dataframe::DataFrameWriteOptions;
use datafusion::error::DataFusionError;
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
use datafusion::execution::SendableRecordBatchStream;
use datafusion::logical_expr::Expr;
use datafusion::logical_expr::{col, Partitioning, ScalarUDF, Signature, SortExpr};
use datafusion::prelude::{ParquetReadOptions, SessionConfig, SessionContext};
use futures::StreamExt;
use jni::objects::{JBooleanArray, JByteArray, JClass, JObject, JObjectArray, JString};
use jni::sys::{jboolean, jbyte, jbyteArray, jint, jlong};
use jni::JNIEnv;
use jni::JavaVM;
use prost::Message;
use tokio::runtime::Runtime;
use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::proto_gen::ParquetReadOptionsProto;
use crate::proto_gen::SessionOptions;
use crate::schema::decode_optional_schema;
static JAVA_VM: OnceLock = OnceLock::new();
#[no_mangle]
pub extern "system" fn JNI_OnLoad(vm: JavaVM, _reserved: *mut std::ffi::c_void) -> jni::sys::jint {
let _ = JAVA_VM.set(vm);
jni::sys::JNI_VERSION_1_8
}
#[allow(dead_code)]
pub(crate) fn jvm() -> &'static JavaVM {
JAVA_VM
.get()
.expect("JNI_OnLoad has not been called; JavaVM unavailable")
}
pub(crate) fn runtime() -> &'static Runtime {
static RT: OnceLock = OnceLock::new();
RT.get_or_init(|| {
let rt = Runtime::new().expect("failed to create Tokio runtime");
// Eagerly install the runtime-metrics accumulator (no-op when the
// `runtime-metrics` Cargo feature is off). Initialising here -- not
// lazily on the first `runtimeStats()` call -- means the
// RuntimeMonitor's sampling baseline coincides with runtime start, so
// poll/park/busy totals reflect activity from the first query onward
// rather than from the first observation.
crate::runtime_metrics::init(rt.handle());
rt
})
}
/// Wrap the (already-built) `RuntimeEnvBuilder`'s memory pool with a
/// `TrackingMemoryPool` so `Java_..._memoryUsage` can report current/peak
/// bytes for this session. Registers the tracker in the process-wide map
/// keyed by the JNI handle. Idempotent: if the builder didn't set a pool,
/// fills in DataFusion's default first so the wrap-and-register has the
/// same behavior either way.
fn install_memory_tracker(
builder: &mut RuntimeEnvBuilder,
) -> std::sync::Arc {
use datafusion::execution::memory_pool::UnboundedMemoryPool;
let inner: std::sync::Arc = builder
.memory_pool
.take()
.unwrap_or_else(|| std::sync::Arc::new(UnboundedMemoryPool::default()));
let tracker = std::sync::Arc::new(crate::memory::TrackingMemoryPool::new(inner));
builder.memory_pool = Some(tracker.clone());
tracker
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_createSessionContext<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |_env| -> JniResult {
let mut runtime_builder = RuntimeEnvBuilder::new();
let tracker = install_memory_tracker(&mut runtime_builder);
let runtime_env = runtime_builder.build()?;
let ctx = SessionContext::new_with_config_rt(SessionConfig::new(), Arc::new(runtime_env));
let handle = Box::into_raw(Box::new(ctx)) as jlong;
crate::memory::register(handle, tracker);
Ok(handle)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_createSessionContextWithOptions<
'local,
>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
options_bytes: JByteArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
let bytes: Vec = env.convert_byte_array(&options_bytes)?;
let opts = SessionOptions::decode(bytes.as_slice())?;
let mut config = SessionConfig::new();
if let Some(v) = opts.batch_size {
config = config.with_batch_size(v as usize);
}
if let Some(v) = opts.target_partitions {
config = config.with_target_partitions(v as usize);
}
if let Some(v) = opts.collect_statistics {
config = config.with_collect_statistics(v);
}
if let Some(v) = opts.information_schema {
config = config.with_information_schema(v);
}
let mut runtime_builder = RuntimeEnvBuilder::new();
if let Some(mem) = opts.memory_limit {
runtime_builder = runtime_builder
.with_memory_limit(mem.max_memory_bytes as usize, mem.memory_fraction);
}
if let Some(dir) = opts.temp_directory {
runtime_builder = runtime_builder.with_temp_file_path(PathBuf::from(dir));
}
if let Some(cm_opts) = opts.cache_manager.as_ref() {
if let Some(cm_config) = crate::cache_manager::build_config(cm_opts)? {
runtime_builder = runtime_builder.with_cache_manager(cm_config);
}
}
// datafusion.runtime.* keys live on RuntimeEnv (separate object from
// SessionConfig) and round-tripping them through getOption/setOption
// has subtle correctness pitfalls (lazy default-tempdir creation,
// upstream's K/M/G integer truncation, OS-specific path separators).
// Reject them here with a clear error so callers fall back to the
// typed memoryLimit() / tempDirectory() setters until a follow-up
// PR designs the side-cache needed to support them safely.
//
// Iteration order matters: some upstream setters have side effects on
// other keys (e.g. `datafusion.optimizer.enable_dynamic_filter_pushdown`
// also rewrites the per-operator `enable_*_dynamic_filter_pushdown`
// flags), so the caller's last write must win. The proto field is
// `repeated ConfigOption` for this reason -- prost's default
// `map` decodes to a HashMap whose iteration order is
// randomized.
for opt in &opts.options {
if opt.key.starts_with("datafusion.runtime.") {
return Err(format!(
"datafusion.runtime.* keys are not supported via setOption yet; \
use SessionContextBuilder.memoryLimit() / .tempDirectory() instead. \
Got: {} = {}",
opt.key, opt.value
)
.into());
}
config.options_mut().set(&opt.key, &opt.value)?;
}
// Wrap the configured pool (default `UnboundedMemoryPool` or whatever
// `with_memory_limit` produced) with a tracker so `memoryUsage()` can
// report current/peak bytes for this session. The tracker is
// transparent to query execution -- it only intercepts grow/shrink to
// update two atomics.
let tracker = install_memory_tracker(&mut runtime_builder);
let runtime_env = runtime_builder.build()?;
let ctx = SessionContext::new_with_config_rt(config, Arc::new(runtime_env));
// Object-store registrations come last because they need a built
// RuntimeEnv to register against. A failure here drops `ctx` (and its
// Arc) on the floor and surfaces as a Java RuntimeException
// via try_unwrap_or_throw.
crate::object_store::apply_registrations(&ctx, &opts.object_stores)?;
let handle = Box::into_raw(Box::new(ctx)) as jlong;
crate::memory::register(handle, tracker);
Ok(handle)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_createDataFrame<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
sql: JString<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let sql_str: String = env.get_string(&sql)?.into();
let df = runtime().block_on(async { ctx.sql(&sql_str).await })?;
Ok(Box::into_raw(Box::new(df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_collectDataFrame<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
ffi_stream_addr: jlong,
) {
try_unwrap_or_throw(&mut env, (), |_env| -> JniResult<()> {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
if ffi_stream_addr == 0 {
return Err("ffi stream address is null".into());
}
let df = unsafe { *Box::from_raw(handle as *mut DataFrame) };
let ffi: FFI_ArrowArrayStream = runtime().block_on(async {
let schema: SchemaRef = Arc::new(df.schema().as_arrow().clone());
let batches = df.collect().await?;
let iter = RecordBatchIterator::new(batches.into_iter().map(Ok), schema);
Ok::<_, DataFusionError>(FFI_ArrowArrayStream::new(Box::new(iter)))
})?;
unsafe {
std::ptr::write(ffi_stream_addr as *mut FFI_ArrowArrayStream, ffi);
}
Ok(())
})
}
/// Bridges DataFusion's async [`SendableRecordBatchStream`] to the synchronous
/// [`RecordBatchReader`] interface that `FFI_ArrowArrayStream` (and therefore
/// the Java `ArrowReader`) consumes. Each call to `next()` drives one
/// `runtime().block_on(stream.next())`, so memory pressure stays bounded by the
/// executor pipeline plus a single in-flight batch.
struct StreamingReader {
schema: SchemaRef,
stream: SendableRecordBatchStream,
}
impl Iterator for StreamingReader {
type Item = Result;
fn next(&mut self) -> Option {
// Arrow's C ABI invokes this iterator through FFI_ArrowArrayStream's
// vtable, outside the JNI handler's try_unwrap_or_throw guard. A panic
// here (buggy UDF, arrow cast that panics, runtime poison) would
// unwind across C/FFI -- undefined behaviour. Catch it and surface as
// an ArrowError so the Java side sees a normal exception instead.
let next = catch_unwind(AssertUnwindSafe(|| runtime().block_on(self.stream.next())));
match next {
Ok(item) => item.map(|r| r.map_err(|e| ArrowError::ExternalError(Box::new(e)))),
Err(panic) => {
let msg = if let Some(s) = panic.downcast_ref::() {
s.clone()
} else if let Some(s) = panic.downcast_ref::<&str>() {
(*s).to_string()
} else {
"rust panic with non-string payload".to_string()
};
Some(Err(ArrowError::ExternalError(
format!("panic in DataFrame stream: {msg}").into(),
)))
}
}
}
}
impl RecordBatchReader for StreamingReader {
fn schema(&self) -> SchemaRef {
self.schema.clone()
}
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_executeStreamDataFrame<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
ffi_stream_addr: jlong,
) {
try_unwrap_or_throw(&mut env, (), |_env| -> JniResult<()> {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
if ffi_stream_addr == 0 {
return Err("ffi stream address is null".into());
}
let df = unsafe { *Box::from_raw(handle as *mut DataFrame) };
let ffi: FFI_ArrowArrayStream = runtime().block_on(async {
let schema: SchemaRef = Arc::new(df.schema().as_arrow().clone());
let stream = df.execute_stream().await?;
let reader = StreamingReader { schema, stream };
Ok::<_, DataFusionError>(FFI_ArrowArrayStream::new(Box::new(reader)))
})?;
unsafe {
std::ptr::write(ffi_stream_addr as *mut FFI_ArrowArrayStream, ffi);
}
Ok(())
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_countRows<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |_env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let n = runtime().block_on(async { df.count().await })?;
Ok(n as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_schemaIpc<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) -> jbyteArray {
try_unwrap_or_throw(
&mut env,
std::ptr::null_mut(),
|env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) };
let schema: SchemaRef = Arc::new(df.schema().as_arrow().clone());
let mut buf: Vec = Vec::new();
{
let mut writer = StreamWriter::try_new(&mut buf, schema.as_ref())?;
writer.finish()?;
}
let arr = env.byte_array_from_slice(&buf)?;
Ok(arr.into_raw())
},
)
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_explainPlan<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
verbose: jboolean,
analyze: jboolean,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |_env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let new_df = df.explain(verbose != 0, analyze != 0)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_cachePlan<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |_env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let new_df = runtime().block_on(async { df.cache().await })?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_describePlan<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |_env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let new_df = runtime().block_on(async { df.describe().await })?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_showDataFrame<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) {
try_unwrap_or_throw(&mut env, (), |_env| -> JniResult<()> {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
runtime().block_on(async { df.show().await })?;
Ok(())
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_showDataFrameWithLimit<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
limit: jint,
) {
try_unwrap_or_throw(&mut env, (), |_env| -> JniResult<()> {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
runtime().block_on(async { df.show_limit(limit as usize).await })?;
Ok(())
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_selectColumns<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
column_names: JObjectArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let len = env.get_array_length(&column_names)?;
let mut owned: Vec = Vec::with_capacity(len as usize);
for i in 0..len {
let elem = env.get_object_array_element(&column_names, i)?;
let jstr: JString = elem.into();
owned.push(env.get_string(&jstr)?.into());
}
let refs: Vec<&str> = owned.iter().map(String::as_str).collect();
let new_df = df.select_columns(&refs)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_filterRows<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
predicate: JString<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let predicate: String = env.get_string(&predicate)?.into();
let expr = df.parse_sql_expr(&predicate)?;
let new_df = df.filter(expr)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_limitRows<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
skip: jint,
fetch: jint,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |_env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let new_df = df.limit(skip as usize, Some(fetch as usize))?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_distinctRows<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |_env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let new_df = df.distinct()?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_dropColumns<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
column_names: JObjectArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let len = env.get_array_length(&column_names)?;
let mut owned: Vec = Vec::with_capacity(len as usize);
for i in 0..len {
let elem = env.get_object_array_element(&column_names, i)?;
let jstr: JString = elem.into();
owned.push(env.get_string(&jstr)?.into());
}
let refs: Vec<&str> = owned.iter().map(String::as_str).collect();
let new_df = df.drop_columns(&refs)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_renameColumn<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
old_name: JString<'local>,
new_name: JString<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let old: String = env.get_string(&old_name)?.into();
let new: String = env.get_string(&new_name)?.into();
let new_df = df.with_column_renamed(&old, &new)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_withColumnExpr<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
name: JString<'local>,
expr: JString<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let name: String = env.get_string(&name)?.into();
let expr: String = env.get_string(&expr)?.into();
let parsed = df.parse_sql_expr(&expr)?;
let new_df = df.with_column(&name, parsed)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_unnestColumns<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
columns: JObjectArray<'local>,
preserve_nulls: jboolean,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let len = env.get_array_length(&columns)?;
let mut owned: Vec = Vec::with_capacity(len as usize);
for i in 0..len {
let elem = env.get_object_array_element(&columns, i)?;
let jstr: JString = elem.into();
owned.push(env.get_string(&jstr)?.into());
}
let refs: Vec<&str> = owned.iter().map(String::as_str).collect();
let opts = UnnestOptions::new().with_preserve_nulls(preserve_nulls != 0);
let new_df = df.unnest_columns_with_options(&refs, opts)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
/// Map a Java {@code JoinType.code()} byte back to upstream's enum.
fn join_type_from_byte(byte: u8) -> JniResult {
match byte {
0 => Ok(JoinType::Inner),
1 => Ok(JoinType::Left),
2 => Ok(JoinType::Right),
3 => Ok(JoinType::Full),
4 => Ok(JoinType::LeftSemi),
5 => Ok(JoinType::RightSemi),
6 => Ok(JoinType::LeftAnti),
7 => Ok(JoinType::RightAnti),
8 => Ok(JoinType::LeftMark),
9 => Ok(JoinType::RightMark),
other => Err(format!("unknown join type byte: {other}").into()),
}
}
/// Build a combined DFSchema for SQL parsing of a join filter or `joinOn` predicate.
/// Mirrors how upstream's `LogicalPlanBuilder::join_detailed` normalises the parsed Expr
/// against `&[&[left_schema, right_schema]]`: tolerate unrelated duplicate-named columns
/// rather than rejecting them via `DFSchema::join`'s `check_names`. `DFSchema::merge`
/// skips duplicates (left side wins for unqualified collisions), which is fine for the
/// SQL-to-Expr step -- the subsequent join planner runs the real ambiguity check.
fn combine_schemas(
left: &datafusion::common::DFSchema,
right: &datafusion::common::DFSchema,
) -> datafusion::common::DFSchema {
let mut combined = left.clone();
combined.merge(right);
combined
}
/// Drain a Java {@code String[]} into an owned {@code Vec}.
fn collect_jstring_array(env: &mut JNIEnv, arr: &JObjectArray) -> JniResult> {
let len = env.get_array_length(arr)?;
let mut owned: Vec = Vec::with_capacity(len as usize);
for i in 0..len {
let elem = env.get_object_array_element(arr, i)?;
let jstr: JString = elem.into();
owned.push(env.get_string(&jstr)?.into());
}
Ok(owned)
}
#[no_mangle]
#[allow(clippy::too_many_arguments)]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_joinDataFrame<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
left_handle: jlong,
right_handle: jlong,
join_type: jbyte,
left_cols: JObjectArray<'local>,
right_cols: JObjectArray<'local>,
filter: JString<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if left_handle == 0 {
return Err("left DataFrame handle is null".into());
}
if right_handle == 0 {
return Err("right DataFrame handle is null".into());
}
let left = unsafe { &*(left_handle as *const DataFrame) }.clone();
let right = unsafe { &*(right_handle as *const DataFrame) }.clone();
let join_type = join_type_from_byte(join_type as u8)?;
let left_owned: Vec = collect_jstring_array(env, &left_cols)?;
let right_owned: Vec = collect_jstring_array(env, &right_cols)?;
let left_refs: Vec<&str> = left_owned.iter().map(String::as_str).collect();
let right_refs: Vec<&str> = right_owned.iter().map(String::as_str).collect();
// The optional residual filter spans both sides and must be parsed against the
// combined schema. parse_sql_expr only sees one DataFrame's schema, so reach into
// the SessionState via into_parts() on a clone. Use DFSchema::merge rather than
// DFSchema::join so the parser tolerates unrelated duplicate unqualified columns
// shared by both sides (e.g. both inputs carrying a `created_at` field) -- merge
// skips the duplicates while join's check_names rejects them. Upstream's join
// path normalises the parsed Expr against both schemas as a precedence list, so
// ambiguous references genuinely used in the filter are still surfaced after
// parsing.
let filter_expr: Option = if filter.is_null() {
None
} else {
let filter_sql: String = env.get_string(&filter)?.into();
let combined = combine_schemas(left.schema(), right.schema());
let (state, _plan) = left.clone().into_parts();
Some(state.create_logical_expr(&filter_sql, &combined)?)
};
let new_df = left.join(right, join_type, &left_refs, &right_refs, filter_expr)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_joinOnDataFrame<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
left_handle: jlong,
right_handle: jlong,
join_type: jbyte,
predicates: JObjectArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if left_handle == 0 {
return Err("left DataFrame handle is null".into());
}
if right_handle == 0 {
return Err("right DataFrame handle is null".into());
}
let left = unsafe { &*(left_handle as *const DataFrame) }.clone();
let right = unsafe { &*(right_handle as *const DataFrame) }.clone();
let join_type = join_type_from_byte(join_type as u8)?;
let predicates_owned: Vec = collect_jstring_array(env, &predicates)?;
// See joinDataFrame for the rationale behind combine_schemas vs DFSchema::join.
let combined = combine_schemas(left.schema(), right.schema());
let (state, _plan) = left.clone().into_parts();
let exprs: Vec = predicates_owned
.iter()
.map(|sql| state.create_logical_expr(sql, &combined))
.collect::>>()?;
let new_df = left.join_on(right, join_type, exprs)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_sortRows<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
columns: JObjectArray<'local>,
ascending: JBooleanArray<'local>,
nulls_first: JBooleanArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let len = env.get_array_length(&columns)? as usize;
let mut names: Vec = Vec::with_capacity(len);
for i in 0..len {
let elem = env.get_object_array_element(&columns, i as i32)?;
let jstr: JString = elem.into();
names.push(env.get_string(&jstr)?.into());
}
// Decode the two parallel boolean arrays via primitive-array region copies.
let mut asc_buf = vec![0u8; len];
env.get_boolean_array_region(&ascending, 0, &mut asc_buf)?;
let mut nf_buf = vec![0u8; len];
env.get_boolean_array_region(&nulls_first, 0, &mut nf_buf)?;
let sort_exprs: Vec = names
.into_iter()
.zip(asc_buf.into_iter().zip(nf_buf))
.map(|(name, (asc, nf))| SortExpr::new(col(&name), asc != 0, nf != 0))
.collect();
let new_df = df.sort(sort_exprs)?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_repartitionRoundRobinRows<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
num_partitions: jint,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |_env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let new_df = df.repartition(Partitioning::RoundRobinBatch(num_partitions as usize))?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_repartitionHashRows<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
num_partitions: jint,
columns: JObjectArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let len = env.get_array_length(&columns)?;
let mut owned: Vec = Vec::with_capacity(len as usize);
for i in 0..len {
let elem = env.get_object_array_element(&columns, i)?;
let jstr: JString = elem.into();
owned.push(env.get_string(&jstr)?.into());
}
let exprs = owned.iter().map(|s| col(s.as_str())).collect();
let new_df = df.repartition(Partitioning::Hash(exprs, num_partitions as usize))?;
Ok(Box::into_raw(Box::new(new_df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_writeParquetWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
path: JString<'local>,
compression: JString<'local>,
single_file_output_set: jboolean,
single_file_output_value: jboolean,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("DataFrame handle is null".into());
}
let df = unsafe { &*(handle as *const DataFrame) }.clone();
let path: String = env.get_string(&path)?.into();
// When the caller left `singleFileOutput` unset, force directory output (`false`)
// rather than leaving DataFusion in `Automatic` mode. Automatic mode treats paths
// with an extension (e.g. `out.parquet`) as single-file targets, which would silently
// contradict the documented "directory unless overridden" default and surprise any
// caller that hands writeParquet a `.parquet` path.
let single_file = if single_file_output_set != 0 {
single_file_output_value != 0
} else {
false
};
let write_opts = DataFrameWriteOptions::new().with_single_file_output(single_file);
let writer_opts: Option = if !compression.is_null() {
let c: String = env.get_string(&compression)?.into();
let mut tpo = TableParquetOptions::default();
tpo.global.compression = Some(c);
Some(tpo)
} else {
None
};
runtime().block_on(df.write_parquet(&path, write_opts, writer_opts))?;
Ok(())
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_DataFrame_closeDataFrame<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) {
try_unwrap_or_throw(&mut env, (), |_env| -> JniResult<()> {
if handle != 0 {
unsafe {
drop(Box::from_raw(handle as *mut DataFrame));
}
}
Ok(())
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_getOptionNative<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
key: JString<'local>,
) -> jni::sys::jstring {
try_unwrap_or_throw(
&mut env,
std::ptr::null_mut(),
|env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let key: String = env.get_string(&key)?.into();
// datafusion.runtime.* keys live on RuntimeEnv and are not yet
// supported via this getter (see the matching restriction in
// createSessionContextWithOptions). Reject them with a clear
// pointer to the typed alternatives.
if key.starts_with("datafusion.runtime.") {
return Err(format!(
"datafusion.runtime.* keys are not supported via getOption yet; \
use SessionContextBuilder typed setters instead. Got: {key}"
)
.into());
}
let config = ctx.copied_config();
for entry in config.options().entries() {
if entry.key == key {
return match entry.value {
Some(v) => Ok(env.new_string(v)?.into_raw()),
None => Ok(std::ptr::null_mut()),
};
}
}
Err(format!("unknown DataFusion config key: {key}").into())
},
)
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_closeSessionContext<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) {
try_unwrap_or_throw(&mut env, (), |_env| -> JniResult<()> {
if handle != 0 {
// Drop our side-table entry first so the tracker Arc's last
// strong reference is the SessionContext->RuntimeEnv chain, then
// drop the SessionContext itself.
crate::memory::unregister(handle);
unsafe {
drop(Box::from_raw(handle as *mut SessionContext));
}
}
Ok(())
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_memoryUsageNative<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) -> jni::sys::jlongArray {
try_unwrap_or_throw(
&mut env,
std::ptr::null_mut(),
|env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
// Look up the tracker by JNI handle. Should always succeed because
// the ctor inserts before publishing the handle to Java.
let tracker = crate::memory::lookup(handle)
.ok_or("memory tracker not registered for this SessionContext")?;
// `snapshot()` returns a consistent (current, peak) pair where
// peak is always >= current even if a concurrent record_grow is
// in-flight. Callers see no transient `current > peak`.
let (current, peak) = tracker.snapshot();
let values: [jlong; 2] = [current as jlong, peak as jlong];
let arr = env.new_long_array(values.len() as jni::sys::jsize)?;
env.set_long_array_region(&arr, 0, &values)?;
Ok(arr.into_raw())
},
)
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_runtimeStatsNative<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
) -> jni::sys::jlongArray {
try_unwrap_or_throw(
&mut env,
std::ptr::null_mut(),
|env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
// Runtime stats are process-global -- the JNI library drives one
// shared multi-threaded runtime -- but we still gate on the
// SessionContext handle so callers can't ask after close().
let stats = crate::runtime_metrics::runtime_stats()?;
let arr = env.new_long_array(stats.len() as jni::sys::jsize)?;
env.set_long_array_region(&arr, 0, &stats)?;
Ok(arr.into_raw())
},
)
}
fn with_parquet_options(
env: &mut JNIEnv,
options_bytes: JByteArray,
schema_ipc_bytes: JByteArray,
f: impl FnOnce(ParquetReadOptions) -> JniResult,
) -> JniResult {
let bytes: Vec = env.convert_byte_array(&options_bytes)?;
let p = ParquetReadOptionsProto::decode(bytes.as_slice())?;
let schema = decode_optional_schema(env, schema_ipc_bytes)?;
let file_ext = p.file_extension;
let mut opts = ParquetReadOptions::default().file_extension(&file_ext);
if let Some(v) = p.parquet_pruning {
opts = opts.parquet_pruning(v);
}
if let Some(v) = p.skip_metadata {
opts = opts.skip_metadata(v);
}
if let Some(v) = p.metadata_size_hint {
opts = opts.metadata_size_hint(Some(v as usize));
}
if let Some(ref s) = schema {
opts = opts.schema(s);
}
f(opts)
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerParquetWithOptions<
'local,
>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
name: JString<'local>,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let name: String = env.get_string(&name)?.into();
let path: String = env.get_string(&path)?.into();
with_parquet_options(env, options_bytes, schema_ipc_bytes, |opts| {
runtime().block_on(async {
ctx.register_parquet(&name, &path, opts).await?;
Ok::<(), DataFusionError>(())
})?;
Ok(())
})
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_readParquetWithOptions<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
path: JString<'local>,
options_bytes: JByteArray<'local>,
schema_ipc_bytes: JByteArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let path: String = env.get_string(&path)?.into();
with_parquet_options(env, options_bytes, schema_ipc_bytes, |opts| {
let df = runtime().block_on(ctx.read_parquet(path, opts))?;
Ok(Box::into_raw(Box::new(df)) as jlong)
})
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerScalarUdf<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
name: JString<'local>,
signature_schema_bytes: JByteArray<'local>,
volatility: jni::sys::jbyte,
udf: JObject<'local>,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
// SAFETY: handle is a valid Box allocated by createSessionContext.
let ctx = unsafe { &*(handle as *const SessionContext) };
let name: String = env.get_string(&name)?.into();
// Decode the signature schema (field 0 = return type, fields 1..N = arg types).
let signature_schema = crate::schema::decode_optional_schema(env, signature_schema_bytes)?
.ok_or("signature schema bytes were null")?;
let fields = signature_schema.fields();
if fields.is_empty() {
return Err("signature schema must have at least a return-type field".into());
}
let return_field = fields[0].clone();
let arg_types: Vec = fields
.iter()
.skip(1)
.map(|f| f.data_type().clone())
.collect();
let volatility = crate::udf::volatility_from_byte(volatility as u8)?;
let signature = Signature::exact(arg_types, volatility);
// Hold references that survive the JNI call.
let udf_global_ref = env.new_global_ref(&udf)?;
let bridge_class_local = env.find_class("org/apache/datafusion/internal/JniBridge")?;
let bridge_class = env.new_global_ref(&bridge_class_local)?;
let invoke_method = env.get_static_method_id(
&bridge_class_local,
"invokeScalarUdf",
"(Lorg/apache/datafusion/ScalarFunction;JJJJ[BJJI)B",
)?;
let java_udf = crate::udf::JavaScalarUdf {
name: name.clone(),
signature,
return_field,
udf_global_ref,
bridge_class,
invoke_method,
};
ctx.register_udf(ScalarUDF::new_from_impl(java_udf));
Ok(())
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_registerTableNative<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
name: JString<'local>,
schema_ipc_bytes: JByteArray<'local>,
provider: JObject<'local>,
) {
try_unwrap_or_throw(&mut env, (), |env| -> JniResult<()> {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
// SAFETY: handle is a valid Box allocated by createSessionContext.
let ctx = unsafe { &*(handle as *const SessionContext) };
let name: String = env.get_string(&name)?.into();
let schema = crate::schema::decode_optional_schema(env, schema_ipc_bytes)?
.ok_or("schema bytes were null")?;
let schema = Arc::new(schema);
let source_global_ref = Arc::new(env.new_global_ref(&provider)?);
let bridge_class_local = env.find_class("org/apache/datafusion/internal/JniBridge")?;
let bridge_class = Arc::new(env.new_global_ref(&bridge_class_local)?);
let invoke_method = env.get_static_method_id(
&bridge_class_local,
"invokeTableScan",
"(Lorg/apache/datafusion/TableProvider;J)V",
)?;
let java_tp = crate::table_provider::JavaTableProvider {
name: name.clone(),
schema,
source_global_ref,
bridge_class,
invoke_method,
};
let _ = ctx.register_table(name.as_str(), Arc::new(java_tp))?;
Ok(())
})
}
apache-datafusion-java-0.1.0/native/src/memory.rs 0000664 0000000 0000000 00000017460 15205164533 0021752 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Session-wide memory accounting wrapper around any `MemoryPool`.
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use datafusion::common::Result;
use datafusion::execution::memory_pool::{MemoryLimit, MemoryPool, MemoryReservation};
use jni::sys::jlong;
/// Wraps an inner `MemoryPool` with two counters: total bytes currently held
/// and the peak observed since the wrapper was constructed. Bookkeeping is two
/// atomic loads/stores per `grow`/`shrink` and adds no contention because the
/// inner pool already serializes its own state.
#[derive(Debug)]
pub struct TrackingMemoryPool {
inner: Arc,
current_bytes: AtomicU64,
peak_bytes: AtomicU64,
}
impl TrackingMemoryPool {
pub fn new(inner: Arc) -> Self {
Self {
inner,
current_bytes: AtomicU64::new(0),
peak_bytes: AtomicU64::new(0),
}
}
// Note: there are intentionally no separate `current_bytes()` /
// `peak_bytes()` accessors. Reading them independently allows a torn
// observation where `current > peak` while a concurrent `record_grow` is
// mid-flight. Callers must use `snapshot()` instead, which clamps.
/// Returns `(current_bytes, peak_bytes)` as a consistent snapshot. Two
/// invariants are maintained:
///
/// 1. **Within a snapshot:** `peak >= current`. This matters when a
/// concurrent `record_grow` is in-flight between its `fetch_add` of
/// `current_bytes` and its `fetch_max` of `peak_bytes` -- a naive
/// snapshot could observe the higher current but the stale peak.
/// 2. **Across snapshots:** `peak` is monotonically non-decreasing for
/// every observer. If reader R1 ever sees peak = X, no later reader R2
/// (on any thread) sees peak < X.
///
/// The read-side clamp `max(stored_peak, observed_current)` alone gives
/// (1) but not (2): it's possible for R1 to *synthesize* a peak of X
/// from a transiently-high current, and then for a `record_shrink` plus
/// the slow path of `record_grow`'s missing `fetch_max` to leave the
/// stored peak briefly below X while R2 polls. R2 would observe a
/// smaller current and a smaller stored peak -- non-monotonic.
///
/// The fix is to persist the clamped value back into `peak_bytes` via
/// `fetch_max` *before* returning. Doing so guarantees every snapshot's
/// returned peak is also reflected in the global atomic, so any
/// subsequent observer's `peak_bytes.load()` is at least as high as the
/// largest peak any earlier observer returned.
///
/// Both atomics are `Relaxed`-ordered; we tolerate slightly stale values
/// in exchange for cheaper writes.
pub fn snapshot(&self) -> (u64, u64) {
// Order matters on the load side too: read `peak` first, then
// `current`. If a `record_grow` is in flight, this order ensures
// that whenever we observe a fresh `current` we also observe at
// least the corresponding (or older) `peak` -- so the eventual
// clamp `max(peak, current)` produces the right value.
let peak_observed = self.peak_bytes.load(Ordering::Relaxed);
let current = self.current_bytes.load(Ordering::Relaxed);
let clamped_peak = peak_observed.max(current);
// Persist the clamped peak so future observers can never see a
// smaller peak than the one we just returned. fetch_max is
// monotonic, so this is a no-op when another thread (or this very
// thread on a previous call) already pushed peak past clamped_peak.
let stored_peak = self.peak_bytes.fetch_max(clamped_peak, Ordering::Relaxed);
(current, stored_peak.max(clamped_peak))
}
fn record_grow(&self, additional: usize) {
// We bump `current` first to avoid double-counting in `peak` if a
// concurrent reader sees a stale current and a fresh peak. The
// snapshot path's read-side `max(peak, current)` clamp closes the
// window where `current` is up-to-date but `peak` is not.
let now = self
.current_bytes
.fetch_add(additional as u64, Ordering::Relaxed)
.saturating_add(additional as u64);
// fetch_max is monotonic so concurrent grows still produce the right peak.
self.peak_bytes.fetch_max(now, Ordering::Relaxed);
}
fn record_shrink(&self, shrink: usize) {
// Saturating: if the inner pool ever calls shrink with a value larger
// than the running total we'd rather report 0 than wrap around.
let prev = self
.current_bytes
.fetch_sub(shrink as u64, Ordering::Relaxed);
if prev < shrink as u64 {
// Restore to zero. Concurrent grows that interleaved are fine --
// they'll just see a transient under-count.
self.current_bytes.store(0, Ordering::Relaxed);
}
}
}
impl MemoryPool for TrackingMemoryPool {
fn register(&self, consumer: &datafusion::execution::memory_pool::MemoryConsumer) {
self.inner.register(consumer);
}
fn unregister(&self, consumer: &datafusion::execution::memory_pool::MemoryConsumer) {
self.inner.unregister(consumer);
}
fn grow(&self, reservation: &MemoryReservation, additional: usize) {
self.inner.grow(reservation, additional);
self.record_grow(additional);
}
fn shrink(&self, reservation: &MemoryReservation, shrink: usize) {
self.inner.shrink(reservation, shrink);
self.record_shrink(shrink);
}
fn try_grow(&self, reservation: &MemoryReservation, additional: usize) -> Result<()> {
self.inner.try_grow(reservation, additional)?;
self.record_grow(additional);
Ok(())
}
fn reserved(&self) -> usize {
self.inner.reserved()
}
fn memory_limit(&self) -> MemoryLimit {
self.inner.memory_limit()
}
}
/// Process-wide registry mapping a `SessionContext`'s native handle to its
/// `TrackingMemoryPool`. Populated at session creation, drained at session
/// close. We can't downcast `Arc` back to the concrete type
/// without an `Any` bound (which the trait does not require), so instead we
/// keep a parallel `Arc` indexed by the JNI handle Java already threads
/// through every call.
fn registry() -> &'static Mutex>> {
static REGISTRY: OnceLock>>> = OnceLock::new();
REGISTRY.get_or_init(|| Mutex::new(HashMap::new()))
}
pub fn register(handle: jlong, pool: Arc) {
registry()
.lock()
.expect("memory registry lock poisoned")
.insert(handle, pool);
}
pub fn lookup(handle: jlong) -> Option> {
registry()
.lock()
.expect("memory registry lock poisoned")
.get(&handle)
.cloned()
}
pub fn unregister(handle: jlong) {
registry()
.lock()
.expect("memory registry lock poisoned")
.remove(&handle);
}
apache-datafusion-java-0.1.0/native/src/object_store.rs 0000664 0000000 0000000 00000030243 15205164533 0023116 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Apply [`ObjectStoreRegistration`] entries from the session-options proto to a
//! [`SessionContext`]'s [`RuntimeEnv`].
//!
//! Each backend (`s3` / `gcs` / `http`) is gated behind its own Cargo feature;
//! the corresponding arm of the proto `oneof` returns a clear error if the
//! feature was not enabled at build time. The default build of this crate
//! enables all three so Java callers don't have to think about features.
use std::sync::Arc;
use datafusion::prelude::SessionContext;
use url::Url;
use crate::errors::JniResult;
use crate::proto_gen::object_store_registration::Backend;
use crate::proto_gen::ObjectStoreRegistration;
#[cfg(feature = "object-store-gcp")]
use crate::proto_gen::GcsOptions;
#[cfg(feature = "object-store-http")]
use crate::proto_gen::HttpOptions;
#[cfg(feature = "object-store-aws")]
use crate::proto_gen::S3Options;
/// Apply every registration in `regs`, in order, to the context's `RuntimeEnv`.
/// If two registrations resolve to the same URL, the later one wins (matching
/// upstream `RuntimeEnv::register_object_store`).
pub(crate) fn apply_registrations(
ctx: &SessionContext,
regs: &[ObjectStoreRegistration],
) -> JniResult<()> {
for reg in regs {
let backend = reg
.backend
.as_ref()
.ok_or("ObjectStoreRegistration.backend is required")?;
let (url, store) = build_store(reg.url.as_deref(), backend)?;
ctx.runtime_env().register_object_store(&url, store);
}
Ok(())
}
#[allow(unused_variables)] // `url_override` is unused on builds with no features
fn build_store(
url_override: Option<&str>,
backend: &Backend,
) -> JniResult<(Url, Arc)> {
match backend {
#[cfg(feature = "object-store-aws")]
Backend::S3(opts) => build_s3(url_override, opts),
#[cfg(not(feature = "object-store-aws"))]
Backend::S3(_) => Err(
"object-store-aws Cargo feature is not enabled in this build of datafusion-jni".into(),
),
#[cfg(feature = "object-store-gcp")]
Backend::Gcs(opts) => build_gcs(url_override, opts),
#[cfg(not(feature = "object-store-gcp"))]
Backend::Gcs(_) => Err(
"object-store-gcp Cargo feature is not enabled in this build of datafusion-jni".into(),
),
#[cfg(feature = "object-store-http")]
Backend::Http(opts) => build_http(url_override, opts),
#[cfg(not(feature = "object-store-http"))]
Backend::Http(_) => Err(
"object-store-http Cargo feature is not enabled in this build of datafusion-jni".into(),
),
}
}
#[cfg(feature = "object-store-aws")]
fn build_s3(
url_override: Option<&str>,
opts: &S3Options,
) -> JniResult<(Url, Arc)> {
use object_store::aws::{AmazonS3Builder, AmazonS3ConfigKey};
if opts.bucket.is_empty() {
return Err("S3Options.bucket is required".into());
}
// Seed from the standard AWS env (AWS_ACCESS_KEY_ID, AWS_DEFAULT_REGION,
// AWS_WEB_IDENTITY_TOKEN_FILE, ECS task creds, etc.) before applying
// explicit Java overrides. Without this, callers running on EC2/ECS/EKS
// who omit Java-side credentials and expect the SDK default-credential
// chain would get an empty builder and fail at request time.
//
// Walk env explicitly (instead of AmazonS3Builder::from_env()) so we can
// skip endpoint keys when the Java caller set `endpoint(...)`.
// AmazonS3Builder::build() picks `s3_endpoint` over `endpoint`
// (object_store 0.13 aws/builder.rs:1209), so a stray
// AWS_ENDPOINT_URL_S3 in the JVM env would otherwise silently override
// the explicit Java endpoint -- breaking MinIO/R2/etc. registrations
// run out of a process with global AWS env vars.
let java_has_endpoint = opts.endpoint.is_some();
let mut b = AmazonS3Builder::default();
for (os_key, os_value) in std::env::vars_os() {
let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) else {
continue;
};
if !key.starts_with("AWS_") {
continue;
}
if java_has_endpoint
&& (key == "AWS_ENDPOINT" || key == "AWS_ENDPOINT_URL" || key == "AWS_ENDPOINT_URL_S3")
{
continue;
}
// Skip AWS_BUCKET / AWS_BUCKET_NAME: the Java caller's bucket is the
// identity of this registration (it's also the host portion of the
// registry-key URL we return). An env-supplied bucket would overwrite
// it here while the URL still says `s3://`, so a query
// for the Java bucket would be served by the env bucket. Apply the
// Java bucket explicitly *after* the loop so it can't be clobbered.
if key == "AWS_BUCKET" || key == "AWS_BUCKET_NAME" {
continue;
}
if let Ok(config_key) = key.to_ascii_lowercase().parse() {
b = b.with_config(config_key, value);
}
}
b = b.with_bucket_name(&opts.bucket);
if let Some(ref v) = opts.region {
b = b.with_region(v);
}
if let Some(ref v) = opts.endpoint {
b = b.with_endpoint(v);
}
if let Some(ref v) = opts.access_key_id {
b = b.with_access_key_id(v);
}
if let Some(ref v) = opts.secret_access_key {
b = b.with_secret_access_key(v);
}
if let Some(ref v) = opts.session_token {
b = b.with_token(v);
}
if let Some(v) = opts.allow_http {
b = b.with_allow_http(v);
}
if let Some(v) = opts.skip_signature {
b = b.with_skip_signature(v);
}
if let Some(v) = opts.imdsv1_fallback {
// AmazonS3Builder::with_imdsv1_fallback() only sets to `true` (no
// boolean arg). To honor explicit `imdsv1Fallback(false)` and
// override an env-seeded `AWS_IMDSV1_FALLBACK=true`, write through
// with_config(...) which always sets the field, both directions.
b = b.with_config(AmazonS3ConfigKey::ImdsV1Fallback, v.to_string());
}
let store = b.build()?;
let url = parse_url(url_override, format!("s3://{}", opts.bucket))?;
Ok((url, Arc::new(store)))
}
#[cfg(feature = "object-store-gcp")]
fn build_gcs(
url_override: Option<&str>,
opts: &GcsOptions,
) -> JniResult<(Url, Arc)> {
use object_store::gcp::GoogleCloudStorageBuilder;
if opts.bucket.is_empty() {
return Err("GcsOptions.bucket is required".into());
}
if opts.service_account_key.is_some() && opts.service_account_path.is_some() {
return Err(
"GcsOptions: service_account_key and service_account_path are mutually exclusive"
.into(),
);
}
// Seed from the standard GCS env (GOOGLE_BUCKET, GOOGLE_SERVICE_ACCOUNT,
// etc.) before applying explicit Java overrides.
//
// Walk env explicitly (instead of GoogleCloudStorageBuilder::from_env())
// so we can skip credential keys that conflict with Java-supplied
// credentials. GoogleCloudStorageBuilder::build() rejects the combo of
// service_account_path AND service_account_key with
// ServiceAccountPathAndKeyProvided (object_store 0.13 gcp/builder.rs:525)
// -- so if env has GOOGLE_SERVICE_ACCOUNT_KEY and Java sets
// serviceAccountPath(...), the build would fail. Skip env credential
// keys when Java provides any of the three credential fields.
let java_has_credential = opts.service_account_key.is_some()
|| opts.service_account_path.is_some()
|| opts.application_credentials.is_some();
let mut b = GoogleCloudStorageBuilder::default();
for (os_key, os_value) in std::env::vars_os() {
let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) else {
continue;
};
if !key.starts_with("GOOGLE_") {
continue;
}
if java_has_credential
&& (key == "GOOGLE_SERVICE_ACCOUNT"
|| key == "GOOGLE_SERVICE_ACCOUNT_PATH"
|| key == "GOOGLE_SERVICE_ACCOUNT_KEY"
|| key == "GOOGLE_APPLICATION_CREDENTIALS")
{
continue;
}
// Skip GOOGLE_BUCKET / GOOGLE_BUCKET_NAME: the Java caller's bucket is
// the identity of this registration (matches the registry-key URL).
// Apply the Java bucket explicitly *after* the loop so an env-supplied
// bucket can't silently route the registered URL to a different
// backend bucket.
if key == "GOOGLE_BUCKET" || key == "GOOGLE_BUCKET_NAME" {
continue;
}
if let Ok(config_key) = key.to_ascii_lowercase().parse() {
b = b.with_config(config_key, value);
}
}
b = b.with_bucket_name(&opts.bucket);
if let Some(ref v) = opts.service_account_key {
b = b.with_service_account_key(v);
}
if let Some(ref v) = opts.service_account_path {
b = b.with_service_account_path(v);
}
if let Some(ref v) = opts.application_credentials {
b = b.with_application_credentials(v);
}
let store = b.build()?;
let url = parse_url(url_override, format!("gs://{}", opts.bucket))?;
Ok((url, Arc::new(store)))
}
#[cfg(feature = "object-store-http")]
fn build_http(
url_override: Option<&str>,
opts: &HttpOptions,
) -> JniResult<(Url, Arc)> {
use object_store::http::HttpBuilder;
let listing = url_override.ok_or(
"HttpOptions: ObjectStoreRegistration.url is required for the HTTP backend (no scheme-default)",
)?;
let listing_url =
Url::parse(listing).map_err(|e| format!("invalid HTTP URL {listing:?}: {e}"))?;
// DataFusion's DefaultObjectStoreRegistry::get_url_key strips paths from
// the registry key (it keeps only scheme + host:port), so a registration
// base like `https://example.com/data/` ends up under key
// `https://example.com`. If we hand HttpBuilder the same pathful URL,
// HttpStore prepends `/data/` to every object path on top of the path
// DataFusion already supplied -- a request for
// `https://example.com/data/file.parquet` becomes
// `https://example.com/data/data/file.parquet`. Strip the URL to
// scheme + authority for both the store base and the registry key so
// they match upstream's lookup semantics; the SQL-side URL still carries
// the full path, and HttpStore appends it once.
if !listing_url.path().is_empty() && listing_url.path() != "/" {
return Err(format!(
"HttpOptions: listing URL must be a host root (no path component); \
got {listing:?}. DataFusion's object-store registry keys ignore the \
path portion of a URL, so a pathful base would cause every object \
read to double-prepend the path. Register one HTTP store per \
scheme+host and let SQL paths carry the rest of the URL."
)
.into());
}
let mut b = HttpBuilder::new().with_url(listing);
if let Some(v) = opts.allow_http {
// allow_http lives on ClientOptions in object_store 0.13, not directly
// on HttpBuilder; route through with_client_options.
b = b.with_client_options(object_store::ClientOptions::new().with_allow_http(v));
}
let store = b.build()?;
Ok((listing_url, Arc::new(store)))
}
fn parse_url(url_override: Option<&str>, default: String) -> JniResult {
let s = url_override.unwrap_or(&default);
Url::parse(s).map_err(|e| format!("invalid object store URL {s:?}: {e}").into())
}
apache-datafusion-java-0.1.0/native/src/proto.rs 0000664 0000000 0000000 00000006277 15205164533 0021611 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use std::sync::Arc;
use datafusion::arrow::datatypes::SchemaRef;
use datafusion::arrow::ipc::writer::StreamWriter;
use datafusion::dataframe::DataFrame;
use datafusion::prelude::SessionContext;
use datafusion_proto::logical_plan::{AsLogicalPlan, DefaultLogicalExtensionCodec};
use datafusion_proto::protobuf::LogicalPlanNode;
use jni::objects::{JByteArray, JClass, JString};
use jni::sys::{jbyteArray, jlong};
use jni::JNIEnv;
use prost::Message;
use crate::errors::{try_unwrap_or_throw, JniResult};
use crate::runtime;
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_createDataFrameFromProto<
'local,
>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
plan_bytes: JByteArray<'local>,
) -> jlong {
try_unwrap_or_throw(&mut env, 0, |env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let bytes: Vec = env.convert_byte_array(&plan_bytes)?;
let node = LogicalPlanNode::decode(bytes.as_slice())?;
let codec = DefaultLogicalExtensionCodec {};
let task_ctx = ctx.task_ctx();
let plan = node.try_into_logical_plan(task_ctx.as_ref(), &codec)?;
let df: DataFrame = runtime().block_on(ctx.execute_logical_plan(plan))?;
Ok(Box::into_raw(Box::new(df)) as jlong)
})
}
#[no_mangle]
pub extern "system" fn Java_org_apache_datafusion_SessionContext_tableSchemaIpc<'local>(
mut env: JNIEnv<'local>,
_class: JClass<'local>,
handle: jlong,
name: JString<'local>,
) -> jbyteArray {
try_unwrap_or_throw(
&mut env,
std::ptr::null_mut(),
|env| -> JniResult {
if handle == 0 {
return Err("SessionContext handle is null".into());
}
let ctx = unsafe { &*(handle as *const SessionContext) };
let name: String = env.get_string(&name)?.into();
let df = runtime().block_on(ctx.table(name.as_str()))?;
let schema: SchemaRef = Arc::new(df.schema().as_arrow().clone());
let mut buf: Vec = Vec::new();
{
let mut writer = StreamWriter::try_new(&mut buf, schema.as_ref())?;
writer.finish()?;
}
let arr = env.byte_array_from_slice(&buf)?;
Ok(arr.into_raw())
},
)
}
apache-datafusion-java-0.1.0/native/src/runtime_metrics.rs 0000664 0000000 0000000 00000020513 15205164533 0023644 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Tokio runtime metrics, gated behind the `runtime-metrics` Cargo feature.
//!
//! `tokio-metrics` only compiles when `--cfg tokio_unstable` is set, so the
//! whole module is conditional on the feature flag. The feature-off variant
//! still exists; `runtime_stats()` returns an `Err` whose message points at
//! the rebuild command. The Java surface (`SessionContext.runtimeStats()`)
//! is therefore stable across both build configurations.
//!
//! Field layout returned across the FFI boundary (in this exact order, all
//! `i64`):
//! 0 numWorkers
//! 1 liveTasksCount
//! 2 globalQueueDepth
//! 3 elapsedNanos
//! 4 totalBusyNanos
//! 5 totalParkCount
//! 6 totalPollsCount
//! 7 totalNoopCount
//! 8 totalStealCount
//! 9 totalLocalScheduleCount
//! 10 totalOverflowCount
#[cfg(not(feature = "runtime-metrics"))]
use crate::errors::JniResult;
/// Number of i64 values in the snapshot array; kept here so the Java side and
/// the feature-off stub agree on the layout.
pub const STATS_FIELD_COUNT: usize = 11;
#[cfg(feature = "runtime-metrics")]
mod imp {
use std::sync::{Mutex, OnceLock};
use std::time::Duration;
use tokio_metrics::{RuntimeIntervals, RuntimeMonitor};
use super::STATS_FIELD_COUNT;
use crate::errors::JniResult;
/// `RuntimeMonitor::intervals().next()` returns *delta* metrics covering
/// the period since the previous call (or, on the very first call, since
/// the iterator was constructed). To present the documented
/// "totals since runtime start" semantic to Java callers, we own the
/// iterator process-wide and accumulate every field that's documented as
/// monotonic.
///
/// Snapshot fields (`workers_count`, `live_tasks_count`,
/// `global_queue_depth`) are point-in-time and pass through unchanged.
struct RuntimeAccumulator {
intervals: RuntimeIntervals,
elapsed: Duration,
total_busy: Duration,
total_park_count: u64,
total_polls_count: u64,
total_noop_count: u64,
total_steal_count: u64,
total_local_schedule_count: u64,
total_overflow_count: u64,
}
static ACC: OnceLock> = OnceLock::new();
fn accumulator() -> &'static Mutex {
// Warm `crate::runtime()` first so the shared Tokio runtime exists
// and its OnceLock initializer has already populated ACC via
// `runtime_metrics::init`. We must NOT call `runtime()` from inside
// `ACC.get_or_init`: that would acquire ACC's once-init slot, then
// (on the very first call) recurse through `RT.get_or_init` ->
// `init()` -> `ACC.get_or_init`, which is same-thread reentrancy on
// the same OnceLock and deadlocks indefinitely.
let _ = crate::runtime();
// After `runtime()` returns, ACC is guaranteed populated. The
// defensive `get_or_init` covers a future build path that constructs
// the runtime without going through `crate::runtime` (none today);
// it is safe to invoke from here because no init lock is held.
ACC.get_or_init(build_accumulator)
}
fn build_accumulator() -> Mutex {
let handle = crate::runtime().handle().clone();
let monitor = RuntimeMonitor::new(&handle);
Mutex::new(RuntimeAccumulator {
intervals: monitor.intervals(),
elapsed: Duration::ZERO,
total_busy: Duration::ZERO,
total_park_count: 0,
total_polls_count: 0,
total_noop_count: 0,
total_steal_count: 0,
total_local_schedule_count: 0,
total_overflow_count: 0,
})
}
/// Eagerly construct the runtime-metrics accumulator. Called from
/// `crate::runtime()` immediately after the shared Tokio runtime is
/// created so the accumulator's interval baseline matches runtime start.
/// Without this, `RuntimeMonitor` would only start sampling at the first
/// `runtimeStats()` call, silently dropping every poll/busy nanosecond
/// that happened before that.
pub fn init(handle: &tokio::runtime::Handle) {
ACC.get_or_init(|| {
let monitor = RuntimeMonitor::new(handle);
Mutex::new(RuntimeAccumulator {
intervals: monitor.intervals(),
elapsed: Duration::ZERO,
total_busy: Duration::ZERO,
total_park_count: 0,
total_polls_count: 0,
total_noop_count: 0,
total_steal_count: 0,
total_local_schedule_count: 0,
total_overflow_count: 0,
})
});
}
pub fn runtime_stats() -> JniResult<[i64; STATS_FIELD_COUNT]> {
let mut acc = accumulator()
.lock()
.map_err(|_| "runtime-metrics accumulator lock poisoned")?;
let delta = acc
.intervals
.next()
.ok_or("tokio-metrics RuntimeMonitor returned no interval")?;
// Workers count and the point-in-time fields are not deltas; they are
// the runtime's current state. Pass through directly.
let workers_count = delta.workers_count;
let live_tasks_count = delta.live_tasks_count;
let global_queue_depth = delta.global_queue_depth;
// Accumulate the deltas.
acc.elapsed = acc.elapsed.saturating_add(delta.elapsed);
acc.total_busy = acc.total_busy.saturating_add(delta.total_busy_duration);
acc.total_park_count = acc.total_park_count.saturating_add(delta.total_park_count);
acc.total_polls_count = acc
.total_polls_count
.saturating_add(delta.total_polls_count);
acc.total_noop_count = acc.total_noop_count.saturating_add(delta.total_noop_count);
acc.total_steal_count = acc
.total_steal_count
.saturating_add(delta.total_steal_count);
acc.total_local_schedule_count = acc
.total_local_schedule_count
.saturating_add(delta.total_local_schedule_count);
acc.total_overflow_count = acc
.total_overflow_count
.saturating_add(delta.total_overflow_count);
Ok([
workers_count as i64,
live_tasks_count as i64,
global_queue_depth as i64,
i128_to_i64_saturating(acc.elapsed.as_nanos() as i128),
i128_to_i64_saturating(acc.total_busy.as_nanos() as i128),
acc.total_park_count as i64,
acc.total_polls_count as i64,
acc.total_noop_count as i64,
acc.total_steal_count as i64,
acc.total_local_schedule_count as i64,
acc.total_overflow_count as i64,
])
}
fn i128_to_i64_saturating(v: i128) -> i64 {
v.clamp(i64::MIN as i128, i64::MAX as i128) as i64
}
}
#[cfg(feature = "runtime-metrics")]
pub use imp::{init, runtime_stats};
/// No-op when the `runtime-metrics` feature is off: there is no monitor to
/// initialise. Kept as a regular function (rather than a `cfg` annotation at
/// every call site) so the runtime construction path stays unconditional.
#[cfg(not(feature = "runtime-metrics"))]
pub fn init(_handle: &tokio::runtime::Handle) {}
#[cfg(not(feature = "runtime-metrics"))]
pub fn runtime_stats() -> JniResult<[i64; STATS_FIELD_COUNT]> {
Err(
"datafusion-jni was built without the `runtime-metrics` Cargo feature; \
rebuild the native crate with \
`RUSTFLAGS=\"--cfg tokio_unstable\" cargo build --features runtime-metrics` \
to enable SessionContext.runtimeStats"
.into(),
)
}
apache-datafusion-java-0.1.0/native/src/schema.rs 0000664 0000000 0000000 00000002646 15205164533 0021702 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use datafusion::arrow::datatypes::Schema;
use datafusion::arrow::ipc::reader::StreamReader;
use jni::objects::JByteArray;
use jni::JNIEnv;
use crate::errors::JniResult;
/// Decode an optional Arrow-IPC schema byte array passed in from Java.
/// Returns `None` if the byte-array reference is null.
pub(crate) fn decode_optional_schema(
env: &mut JNIEnv,
bytes: JByteArray,
) -> JniResult> {
if bytes.is_null() {
return Ok(None);
}
let buf: Vec = env.convert_byte_array(&bytes)?;
let reader = StreamReader::try_new(std::io::Cursor::new(buf), None)?;
Ok(Some((*reader.schema()).clone()))
}
apache-datafusion-java-0.1.0/native/src/table_provider.rs 0000664 0000000 0000000 00000023071 15205164533 0023436 0 ustar 00root root 0000000 0000000 // Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Java-backed [`TableProvider`] implementation.
//!
//! Used by `SessionContext::registerTable` on the Java side to register user-implemented
//! `TableProvider`s. The internal struct here mirrors the role of DataFusion's Rust
//! `TableProvider` trait; it currently only supports a single-partition, no-pushdown scan,
//! with future pushdown and partitioning support tracked as follow-ups.
use std::any::Any;
use std::fmt;
use std::sync::Arc;
use async_trait::async_trait;
use datafusion::arrow::datatypes::SchemaRef;
use datafusion::arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
use datafusion::arrow::record_batch::{RecordBatch, RecordBatchReader};
use datafusion::catalog::Session;
use datafusion::datasource::{TableProvider, TableType};
use datafusion::error::{DataFusionError, Result};
use datafusion::execution::context::TaskContext;
use datafusion::logical_expr::Expr;
use datafusion::physical_expr::EquivalenceProperties;
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
SendableRecordBatchStream,
};
use futures::stream::StreamExt;
use jni::objects::{GlobalRef, JStaticMethodID};
use jni::signature::{Primitive, ReturnType};
use jni::sys::{jlong, jvalue};
use crate::jni_util::jthrowable_to_string;
pub(crate) struct JavaTableProvider {
pub(crate) name: String,
pub(crate) schema: SchemaRef,
pub(crate) source_global_ref: Arc,
pub(crate) bridge_class: Arc,
pub(crate) invoke_method: JStaticMethodID,
}
// SAFETY: see the matching unsafe impls on JavaScalarUdf. The GlobalRefs keep
// the Java objects alive; JStaticMethodID points into the class held by
// bridge_class; nothing is mutated after construction.
unsafe impl Send for JavaTableProvider {}
unsafe impl Sync for JavaTableProvider {}
impl fmt::Debug for JavaTableProvider {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("JavaTableProvider")
.field("name", &self.name)
.field("schema", &self.schema)
.finish()
}
}
#[async_trait]
impl TableProvider for JavaTableProvider {
fn as_any(&self) -> &dyn Any {
self
}
fn schema(&self) -> SchemaRef {
Arc::clone(&self.schema)
}
fn table_type(&self) -> TableType {
TableType::Base
}
async fn scan(
&self,
_state: &dyn Session,
projection: Option<&Vec>,
_filters: &[Expr],
_limit: Option,
) -> Result> {
let projected_schema = match projection {
Some(p) => Arc::new(self.schema.project(p)?),
None => Arc::clone(&self.schema),
};
let plan_properties = Arc::new(PlanProperties::new(
EquivalenceProperties::new(Arc::clone(&projected_schema)),
Partitioning::UnknownPartitioning(1),
EmissionType::Incremental,
Boundedness::Bounded,
));
Ok(Arc::new(JavaScanExec {
name: self.name.clone(),
full_schema: Arc::clone(&self.schema),
projected_schema,
projection: projection.cloned(),
source_global_ref: Arc::clone(&self.source_global_ref),
bridge_class: Arc::clone(&self.bridge_class),
invoke_method: self.invoke_method,
plan_properties,
}))
}
}
pub(crate) struct JavaScanExec {
name: String,
full_schema: SchemaRef,
projected_schema: SchemaRef,
projection: Option>,
source_global_ref: Arc,
bridge_class: Arc,
invoke_method: JStaticMethodID,
plan_properties: Arc