diff --git a/python/setup.cfg b/python/setup.cfg index 5c58314131e0..a707fe123d77 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -42,6 +42,11 @@ package_dir = = src packages = find: python_requires = >=3.7 - +[options.extras_require] +arrow = + pyarrow +dev= + tox-travis==0.12 + pytest [options.packages.find] where = src diff --git a/python/setup.py b/python/setup.py deleted file mode 100644 index 5073c558a252..000000000000 --- a/python/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from setuptools import setup - -setup( - name="py-iceberg", - install_requires=[], - extras_require={ - "dev": [ - "tox-travis==0.12", - "pytest", - ], - }, -) diff --git a/python/src/iceberg/expressions.py b/python/src/iceberg/expressions.py index 48d1ae24f023..b34d08212594 100644 --- a/python/src/iceberg/expressions.py +++ b/python/src/iceberg/expressions.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - from enum import Enum, auto @@ -22,10 +21,10 @@ class Operation(Enum): """Operations to be used as components in expressions Operations can be negated by calling the negate method. - >>> print(Operation.TRUE.negate()) - Operation.FALSE - >>> print(Operation.IS_NULL.negate()) - Operation.NOT_NULL + >>> Operation.TRUE.negate() + + >>> Operation.IS_NULL.negate() + The above example uses the OPERATION_NEGATIONS map which maps each enum to it's opposite enum. @@ -53,7 +52,7 @@ class Operation(Enum): AND = auto() OR = auto() - def negate(self): + def negate(self) -> "Operation": """Returns the operation used when this is negated.""" try: diff --git a/python/src/iceberg/io/pyarrow.py b/python/src/iceberg/io/pyarrow.py index 5eff2d239294..2cf5b9fc15a7 100644 --- a/python/src/iceberg/io/pyarrow.py +++ b/python/src/iceberg/io/pyarrow.py @@ -43,9 +43,14 @@ class PyArrowFile(InputFile, OutputFile): Examples: >>> from iceberg.io.pyarrow import PyArrowFile >>> input_file = PyArrowFile("s3://foo/bar.txt") - >>> file_content = input_file.open().read() # Read the contents of the PyArrowFile instance + >>> # Read the contents of the PyArrowFile instance + >>> # Make sure that you have permissions to read/write + >>> # file_content = input_file.open().read() + >>> output_file = PyArrowFile("s3://baz/qux.txt") - >>> output_file.create().write(b'foobytes') # Write bytes to the PyArrowFile instance + >>> # Write bytes to a file + >>> # Make sure that you have permissions to read/write + >>> # output_file.create().write(b'foobytes') """ def __init__(self, location: str): diff --git a/python/src/iceberg/types.py b/python/src/iceberg/types.py index af1f595663f8..2fd68691b190 100644 --- a/python/src/iceberg/types.py +++ b/python/src/iceberg/types.py @@ -20,10 +20,11 @@ describe an Iceberg table schema, these classes can be used in the construction of a StructType instance. Example: - >>> StructType( - NestedField(True, 1, "required_field", StringType()), - NestedField(False, 2, "optional_field", IntegerType()) - ) + >>> str(StructType( + ... NestedField(1, "required_field", StringType(), True), + ... NestedField(2, "optional_field", IntegerType()) + ... )) + 'struct<1: required_field: optional string, 2: optional_field: optional int>' Notes: - https://iceberg.apache.org/#spec/#primitive-types @@ -98,7 +99,7 @@ class DecimalType(PrimitiveType): Example: >>> DecimalType(32, 3) DecimalType(precision=32, scale=3) - >>> DecimalType(8, 3)==DecimalType(8, 3) + >>> DecimalType(8, 3) == DecimalType(8, 3) True """ @@ -201,10 +202,11 @@ class StructType(IcebergType): """A struct type in Iceberg Example: - >>> StructType( - NestedField(True, 1, "required_field", StringType()), - NestedField(False, 2, "optional_field", IntegerType()) - ) + >>> str(StructType( + ... NestedField(1, "required_field", StringType(), True), + ... NestedField(2, "optional_field", IntegerType()) + ... )) + 'struct<1: required_field: optional string, 2: optional_field: optional int>' """ _instances: Dict[Tuple[NestedField, ...], "StructType"] = {} @@ -231,7 +233,7 @@ class ListType(IcebergType): Example: >>> ListType(element_id=3, element_type=StringType(), element_is_optional=True) - ListType(element=NestedField(is_optional=True, field_id=3, name='element', field_type=StringType(), doc=None)) + ListType(element_id=3, element_type=StringType(), element_is_optional=True) """ _instances: Dict[Tuple[bool, int, IcebergType], "ListType"] = {} @@ -275,10 +277,7 @@ class MapType(IcebergType): Example: >>> MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_is_optional=True) - MapType( - key=NestedField(is_optional=False, field_id=1, name='key', field_type=StringType(), doc=None), - value=NestedField(is_optional=True, field_id=2, name='value', field_type=IntegerType(), doc=None) - ) + MapType(key_id=1, key_type=StringType(), value_id=2, value_type=IntegerType(), value_is_optional=True) """ _instances: Dict[Tuple[int, IcebergType, int, IcebergType, bool], "MapType"] = {} diff --git a/python/tox.ini b/python/tox.ini index 214df438c9b5..dcd1ca766832 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -20,11 +20,12 @@ envlist = py37,py38,py39,linters [testenv] usedevelop = true +extras = arrow deps = coverage mock pytest - pyarrow + pytest-checkdocs setenv = COVERAGE_FILE = test-reports/{envname}/.coverage PYTEST_ADDOPTS = --junitxml=test-reports/{envname}/junit.xml -vv @@ -50,17 +51,17 @@ deps = isort autoflake commands = - autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports setup.py src tests - isort --profile black --check-only setup.py src tests - black --line-length 130 --check --diff setup.py src tests + autoflake -r --check --ignore-init-module-imports --remove-all-unused-imports src tests + isort --profile black --check-only src tests + black --line-length 130 --check --diff src tests [testenv:format] deps = {[testenv:format-check]deps} commands = - autoflake -r --in-place --ignore-init-module-imports --remove-all-unused-imports setup.py src tests - isort --profile black setup.py src tests - black --line-length 130 setup.py src tests + autoflake -r --in-place --ignore-init-module-imports --remove-all-unused-imports src tests + isort --profile black src tests + black --line-length 130 src tests [testenv:type-check] deps = @@ -86,6 +87,7 @@ commands = [pytest] norecursedirs=.* +addopts = --doctest-modules [gh-actions] python =