Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions pyathena/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,12 @@ class DBAPITypeObject(FrozenSet[str]):
def __eq__(self, other: object):
if isinstance(other, frozenset):
return frozenset.__eq__(self, other)
else:
return other in self
return other in self

def __ne__(self, other: object):
if isinstance(other, frozenset):
return frozenset.__ne__(self, other)
else:
return other not in self
return other not in self

def __hash__(self):
return frozenset.__hash__(self)
Expand Down
5 changes: 2 additions & 3 deletions pyathena/arrow/async_cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,7 @@ def get_default_converter(
) -> Union[DefaultArrowTypeConverter, DefaultArrowUnloadTypeConverter, Any]:
if unload:
return DefaultArrowUnloadTypeConverter()
else:
return DefaultArrowTypeConverter()
return DefaultArrowTypeConverter()

@property
def arraysize(self) -> int:
Expand All @@ -80,7 +79,7 @@ def _collect_result_set(
kwargs: Optional[Dict[str, Any]] = None,
) -> AthenaArrowResultSet:
if kwargs is None:
kwargs = dict()
kwargs = {}
query_execution = cast(AthenaQueryExecution, self._poll(query_id))
return AthenaArrowResultSet(
connection=self._connection,
Expand Down
7 changes: 3 additions & 4 deletions pyathena/arrow/converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,9 @@
def _to_date(value: Optional[Union[str, datetime]]) -> Optional[date]:
if value is None:
return None
elif isinstance(value, datetime):
if isinstance(value, datetime):
return value.date()
else:
return datetime.strptime(value, "%Y-%m-%d").date()
return datetime.strptime(value, "%Y-%m-%d").date()


_DEFAULT_ARROW_CONVERTERS: Dict[str, Callable[[Optional[str]], Optional[Any]]] = {
Expand Down Expand Up @@ -82,7 +81,7 @@ def convert(self, type_: str, value: Optional[str]) -> Optional[Any]:
class DefaultArrowUnloadTypeConverter(Converter):
def __init__(self) -> None:
super().__init__(
mappings=dict(),
mappings={},
default=_to_default,
)

Expand Down
3 changes: 1 addition & 2 deletions pyathena/arrow/cursor.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,7 @@ def get_default_converter(
) -> Union[DefaultArrowTypeConverter, DefaultArrowUnloadTypeConverter, Any]:
if unload:
return DefaultArrowUnloadTypeConverter()
else:
return DefaultArrowTypeConverter()
return DefaultArrowTypeConverter()

@property
def arraysize(self) -> int:
Expand Down
12 changes: 6 additions & 6 deletions pyathena/arrow/result_set.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def __init__(
else:
import pyarrow as pa

self._table = pa.Table.from_pydict(dict())
self._table = pa.Table.from_pydict({})
self._batches = iter(self._table.to_batches(arraysize))

def __s3_file_system(self):
Expand Down Expand Up @@ -199,14 +199,14 @@ def _read_csv(self) -> "Table":
if not self.output_location:
raise ProgrammingError("OutputLocation is none or empty.")
if not self.output_location.endswith((".csv", ".txt")):
return pa.Table.from_pydict(dict())
return pa.Table.from_pydict({})
if self.substatement_type and self.substatement_type.upper() in (
"UPDATE",
"DELETE",
"MERGE",
"VACUUM_TABLE",
):
return pa.Table.from_pydict(dict())
return pa.Table.from_pydict({})
length = self._get_content_length()
if length and self.output_location.endswith(".txt"):
description = self.description if self.description else []
Expand All @@ -232,7 +232,7 @@ def _read_csv(self) -> "Table":
escape_char=False,
)
else:
return pa.Table.from_pydict(dict())
return pa.Table.from_pydict({})

bucket, key = parse_output_location(self.output_location)
try:
Expand All @@ -256,7 +256,7 @@ def _read_parquet(self) -> "Table":

manifests = self._read_data_manifest()
if not manifests:
return pa.Table.from_pydict(dict())
return pa.Table.from_pydict({})
if not self._unload_location:
self._unload_location = "/".join(manifests[0].split("/")[:-1]) + "/"

Expand All @@ -283,5 +283,5 @@ def close(self) -> None:
import pyarrow as pa

super().close()
self._table = pa.Table.from_pydict(dict())
self._table = pa.Table.from_pydict({})
self._batches = []
31 changes: 15 additions & 16 deletions pyathena/arrow/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,42 +29,41 @@ def get_athena_type(type_: "DataType") -> Tuple[str, int, int]:

if type_.id in [types.Type_BOOL]: # 1
return "boolean", 0, 0
elif type_.id in [types.Type_UINT8, types.Type_INT8]: # 2, 3
if type_.id in [types.Type_UINT8, types.Type_INT8]: # 2, 3
return "tinyint", 3, 0
elif type_.id in [types.Type_UINT16, types.Type_INT16]: # 4, 5
if type_.id in [types.Type_UINT16, types.Type_INT16]: # 4, 5
return "smallint", 5, 0
elif type_.id in [types.Type_UINT32, types.Type_INT32]: # 6, 7
if type_.id in [types.Type_UINT32, types.Type_INT32]: # 6, 7
return "integer", 10, 0
elif type_.id in [types.Type_UINT64, types.Type_INT64]: # 8, 9
if type_.id in [types.Type_UINT64, types.Type_INT64]: # 8, 9
return "bigint", 19, 0
elif type_.id in [types.Type_HALF_FLOAT, types.Type_FLOAT]: # 10, 11
if type_.id in [types.Type_HALF_FLOAT, types.Type_FLOAT]: # 10, 11
return "float", 17, 0
elif type_.id in [types.Type_DOUBLE]: # 12
if type_.id in [types.Type_DOUBLE]: # 12
return "double", 17, 0
elif type_.id in [types.Type_STRING, types.Type_LARGE_STRING]: # 13, 34
if type_.id in [types.Type_STRING, types.Type_LARGE_STRING]: # 13, 34
return "varchar", 2147483647, 0
elif type_.id in [
if type_.id in [
types.Type_BINARY,
types.Type_FIXED_SIZE_BINARY,
types.Type_LARGE_BINARY,
]: # 14, 15, 35
return "varbinary", 1073741824, 0
elif type_.id in [types.Type_DATE32, types.Type_DATE64]: # 16, 17
if type_.id in [types.Type_DATE32, types.Type_DATE64]: # 16, 17
return "date", 0, 0
elif type_.id == types.Type_TIMESTAMP: # 18
if type_.id == types.Type_TIMESTAMP: # 18
return "timestamp", 3, 0
elif type_.id in [types.Type_DECIMAL128, types.Decimal256Type]: # 23, 24
if type_.id in [types.Type_DECIMAL128, types.Decimal256Type]: # 23, 24
type_ = cast(types.Decimal128Type, type_)
return "decimal", type_.precision, type_.scale
elif type_.id in [
if type_.id in [
types.Type_LIST,
types.Type_FIXED_SIZE_LIST,
types.Type_LARGE_LIST,
]: # 25, 32, 36
return "array", 0, 0
elif type_.id in [types.Type_STRUCT]: # 26
if type_.id in [types.Type_STRUCT]: # 26
return "row", 0, 0
elif type_.id in [types.Type_MAP]: # 30
if type_.id in [types.Type_MAP]: # 30
return "map", 0, 0
else:
return "string", 2147483647, 0
return "string", 2147483647, 0
10 changes: 4 additions & 6 deletions pyathena/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,7 @@ def __next__(self):
row = self.fetchone()
if row is None:
raise StopIteration
else:
return row
return row

def __iter__(self):
return self
Expand Down Expand Up @@ -482,8 +481,7 @@ def __poll(self, query_id: str) -> Union[AthenaQueryExecution, AthenaCalculation
AthenaQueryExecution.STATE_CANCELLED,
]:
return query_execution
else:
time.sleep(self._poll_interval)
time.sleep(self._poll_interval)

def _poll(self, query_id: str) -> Union[AthenaQueryExecution, AthenaCalculationExecution]:
try:
Expand Down Expand Up @@ -654,11 +652,11 @@ def _cancel(self, query_id: str) -> None:
_logger.exception("Failed to cancel query.")
raise OperationalError(*e.args) from e

def setinputsizes(self, sizes):
def setinputsizes(self, sizes): # noqa: B027
"""Does nothing by default"""
pass

def setoutputsize(self, size, column=None):
def setoutputsize(self, size, column=None): # noqa: B027
"""Does nothing by default"""
pass

Expand Down
6 changes: 3 additions & 3 deletions pyathena/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def __init__(
converter: Optional[Converter] = None,
formatter: Optional[Formatter] = None,
retry_config: Optional[RetryConfig] = None,
cursor_class: Optional[Type[ConnectionCursor]] = cast(Type[ConnectionCursor], Cursor),
cursor_class: Optional[Type[ConnectionCursor]] = None,
cursor_kwargs: Optional[Dict[str, Any]] = None,
kill_on_interrupt: bool = True,
session: Optional[Session] = None,
Expand Down Expand Up @@ -234,8 +234,8 @@ def __init__(
self._converter = converter
self._formatter = formatter if formatter else DefaultParameterFormatter()
self._retry_config = retry_config if retry_config else RetryConfig()
self.cursor_class = cast(Type[ConnectionCursor], cursor_class)
self.cursor_kwargs = cursor_kwargs if cursor_kwargs else dict()
self.cursor_class = cursor_class if cursor_class else cast(Type[ConnectionCursor], Cursor)
self.cursor_kwargs = cursor_kwargs if cursor_kwargs else {}
self.kill_on_interrupt = kill_on_interrupt
self.result_reuse_enable = result_reuse_enable
self.result_reuse_minutes = result_reuse_minutes
Expand Down
4 changes: 2 additions & 2 deletions pyathena/converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,12 +117,12 @@ def __init__(
if mappings:
self._mappings = mappings
else:
self._mappings = dict()
self._mappings = {}
self._default = default
if types:
self._types = types
else:
self._types = dict()
self._types = {}

@property
def mappings(self) -> Dict[str, Callable[[Optional[str]], Optional[Any]]]:
Expand Down
39 changes: 17 additions & 22 deletions pyathena/fastparquet/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,34 +37,29 @@ def get_athena_type(type_: "SchemaElement") -> Tuple[str, int, int]:

if type_.type in [Type.BOOLEAN]:
return "boolean", 0, 0
elif type_.type in [Type.INT32]:
if type_.type in [Type.INT32]:
if type_.converted_type == ConvertedType.DATE:
return "date", 0, 0
else:
return "integer", 10, 0
elif type_.type in [Type.INT64]:
return "integer", 10, 0
if type_.type in [Type.INT64]:
return "bigint", 19, 0
elif type_.type in [Type.INT96]:
if type_.type in [Type.INT96]:
return "timestamp", 3, 0
elif type_.type in [Type.FLOAT]:
if type_.type in [Type.FLOAT]:
return "float", 17, 0
elif type_.type in [Type.DOUBLE]:
if type_.type in [Type.DOUBLE]:
return "double", 17, 0
elif type_.type in [Type.BYTE_ARRAY, Type.FIXED_LEN_BYTE_ARRAY]:
if type_.type in [Type.BYTE_ARRAY, Type.FIXED_LEN_BYTE_ARRAY]:
if type_.converted_type == ConvertedType.UTF8:
return "varchar", 2147483647, 0
elif type_.converted_type == ConvertedType.DECIMAL:
if type_.converted_type == ConvertedType.DECIMAL:
return "decimal", type_.precision, type_.scale
else:
return "varbinary", 1073741824, 0
else:
if type_.converted_type == ConvertedType.LIST:
return "array", 0, 0
elif type_.converted_type == ConvertedType.MAP:
return "map", 0, 0
else:
children = getattr(type_, "children", [])
if type_.type is None and type_.converted_type is None and children:
return "row", 0, 0
else:
return "string", 2147483647, 0
return "varbinary", 1073741824, 0
if type_.converted_type == ConvertedType.LIST:
return "array", 0, 0
if type_.converted_type == ConvertedType.MAP:
return "map", 0, 0
children = getattr(type_, "children", [])
if type_.type is None and type_.converted_type is None and children:
return "row", 0, 0
return "string", 2147483647, 0
Loading
Loading