457 lines
14 KiB
Python
457 lines
14 KiB
Python
|
import asyncio
|
||
|
|
import enum
|
||
|
|
import io
|
||
|
|
import json
|
||
|
|
import mimetypes
|
||
|
|
import os
|
||
|
|
import warnings
|
||
|
|
from abc import ABC, abstractmethod
|
||
|
|
from itertools import chain
|
||
|
|
from typing import (
|
||
|
|
IO,
|
||
|
|
TYPE_CHECKING,
|
||
|
|
Any,
|
||
|
|
ByteString,
|
||
|
|
Dict,
|
||
|
|
Iterable,
|
||
|
|
Optional,
|
||
|
|
Text,
|
||
|
|
TextIO,
|
||
|
|
Tuple,
|
||
|
|
Type,
|
||
|
|
Union,
|
||
|
|
)
|
||
|
|
|
||
|
|
from multidict import CIMultiDict
|
||
|
|
|
||
|
|
from . import hdrs
|
||
|
|
from .abc import AbstractStreamWriter
|
||
|
|
from .helpers import (
|
||
|
|
PY_36,
|
||
|
|
content_disposition_header,
|
||
|
|
guess_filename,
|
||
|
|
parse_mimetype,
|
||
|
|
sentinel,
|
||
|
|
)
|
||
|
|
from .streams import DEFAULT_LIMIT, StreamReader
|
||
|
|
from .typedefs import JSONEncoder, _CIMultiDict
|
||
|
|
|
||
|
|
__all__ = ('PAYLOAD_REGISTRY', 'get_payload', 'payload_type', 'Payload',
|
||
|
|
'BytesPayload', 'StringPayload',
|
||
|
|
'IOBasePayload', 'BytesIOPayload', 'BufferedReaderPayload',
|
||
|
|
'TextIOPayload', 'StringIOPayload', 'JsonPayload',
|
||
|
|
'AsyncIterablePayload')
|
||
|
|
|
||
|
|
TOO_LARGE_BYTES_BODY = 2 ** 20 # 1 MB
|
||
|
|
|
||
|
|
|
||
|
|
if TYPE_CHECKING: # pragma: no cover
|
||
|
|
from typing import List # noqa
|
||
|
|
|
||
|
|
|
||
|
|
class LookupError(Exception):
|
||
|
|
pass
|
||
|
|
|
||
|
|
|
||
|
|
class Order(str, enum.Enum):
|
||
|
|
normal = 'normal'
|
||
|
|
try_first = 'try_first'
|
||
|
|
try_last = 'try_last'
|
||
|
|
|
||
|
|
|
||
|
|
def get_payload(data: Any, *args: Any, **kwargs: Any) -> 'Payload':
|
||
|
|
return PAYLOAD_REGISTRY.get(data, *args, **kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
def register_payload(factory: Type['Payload'],
|
||
|
|
type: Any,
|
||
|
|
*,
|
||
|
|
order: Order=Order.normal) -> None:
|
||
|
|
PAYLOAD_REGISTRY.register(factory, type, order=order)
|
||
|
|
|
||
|
|
|
||
|
|
class payload_type:
|
||
|
|
|
||
|
|
def __init__(self, type: Any, *, order: Order=Order.normal) -> None:
|
||
|
|
self.type = type
|
||
|
|
self.order = order
|
||
|
|
|
||
|
|
def __call__(self, factory: Type['Payload']) -> Type['Payload']:
|
||
|
|
register_payload(factory, self.type, order=self.order)
|
||
|
|
return factory
|
||
|
|
|
||
|
|
|
||
|
|
class PayloadRegistry:
|
||
|
|
"""Payload registry.
|
||
|
|
|
||
|
|
note: we need zope.interface for more efficient adapter search
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self) -> None:
|
||
|
|
self._first = [] # type: List[Tuple[Type[Payload], Any]]
|
||
|
|
self._normal = [] # type: List[Tuple[Type[Payload], Any]]
|
||
|
|
self._last = [] # type: List[Tuple[Type[Payload], Any]]
|
||
|
|
|
||
|
|
def get(self,
|
||
|
|
data: Any,
|
||
|
|
*args: Any,
|
||
|
|
_CHAIN: Any=chain,
|
||
|
|
**kwargs: Any) -> 'Payload':
|
||
|
|
if isinstance(data, Payload):
|
||
|
|
return data
|
||
|
|
for factory, type in _CHAIN(self._first, self._normal, self._last):
|
||
|
|
if isinstance(data, type):
|
||
|
|
return factory(data, *args, **kwargs)
|
||
|
|
|
||
|
|
raise LookupError()
|
||
|
|
|
||
|
|
def register(self,
|
||
|
|
factory: Type['Payload'],
|
||
|
|
type: Any,
|
||
|
|
*,
|
||
|
|
order: Order=Order.normal) -> None:
|
||
|
|
if order is Order.try_first:
|
||
|
|
self._first.append((factory, type))
|
||
|
|
elif order is Order.normal:
|
||
|
|
self._normal.append((factory, type))
|
||
|
|
elif order is Order.try_last:
|
||
|
|
self._last.append((factory, type))
|
||
|
|
else:
|
||
|
|
raise ValueError("Unsupported order {!r}".format(order))
|
||
|
|
|
||
|
|
|
||
|
|
class Payload(ABC):
|
||
|
|
|
||
|
|
_default_content_type = 'application/octet-stream' # type: str
|
||
|
|
_size = None # type: Optional[int]
|
||
|
|
|
||
|
|
def __init__(self,
|
||
|
|
value: Any,
|
||
|
|
headers: Optional[
|
||
|
|
Union[
|
||
|
|
_CIMultiDict,
|
||
|
|
Dict[str, str],
|
||
|
|
Iterable[Tuple[str, str]]
|
||
|
|
]
|
||
|
|
] = None,
|
||
|
|
content_type: Optional[str]=sentinel,
|
||
|
|
filename: Optional[str]=None,
|
||
|
|
encoding: Optional[str]=None,
|
||
|
|
**kwargs: Any) -> None:
|
||
|
|
self._encoding = encoding
|
||
|
|
self._filename = filename
|
||
|
|
self._headers = CIMultiDict() # type: _CIMultiDict
|
||
|
|
self._value = value
|
||
|
|
if content_type is not sentinel and content_type is not None:
|
||
|
|
self._headers[hdrs.CONTENT_TYPE] = content_type
|
||
|
|
elif self._filename is not None:
|
||
|
|
content_type = mimetypes.guess_type(self._filename)[0]
|
||
|
|
if content_type is None:
|
||
|
|
content_type = self._default_content_type
|
||
|
|
self._headers[hdrs.CONTENT_TYPE] = content_type
|
||
|
|
else:
|
||
|
|
self._headers[hdrs.CONTENT_TYPE] = self._default_content_type
|
||
|
|
self._headers.update(headers or {})
|
||
|
|
|
||
|
|
@property
|
||
|
|
def size(self) -> Optional[int]:
|
||
|
|
"""Size of the payload."""
|
||
|
|
return self._size
|
||
|
|
|
||
|
|
@property
|
||
|
|
def filename(self) -> Optional[str]:
|
||
|
|
"""Filename of the payload."""
|
||
|
|
return self._filename
|
||
|
|
|
||
|
|
@property
|
||
|
|
def headers(self) -> _CIMultiDict:
|
||
|
|
"""Custom item headers"""
|
||
|
|
return self._headers
|
||
|
|
|
||
|
|
@property
|
||
|
|
def _binary_headers(self) -> bytes:
|
||
|
|
return ''.join(
|
||
|
|
[k + ': ' + v + '\r\n' for k, v in self.headers.items()]
|
||
|
|
).encode('utf-8') + b'\r\n'
|
||
|
|
|
||
|
|
@property
|
||
|
|
def encoding(self) -> Optional[str]:
|
||
|
|
"""Payload encoding"""
|
||
|
|
return self._encoding
|
||
|
|
|
||
|
|
@property
|
||
|
|
def content_type(self) -> str:
|
||
|
|
"""Content type"""
|
||
|
|
return self._headers[hdrs.CONTENT_TYPE]
|
||
|
|
|
||
|
|
def set_content_disposition(self,
|
||
|
|
disptype: str,
|
||
|
|
quote_fields: bool=True,
|
||
|
|
**params: Any) -> None:
|
||
|
|
"""Sets ``Content-Disposition`` header."""
|
||
|
|
self._headers[hdrs.CONTENT_DISPOSITION] = content_disposition_header(
|
||
|
|
disptype, quote_fields=quote_fields, **params)
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
async def write(self, writer: AbstractStreamWriter) -> None:
|
||
|
|
"""Write payload.
|
||
|
|
|
||
|
|
writer is an AbstractStreamWriter instance:
|
||
|
|
"""
|
||
|
|
|
||
|
|
|
||
|
|
class BytesPayload(Payload):
|
||
|
|
|
||
|
|
def __init__(self,
|
||
|
|
value: ByteString,
|
||
|
|
*args: Any,
|
||
|
|
**kwargs: Any) -> None:
|
||
|
|
if not isinstance(value, (bytes, bytearray, memoryview)):
|
||
|
|
raise TypeError("value argument must be byte-ish, not (!r)"
|
||
|
|
.format(type(value)))
|
||
|
|
|
||
|
|
if 'content_type' not in kwargs:
|
||
|
|
kwargs['content_type'] = 'application/octet-stream'
|
||
|
|
|
||
|
|
super().__init__(value, *args, **kwargs)
|
||
|
|
|
||
|
|
self._size = len(value)
|
||
|
|
|
||
|
|
if self._size > TOO_LARGE_BYTES_BODY:
|
||
|
|
if PY_36:
|
||
|
|
kwargs = {'source': self}
|
||
|
|
else:
|
||
|
|
kwargs = {}
|
||
|
|
warnings.warn("Sending a large body directly with raw bytes might"
|
||
|
|
" lock the event loop. You should probably pass an "
|
||
|
|
"io.BytesIO object instead", ResourceWarning,
|
||
|
|
**kwargs)
|
||
|
|
|
||
|
|
async def write(self, writer: AbstractStreamWriter) -> None:
|
||
|
|
await writer.write(self._value)
|
||
|
|
|
||
|
|
|
||
|
|
class StringPayload(BytesPayload):
|
||
|
|
|
||
|
|
def __init__(self,
|
||
|
|
value: Text,
|
||
|
|
*args: Any,
|
||
|
|
encoding: Optional[str]=None,
|
||
|
|
content_type: Optional[str]=None,
|
||
|
|
**kwargs: Any) -> None:
|
||
|
|
|
||
|
|
if encoding is None:
|
||
|
|
if content_type is None:
|
||
|
|
real_encoding = 'utf-8'
|
||
|
|
content_type = 'text/plain; charset=utf-8'
|
||
|
|
else:
|
||
|
|
mimetype = parse_mimetype(content_type)
|
||
|
|
real_encoding = mimetype.parameters.get('charset', 'utf-8')
|
||
|
|
else:
|
||
|
|
if content_type is None:
|
||
|
|
content_type = 'text/plain; charset=%s' % encoding
|
||
|
|
real_encoding = encoding
|
||
|
|
|
||
|
|
super().__init__(
|
||
|
|
value.encode(real_encoding),
|
||
|
|
encoding=real_encoding,
|
||
|
|
content_type=content_type,
|
||
|
|
*args,
|
||
|
|
**kwargs,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
class StringIOPayload(StringPayload):
|
||
|
|
|
||
|
|
def __init__(self,
|
||
|
|
value: IO[str],
|
||
|
|
*args: Any,
|
||
|
|
**kwargs: Any) -> None:
|
||
|
|
super().__init__(value.read(), *args, **kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
class IOBasePayload(Payload):
|
||
|
|
|
||
|
|
def __init__(self,
|
||
|
|
value: IO[Any],
|
||
|
|
disposition: str='attachment',
|
||
|
|
*args: Any,
|
||
|
|
**kwargs: Any) -> None:
|
||
|
|
if 'filename' not in kwargs:
|
||
|
|
kwargs['filename'] = guess_filename(value)
|
||
|
|
|
||
|
|
super().__init__(value, *args, **kwargs)
|
||
|
|
|
||
|
|
if self._filename is not None and disposition is not None:
|
||
|
|
if hdrs.CONTENT_DISPOSITION not in self.headers:
|
||
|
|
self.set_content_disposition(
|
||
|
|
disposition, filename=self._filename
|
||
|
|
)
|
||
|
|
|
||
|
|
async def write(self, writer: AbstractStreamWriter) -> None:
|
||
|
|
loop = asyncio.get_event_loop()
|
||
|
|
try:
|
||
|
|
chunk = await loop.run_in_executor(
|
||
|
|
None, self._value.read, DEFAULT_LIMIT
|
||
|
|
)
|
||
|
|
while chunk:
|
||
|
|
await writer.write(chunk)
|
||
|
|
chunk = await loop.run_in_executor(
|
||
|
|
None, self._value.read, DEFAULT_LIMIT
|
||
|
|
)
|
||
|
|
finally:
|
||
|
|
await loop.run_in_executor(None, self._value.close)
|
||
|
|
|
||
|
|
|
||
|
|
class TextIOPayload(IOBasePayload):
|
||
|
|
|
||
|
|
def __init__(self,
|
||
|
|
value: TextIO,
|
||
|
|
*args: Any,
|
||
|
|
encoding: Optional[str]=None,
|
||
|
|
content_type: Optional[str]=None,
|
||
|
|
**kwargs: Any) -> None:
|
||
|
|
|
||
|
|
if encoding is None:
|
||
|
|
if content_type is None:
|
||
|
|
encoding = 'utf-8'
|
||
|
|
content_type = 'text/plain; charset=utf-8'
|
||
|
|
else:
|
||
|
|
mimetype = parse_mimetype(content_type)
|
||
|
|
encoding = mimetype.parameters.get('charset', 'utf-8')
|
||
|
|
else:
|
||
|
|
if content_type is None:
|
||
|
|
content_type = 'text/plain; charset=%s' % encoding
|
||
|
|
|
||
|
|
super().__init__(
|
||
|
|
value,
|
||
|
|
content_type=content_type,
|
||
|
|
encoding=encoding,
|
||
|
|
*args,
|
||
|
|
**kwargs,
|
||
|
|
)
|
||
|
|
|
||
|
|
@property
|
||
|
|
def size(self) -> Optional[int]:
|
||
|
|
try:
|
||
|
|
return os.fstat(self._value.fileno()).st_size - self._value.tell()
|
||
|
|
except OSError:
|
||
|
|
return None
|
||
|
|
|
||
|
|
async def write(self, writer: AbstractStreamWriter) -> None:
|
||
|
|
loop = asyncio.get_event_loop()
|
||
|
|
try:
|
||
|
|
chunk = await loop.run_in_executor(
|
||
|
|
None, self._value.read, DEFAULT_LIMIT
|
||
|
|
)
|
||
|
|
while chunk:
|
||
|
|
await writer.write(chunk.encode(self._encoding))
|
||
|
|
chunk = await loop.run_in_executor(
|
||
|
|
None, self._value.read, DEFAULT_LIMIT
|
||
|
|
)
|
||
|
|
finally:
|
||
|
|
await loop.run_in_executor(None, self._value.close)
|
||
|
|
|
||
|
|
|
||
|
|
class BytesIOPayload(IOBasePayload):
|
||
|
|
|
||
|
|
@property
|
||
|
|
def size(self) -> int:
|
||
|
|
position = self._value.tell()
|
||
|
|
end = self._value.seek(0, os.SEEK_END)
|
||
|
|
self._value.seek(position)
|
||
|
|
return end - position
|
||
|
|
|
||
|
|
|
||
|
|
class BufferedReaderPayload(IOBasePayload):
|
||
|
|
|
||
|
|
@property
|
||
|
|
def size(self) -> Optional[int]:
|
||
|
|
try:
|
||
|
|
return os.fstat(self._value.fileno()).st_size - self._value.tell()
|
||
|
|
except OSError:
|
||
|
|
# data.fileno() is not supported, e.g.
|
||
|
|
# io.BufferedReader(io.BytesIO(b'data'))
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
class JsonPayload(BytesPayload):
|
||
|
|
|
||
|
|
def __init__(self,
|
||
|
|
value: Any,
|
||
|
|
encoding: str='utf-8',
|
||
|
|
content_type: str='application/json',
|
||
|
|
dumps: JSONEncoder=json.dumps,
|
||
|
|
*args: Any,
|
||
|
|
**kwargs: Any) -> None:
|
||
|
|
|
||
|
|
super().__init__(
|
||
|
|
dumps(value).encode(encoding),
|
||
|
|
content_type=content_type, encoding=encoding, *args, **kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
if TYPE_CHECKING: # pragma: no cover
|
||
|
|
from typing import AsyncIterator, AsyncIterable
|
||
|
|
|
||
|
|
_AsyncIterator = AsyncIterator[bytes]
|
||
|
|
_AsyncIterable = AsyncIterable[bytes]
|
||
|
|
else:
|
||
|
|
from collections.abc import AsyncIterable, AsyncIterator
|
||
|
|
|
||
|
|
_AsyncIterator = AsyncIterator
|
||
|
|
_AsyncIterable = AsyncIterable
|
||
|
|
|
||
|
|
|
||
|
|
class AsyncIterablePayload(Payload):
|
||
|
|
|
||
|
|
_iter = None # type: Optional[_AsyncIterator]
|
||
|
|
|
||
|
|
def __init__(self,
|
||
|
|
value: _AsyncIterable,
|
||
|
|
*args: Any,
|
||
|
|
**kwargs: Any) -> None:
|
||
|
|
if not isinstance(value, AsyncIterable):
|
||
|
|
raise TypeError("value argument must support "
|
||
|
|
"collections.abc.AsyncIterablebe interface, "
|
||
|
|
"got {!r}".format(type(value)))
|
||
|
|
|
||
|
|
if 'content_type' not in kwargs:
|
||
|
|
kwargs['content_type'] = 'application/octet-stream'
|
||
|
|
|
||
|
|
super().__init__(value, *args, **kwargs)
|
||
|
|
|
||
|
|
self._iter = value.__aiter__()
|
||
|
|
|
||
|
|
async def write(self, writer: AbstractStreamWriter) -> None:
|
||
|
|
if self._iter:
|
||
|
|
try:
|
||
|
|
# iter is not None check prevents rare cases
|
||
|
|
# when the case iterable is used twice
|
||
|
|
while True:
|
||
|
|
chunk = await self._iter.__anext__()
|
||
|
|
await writer.write(chunk)
|
||
|
|
except StopAsyncIteration:
|
||
|
|
self._iter = None
|
||
|
|
|
||
|
|
|
||
|
|
class StreamReaderPayload(AsyncIterablePayload):
|
||
|
|
|
||
|
|
def __init__(self, value: StreamReader, *args: Any, **kwargs: Any) -> None:
|
||
|
|
super().__init__(value.iter_any(), *args, **kwargs)
|
||
|
|
|
||
|
|
|
||
|
|
PAYLOAD_REGISTRY = PayloadRegistry()
|
||
|
|
PAYLOAD_REGISTRY.register(BytesPayload, (bytes, bytearray, memoryview))
|
||
|
|
PAYLOAD_REGISTRY.register(StringPayload, str)
|
||
|
|
PAYLOAD_REGISTRY.register(StringIOPayload, io.StringIO)
|
||
|
|
PAYLOAD_REGISTRY.register(TextIOPayload, io.TextIOBase)
|
||
|
|
PAYLOAD_REGISTRY.register(BytesIOPayload, io.BytesIO)
|
||
|
|
PAYLOAD_REGISTRY.register(
|
||
|
|
BufferedReaderPayload, (io.BufferedReader, io.BufferedRandom))
|
||
|
|
PAYLOAD_REGISTRY.register(IOBasePayload, io.IOBase)
|
||
|
|
PAYLOAD_REGISTRY.register(StreamReaderPayload, StreamReader)
|
||
|
|
# try_last for giving a chance to more specialized async interables like
|
||
|
|
# multidict.BodyPartReaderPayload override the default
|
||
|
|
PAYLOAD_REGISTRY.register(AsyncIterablePayload, AsyncIterable,
|
||
|
|
order=Order.try_last)
|