From 993c9498c4276836864b01f66f5d08676a994520 Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Thu, 14 Mar 2024 13:53:57 +0100 Subject: Add webview support to OpenaiChat --- g4f/requests/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'g4f/requests') diff --git a/g4f/requests/__init__.py b/g4f/requests/__init__.py index f2946fc1..3bb68e9c 100644 --- a/g4f/requests/__init__.py +++ b/g4f/requests/__init__.py @@ -81,22 +81,25 @@ def get_session_from_browser(url: str, webdriver: WebDriver = None, proxy: str = impersonate="chrome" ) +def is_cloudflare(text: str): + return '
' in text or "Just a moment..." in text + async def raise_for_status_async(response: Union[StreamResponse, ClientResponse], message: str = None): if response.status in (429, 402): raise RateLimitError(f"Response {response.status}: Rate limit reached") message = await response.text() if not response.ok and message is None else message - if response.status == 403 and "Just a moment..." in message: + if response.status == 403 and is_cloudflare(message): raise ResponseStatusError(f"Response {response.status}: Cloudflare detected") elif not response.ok: raise ResponseStatusError(f"Response {response.status}: {message}") def raise_for_status(response: Union[StreamResponse, ClientResponse, Response, RequestsResponse], message: str = None): - if isinstance(response, StreamSession) or isinstance(response, ClientResponse): + if hasattr(response, "status"): return raise_for_status_async(response, message) if response.status_code in (429, 402): raise RateLimitError(f"Response {response.status_code}: Rate limit reached") - elif response.status_code == 403 and "Just a moment..." in response.text: + elif response.status_code == 403 and is_cloudflare(response.text): raise ResponseStatusError(f"Response {response.status_code}: Cloudflare detected") elif not response.ok: raise ResponseStatusError(f"Response {response.status_code}: {response.text if message is None else message}") \ No newline at end of file -- cgit v1.2.3 From 8cc6000ffbf4e12bf6c1d5e5878d376e36857ec0 Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Fri, 15 Mar 2024 11:46:06 +0100 Subject: Add Pyinstaller support, Use curl_cffi in You provider --- g4f/requests/__init__.py | 31 ++++++++++++++++++++++++++++++- g4f/requests/aiohttp.py | 8 ++++++-- g4f/requests/defaults.py | 10 ++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) (limited to 'g4f/requests') diff --git a/g4f/requests/__init__.py b/g4f/requests/__init__.py index 3bb68e9c..cfc6af42 100644 --- a/g4f/requests/__init__.py +++ b/g4f/requests/__init__.py @@ -12,11 +12,40 @@ except ImportError: from typing import Type as Session, Type as Response from .aiohttp import StreamResponse, StreamSession has_curl_cffi = False +try: + import webview + import asyncio + has_webview = True +except ImportError: + has_webview = False from ..webdriver import WebDriver, WebDriverSession from ..webdriver import bypass_cloudflare, get_driver_cookies from ..errors import MissingRequirementsError, RateLimitError, ResponseStatusError -from .defaults import DEFAULT_HEADERS +from .defaults import DEFAULT_HEADERS, WEBVIEW_HAEDERS + +async def get_args_from_webview(url: str): + if not has_webview: + raise MissingRequirementsError('Install "webview" package') + window = webview.create_window("", url, hidden=True) + await asyncio.sleep(2) + body = None + while body is None: + try: + await asyncio.sleep(1) + body = window.dom.get_element("body:not(.no-js)") + except: + ... + headers = { + **WEBVIEW_HAEDERS, + "User-Agent": window.evaluate_js("this.navigator.userAgent"), + "Accept-Language": window.evaluate_js("this.navigator.language"), + "Referer": window.real_url + } + cookies = [list(*cookie.items()) for cookie in window.get_cookies()] + cookies = dict([(name, cookie.value) for name, cookie in cookies]) + window.destroy() + return {"headers": headers, "cookies": cookies} def get_args_from_browser( url: str, diff --git a/g4f/requests/aiohttp.py b/g4f/requests/aiohttp.py index 6979b20a..505086a1 100644 --- a/g4f/requests/aiohttp.py +++ b/g4f/requests/aiohttp.py @@ -1,6 +1,6 @@ from __future__ import annotations -from aiohttp import ClientSession, ClientResponse, ClientTimeout, BaseConnector +from aiohttp import ClientSession, ClientResponse, ClientTimeout, BaseConnector, FormData from typing import AsyncIterator, Any, Optional from .defaults import DEFAULT_HEADERS @@ -43,4 +43,8 @@ def get_connector(connector: BaseConnector = None, proxy: str = None, rdns: bool connector = ProxyConnector.from_url(proxy, rdns=rdns) except ImportError: raise MissingRequirementsError('Install "aiohttp_socks" package for proxy support') - return connector \ No newline at end of file + return connector + +class CurlMime(FormData): + def addpart(self, name: str, content_type: str = None, filename: str = None, data: bytes = None): + self.add_field(name, data, content_type=content_type, filename=filename) \ No newline at end of file diff --git a/g4f/requests/defaults.py b/g4f/requests/defaults.py index 2457f046..3183eb5a 100644 --- a/g4f/requests/defaults.py +++ b/g4f/requests/defaults.py @@ -16,4 +16,14 @@ DEFAULT_HEADERS = { "referer": "", "accept-encoding": "gzip, deflate, br", "accept-language": "en-US", +} +WEBVIEW_HAEDERS = { + "Accept": "*/*", + "Accept-Encoding": "gzip, deflate, br", + "Accept-Language": "", + "Referer": "", + "Sec-Fetch-Dest": "empty", + "Sec-Fetch-Mode": "cors", + "Sec-Fetch-Site": "same-origin", + "User-Agent": "", } \ No newline at end of file -- cgit v1.2.3 From b7eee50930dbd782d7c068d1d29cd270b97bc741 Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Fri, 15 Mar 2024 14:55:53 +0100 Subject: Move raise_for_status, Create FormData wrapper --- g4f/requests/__init__.py | 38 ++++++-------------------------------- g4f/requests/aiohttp.py | 6 +----- g4f/requests/curl_cffi.py | 8 +++++++- g4f/requests/raise_for_status.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 38 deletions(-) create mode 100644 g4f/requests/raise_for_status.py (limited to 'g4f/requests') diff --git a/g4f/requests/__init__.py b/g4f/requests/__init__.py index cfc6af42..e65de99a 100644 --- a/g4f/requests/__init__.py +++ b/g4f/requests/__init__.py @@ -1,16 +1,12 @@ from __future__ import annotations -from typing import Union -from aiohttp import ClientResponse -from requests import Response as RequestsResponse - try: from curl_cffi.requests import Session, Response - from .curl_cffi import StreamResponse, StreamSession + from .curl_cffi import StreamResponse, StreamSession, FormData has_curl_cffi = True except ImportError: from typing import Type as Session, Type as Response - from .aiohttp import StreamResponse, StreamSession + from .aiohttp import StreamResponse, StreamSession, FormData has_curl_cffi = False try: import webview @@ -19,12 +15,13 @@ try: except ImportError: has_webview = False +from .raise_for_status import raise_for_status from ..webdriver import WebDriver, WebDriverSession from ..webdriver import bypass_cloudflare, get_driver_cookies -from ..errors import MissingRequirementsError, RateLimitError, ResponseStatusError +from ..errors import MissingRequirementsError from .defaults import DEFAULT_HEADERS, WEBVIEW_HAEDERS -async def get_args_from_webview(url: str): +async def get_args_from_webview(url: str) -> dict: if not has_webview: raise MissingRequirementsError('Install "webview" package') window = webview.create_window("", url, hidden=True) @@ -108,27 +105,4 @@ def get_session_from_browser(url: str, webdriver: WebDriver = None, proxy: str = proxies={"https": proxy, "http": proxy}, timeout=timeout, impersonate="chrome" - ) - -def is_cloudflare(text: str): - return '
' in text or "Just a moment..." in text - -async def raise_for_status_async(response: Union[StreamResponse, ClientResponse], message: str = None): - if response.status in (429, 402): - raise RateLimitError(f"Response {response.status}: Rate limit reached") - message = await response.text() if not response.ok and message is None else message - if response.status == 403 and is_cloudflare(message): - raise ResponseStatusError(f"Response {response.status}: Cloudflare detected") - elif not response.ok: - raise ResponseStatusError(f"Response {response.status}: {message}") - -def raise_for_status(response: Union[StreamResponse, ClientResponse, Response, RequestsResponse], message: str = None): - if hasattr(response, "status"): - return raise_for_status_async(response, message) - - if response.status_code in (429, 402): - raise RateLimitError(f"Response {response.status_code}: Rate limit reached") - elif response.status_code == 403 and is_cloudflare(response.text): - raise ResponseStatusError(f"Response {response.status_code}: Cloudflare detected") - elif not response.ok: - raise ResponseStatusError(f"Response {response.status_code}: {response.text if message is None else message}") \ No newline at end of file + ) \ No newline at end of file diff --git a/g4f/requests/aiohttp.py b/g4f/requests/aiohttp.py index 505086a1..16b052eb 100644 --- a/g4f/requests/aiohttp.py +++ b/g4f/requests/aiohttp.py @@ -43,8 +43,4 @@ def get_connector(connector: BaseConnector = None, proxy: str = None, rdns: bool connector = ProxyConnector.from_url(proxy, rdns=rdns) except ImportError: raise MissingRequirementsError('Install "aiohttp_socks" package for proxy support') - return connector - -class CurlMime(FormData): - def addpart(self, name: str, content_type: str = None, filename: str = None, data: bytes = None): - self.add_field(name, data, content_type=content_type, filename=filename) \ No newline at end of file + return connector \ No newline at end of file diff --git a/g4f/requests/curl_cffi.py b/g4f/requests/curl_cffi.py index cfcdd63b..91142365 100644 --- a/g4f/requests/curl_cffi.py +++ b/g4f/requests/curl_cffi.py @@ -1,6 +1,6 @@ from __future__ import annotations -from curl_cffi.requests import AsyncSession, Response +from curl_cffi.requests import AsyncSession, Response, CurlMime from typing import AsyncGenerator, Any from functools import partialmethod import json @@ -65,6 +65,8 @@ class StreamSession(AsyncSession): def request( self, method: str, url: str, **kwargs ) -> StreamResponse: + if isinstance(kwargs.get("data"), CurlMime): + kwargs["multipart"] = kwargs.pop("data") """Create and return a StreamResponse object for the given HTTP request.""" return StreamResponse(super().request(method, url, stream=True, **kwargs)) @@ -75,3 +77,7 @@ class StreamSession(AsyncSession): put = partialmethod(request, "PUT") patch = partialmethod(request, "PATCH") delete = partialmethod(request, "DELETE") + +class FormData(CurlMime): + def add_field(self, name, data=None, content_type: str = None, filename: str = None) -> None: + self.addpart(name, content_type=content_type, filename=filename, data=data) \ No newline at end of file diff --git a/g4f/requests/raise_for_status.py b/g4f/requests/raise_for_status.py new file mode 100644 index 00000000..9e8e141c --- /dev/null +++ b/g4f/requests/raise_for_status.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import Union +from aiohttp import ClientResponse +from requests import Response as RequestsResponse + +from ..errors import ResponseStatusError, RateLimitError +from . import Response, StreamResponse + +class CloudflareError(ResponseStatusError): + ... + +def is_cloudflare(text: str) -> bool: + return '
' in text or "Just a moment..." in text + +async def raise_for_status_async(response: Union[StreamResponse, ClientResponse], message: str = None): + if response.status in (429, 402): + raise RateLimitError(f"Response {response.status}: Rate limit reached") + message = await response.text() if not response.ok and message is None else message + if response.status == 403 and is_cloudflare(message): + raise CloudflareError(f"Response {response.status}: Cloudflare detected") + elif not response.ok: + raise ResponseStatusError(f"Response {response.status}: {message}") + +def raise_for_status(response: Union[Response, StreamResponse, ClientResponse, RequestsResponse], message: str = None): + if hasattr(response, "status"): + return raise_for_status_async(response, message) + + if response.status_code in (429, 402): + raise RateLimitError(f"Response {response.status_code}: Rate limit reached") + elif response.status_code == 403 and is_cloudflare(response.text): + raise CloudflareError(f"Response {response.status_code}: Cloudflare detected") + elif not response.ok: + raise ResponseStatusError(f"Response {response.status_code}: {response.text if message is None else message}") \ No newline at end of file -- cgit v1.2.3