From 6ce493d4dfc2884832ff5b5be4479a55818b2fe7 Mon Sep 17 00:00:00 2001
From: H Lohaus <hlohaus@users.noreply.github.com>
Date: Sat, 16 Nov 2024 13:19:51 +0100
Subject: Fix api streaming, fix AsyncClient (#2357)

* Fix api streaming, fix AsyncClient, Improve Client class, Some providers fixes, Update models list, Fix some tests, Update model list in Airforce provid
er, Add OpenAi image generation url to api, Fix reload and debug in api arguments, Fix websearch in gui

* Fix Cloadflare and Pi and AmigoChat provider

* Fix conversation support in DDG provider, Add cloudflare bypass with nodriver

* Fix unittests without curl_cffi
---
 g4f/requests/__init__.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 61 insertions(+), 1 deletion(-)

(limited to 'g4f/requests/__init__.py')

diff --git a/g4f/requests/__init__.py b/g4f/requests/__init__.py
index 80fc44b3..a8c0e286 100644
--- a/g4f/requests/__init__.py
+++ b/g4f/requests/__init__.py
@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+from urllib.parse import urlparse
+from typing import Iterator
+from http.cookies import Morsel
 try:
     from curl_cffi.requests import Session, Response
     from .curl_cffi import StreamResponse, StreamSession, FormData
@@ -14,11 +17,19 @@ try:
     has_webview = True
 except ImportError:
     has_webview = False
+try:
+    import nodriver
+    from nodriver.cdp.network import CookieParam
+    has_nodriver = True
+except ImportError:
+    has_nodriver = False
 
+from .. import debug
 from .raise_for_status import raise_for_status
 from ..webdriver import WebDriver, WebDriverSession
 from ..webdriver import bypass_cloudflare, get_driver_cookies
 from ..errors import MissingRequirementsError
+from ..typing import Cookies
 from .defaults import DEFAULT_HEADERS, WEBVIEW_HAEDERS
 
 async def get_args_from_webview(url: str) -> dict:
@@ -105,4 +116,53 @@ def get_session_from_browser(url: str, webdriver: WebDriver = None, proxy: str =
         proxies={"https": proxy, "http": proxy},
         timeout=timeout,
         impersonate="chrome"
-    )
\ No newline at end of file
+    )
+def get_cookie_params_from_dict(cookies: Cookies, url: str = None, domain: str = None) -> list[CookieParam]:
+    [CookieParam.from_json({
+        "name": key,
+        "value": value,
+        "url": url,
+        "domain": domain
+    }) for key, value in cookies.items()]
+
+async def get_args_from_nodriver(
+    url: str,
+    proxy: str = None,
+    timeout: int = 120,
+    cookies: Cookies = None
+) -> dict:
+    if not has_nodriver:
+        raise MissingRequirementsError('Install "nodriver" package | pip install -U nodriver')
+    if debug.logging:
+        print(f"Open nodriver with url: {url}")
+    browser = await nodriver.start(
+        browser_args=None if proxy is None else [f"--proxy-server={proxy}"],
+    )
+    domain = urlparse(url).netloc
+    if cookies is None:
+        cookies = {}
+    else:
+        await browser.cookies.set_all(get_cookie_params_from_dict(cookies, url=url, domain=domain))
+    page = await browser.get(url)
+    for c in await browser.cookies.get_all():
+        if c.domain.endswith(domain):
+            cookies[c.name] = c.value
+    user_agent = await page.evaluate("window.navigator.userAgent")
+    await page.wait_for("body:not(.no-js)", timeout=timeout)
+    await page.close()
+    browser.stop()
+    return {
+        "cookies": cookies,
+        "headers": {
+            **DEFAULT_HEADERS,
+            "user-agent": user_agent,
+            "referer": url,
+        },
+        "proxy": proxy
+    }
+
+def merge_cookies(cookies: Iterator[Morsel], response: Response) -> Cookies:
+    if cookies is None:
+        cookies = {}
+    for cookie in response.cookies.jar:
+        cookies[cookie.name] = cookie.value
\ No newline at end of file
-- 
cgit v1.2.3