summaryrefslogtreecommitdiffstats
path: root/g4f/Provider
diff options
context:
space:
mode:
authorHeiner Lohaus <hlohaus@users.noreply.github.com>2024-05-18 07:37:37 +0200
committerHeiner Lohaus <hlohaus@users.noreply.github.com>2024-05-18 07:37:37 +0200
commitb1dafc0ef79bdd94f69c783877217d8a5524d460 (patch)
tree7851c85bafa452d0fa9cce74aa7b383aa704f101 /g4f/Provider
parentMerge pull request #1969 from hlohaus/leech (diff)
downloadgpt4free-b1dafc0ef79bdd94f69c783877217d8a5524d460.tar
gpt4free-b1dafc0ef79bdd94f69c783877217d8a5524d460.tar.gz
gpt4free-b1dafc0ef79bdd94f69c783877217d8a5524d460.tar.bz2
gpt4free-b1dafc0ef79bdd94f69c783877217d8a5524d460.tar.lz
gpt4free-b1dafc0ef79bdd94f69c783877217d8a5524d460.tar.xz
gpt4free-b1dafc0ef79bdd94f69c783877217d8a5524d460.tar.zst
gpt4free-b1dafc0ef79bdd94f69c783877217d8a5524d460.zip
Diffstat (limited to 'g4f/Provider')
-rw-r--r--g4f/Provider/Liaobots.py79
-rw-r--r--g4f/Provider/needs_auth/Gemini.py36
-rw-r--r--g4f/Provider/needs_auth/OpenaiChat.py12
-rw-r--r--g4f/Provider/openai/har_file.py30
-rw-r--r--g4f/Provider/openai/proofofwork.py48
5 files changed, 134 insertions, 71 deletions
diff --git a/g4f/Provider/Liaobots.py b/g4f/Provider/Liaobots.py
index deb7899c..75ecf300 100644
--- a/g4f/Provider/Liaobots.py
+++ b/g4f/Provider/Liaobots.py
@@ -10,6 +10,15 @@ from .helper import get_connector
from ..requests import raise_for_status
models = {
+ "gpt-4o": {
+ "context": "8K",
+ "id": "gpt-4o-free",
+ "maxLength": 31200,
+ "model": "ChatGPT",
+ "name": "GPT-4o-free",
+ "provider": "OpenAI",
+ "tokenLimit": 7800,
+ },
"gpt-3.5-turbo": {
"id": "gpt-3.5-turbo",
"name": "GPT-3.5-Turbo",
@@ -95,7 +104,7 @@ class Liaobots(AsyncGeneratorProvider, ProviderModelMixin):
model_aliases = {
"claude-v2": "claude-2"
}
- _auth_code = None
+ _auth_code = ""
_cookie_jar = None
@classmethod
@@ -120,7 +129,13 @@ class Liaobots(AsyncGeneratorProvider, ProviderModelMixin):
cookie_jar=cls._cookie_jar,
connector=get_connector(connector, proxy, True)
) as session:
- cls._auth_code = auth if isinstance(auth, str) else cls._auth_code
+ data = {
+ "conversationId": str(uuid.uuid4()),
+ "model": models[cls.get_model(model)],
+ "messages": messages,
+ "key": "",
+ "prompt": kwargs.get("system_message", "You are a helpful assistant."),
+ }
if not cls._auth_code:
async with session.post(
"https://liaobots.work/recaptcha/api/login",
@@ -128,31 +143,49 @@ class Liaobots(AsyncGeneratorProvider, ProviderModelMixin):
verify_ssl=False
) as response:
await raise_for_status(response)
+ try:
async with session.post(
"https://liaobots.work/api/user",
- json={"authcode": ""},
+ json={"authcode": cls._auth_code},
verify_ssl=False
) as response:
await raise_for_status(response)
cls._auth_code = (await response.json(content_type=None))["authCode"]
+ if not cls._auth_code:
+ raise RuntimeError("Empty auth code")
cls._cookie_jar = session.cookie_jar
-
- data = {
- "conversationId": str(uuid.uuid4()),
- "model": models[cls.get_model(model)],
- "messages": messages,
- "key": "",
- "prompt": kwargs.get("system_message", "You are a helpful assistant."),
- }
- async with session.post(
- "https://liaobots.work/api/chat",
- json=data,
- headers={"x-auth-code": cls._auth_code},
- verify_ssl=False
- ) as response:
- await raise_for_status(response)
- async for chunk in response.content.iter_any():
- if b"<html coupert-item=" in chunk:
- raise RuntimeError("Invalid session")
- if chunk:
- yield chunk.decode(errors="ignore")
+ async with session.post(
+ "https://liaobots.work/api/chat",
+ json=data,
+ headers={"x-auth-code": cls._auth_code},
+ verify_ssl=False
+ ) as response:
+ await raise_for_status(response)
+ async for chunk in response.content.iter_any():
+ if b"<html coupert-item=" in chunk:
+ raise RuntimeError("Invalid session")
+ if chunk:
+ yield chunk.decode(errors="ignore")
+ except:
+ async with session.post(
+ "https://liaobots.work/api/user",
+ json={"authcode": "pTIQr4FTnVRfr"},
+ verify_ssl=False
+ ) as response:
+ await raise_for_status(response)
+ cls._auth_code = (await response.json(content_type=None))["authCode"]
+ if not cls._auth_code:
+ raise RuntimeError("Empty auth code")
+ cls._cookie_jar = session.cookie_jar
+ async with session.post(
+ "https://liaobots.work/api/chat",
+ json=data,
+ headers={"x-auth-code": cls._auth_code},
+ verify_ssl=False
+ ) as response:
+ await raise_for_status(response)
+ async for chunk in response.content.iter_any():
+ if b"<html coupert-item=" in chunk:
+ raise RuntimeError("Invalid session")
+ if chunk:
+ yield chunk.decode(errors="ignore")
diff --git a/g4f/Provider/needs_auth/Gemini.py b/g4f/Provider/needs_auth/Gemini.py
index e468f64a..f9b1c4a5 100644
--- a/g4f/Provider/needs_auth/Gemini.py
+++ b/g4f/Provider/needs_auth/Gemini.py
@@ -4,6 +4,7 @@ import os
import json
import random
import re
+import base64
from aiohttp import ClientSession, BaseConnector
@@ -22,7 +23,7 @@ from ..base_provider import AsyncGeneratorProvider
from ..helper import format_prompt, get_cookies
from ...requests.raise_for_status import raise_for_status
from ...errors import MissingAuthError, MissingRequirementsError
-from ...image import to_bytes, to_data_uri, ImageResponse
+from ...image import to_bytes, ImageResponse, ImageDataResponse
from ...webdriver import get_browser, get_driver_cookies
REQUEST_HEADERS = {
@@ -122,6 +123,7 @@ class Gemini(AsyncGeneratorProvider):
connector: BaseConnector = None,
image: ImageType = None,
image_name: str = None,
+ response_format: str = None,
**kwargs
) -> AsyncResult:
prompt = format_prompt(messages)
@@ -192,22 +194,22 @@ class Gemini(AsyncGeneratorProvider):
if image_prompt:
images = [image[0][3][3] for image in response_part[4][0][12][7][0]]
resolved_images = []
- preview = []
- for image in images:
- async with client.get(image, allow_redirects=False) as fetch:
- image = fetch.headers["location"]
- async with client.get(image, allow_redirects=False) as fetch:
- image = fetch.headers["location"]
- resolved_images.append(image)
- preview.append(image.replace('=s512', '=s200'))
- # preview_url = image.replace('=s512', '=s200')
- # async with client.get(preview_url) as fetch:
- # preview_data = to_data_uri(await fetch.content.read())
- # async with client.get(image) as fetch:
- # data = to_data_uri(await fetch.content.read())
- # preview.append(preview_data)
- # resolved_images.append(data)
- yield ImageResponse(resolved_images, image_prompt, {"orginal_links": images, "preview": preview})
+ if response_format == "b64_json":
+ for image in images:
+ async with client.get(image) as response:
+ data = base64.b64encode(await response.content.read()).decode()
+ resolved_images.append(data)
+ yield ImageDataResponse(resolved_images, image_prompt)
+ else:
+ preview = []
+ for image in images:
+ async with client.get(image, allow_redirects=False) as fetch:
+ image = fetch.headers["location"]
+ async with client.get(image, allow_redirects=False) as fetch:
+ image = fetch.headers["location"]
+ resolved_images.append(image)
+ preview.append(image.replace('=s512', '=s200'))
+ yield ImageResponse(resolved_images, image_prompt, {"orginal_links": images, "preview": preview})
def build_request(
prompt: str,
diff --git a/g4f/Provider/needs_auth/OpenaiChat.py b/g4f/Provider/needs_auth/OpenaiChat.py
index b4b8bb02..28d0558b 100644
--- a/g4f/Provider/needs_auth/OpenaiChat.py
+++ b/g4f/Provider/needs_auth/OpenaiChat.py
@@ -61,7 +61,7 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):
supports_system_message = True
default_model = None
default_vision_model = "gpt-4o"
- models = ["gpt-3.5-turbo", "gpt-4", "gpt-4-gizmo", "gpt-4o"]
+ models = ["gpt-3.5-turbo", "gpt-4", "gpt-4-gizmo", "gpt-4o", "auto"]
model_aliases = {
"text-davinci-002-render-sha": "gpt-3.5-turbo",
"": "gpt-3.5-turbo",
@@ -394,10 +394,11 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):
print(f"{e.__class__.__name__}: {e}")
arkose_token = None
+ proofTokens = None
if cls.default_model is None:
error = None
try:
- arkose_token, api_key, cookies, headers = await getArkoseAndAccessToken(proxy)
+ arkose_token, api_key, cookies, headers, proofTokens = await getArkoseAndAccessToken(proxy)
cls._create_request_args(cookies, headers)
cls._set_api_key(api_key)
except NoValidHarFileError as e:
@@ -413,17 +414,17 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):
if cls._api_key is None else
f"{cls.url}/backend-api/sentinel/chat-requirements",
json={"conversation_mode_kind": "primary_assistant"},
+ #json={"p": generate_proof_token(True, user_agent=cls._headers["user-agent"], proofTokens=proofTokens)},
headers=cls._headers
) as response:
cls._update_request_args(session)
await raise_for_status(response)
data = await response.json()
- blob = data["arkose"]["dx"]
- need_arkose = data["arkose"]["required"]
+ need_arkose = data.get("arkose", {}).get("required")
chat_token = data["token"]
proofofwork = ""
if "proofofwork" in data:
- proofofwork = generate_proof_token(**data["proofofwork"], user_agent=cls._headers["user-agent"])
+ proofofwork = generate_proof_token(**data["proofofwork"], user_agent=cls._headers["user-agent"], proofTokens=proofTokens)
if need_arkose and arkose_token is None:
arkose_token, api_key, cookies, headers = await getArkoseAndAccessToken(proxy)
@@ -435,7 +436,6 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin):
if debug.logging:
print(
'Arkose:', False if not need_arkose else arkose_token[:12]+"...",
- 'Turnstile:', data["turnstile"]["required"],
'Proofofwork:', False if proofofwork is None else proofofwork[:12]+"...",
)
diff --git a/g4f/Provider/openai/har_file.py b/g4f/Provider/openai/har_file.py
index 220c20bf..eefe305f 100644
--- a/g4f/Provider/openai/har_file.py
+++ b/g4f/Provider/openai/har_file.py
@@ -12,6 +12,7 @@ from copy import deepcopy
from .crypt import decrypt, encrypt
from ...requests import StreamSession
+from ... import debug
class NoValidHarFileError(Exception):
...
@@ -31,6 +32,7 @@ chatArk: arkReq = None
accessToken: str = None
cookies: dict = None
headers: dict = None
+proofTokens: list = []
def readHAR():
dirPath = "./"
@@ -54,6 +56,15 @@ def readHAR():
# Error: not a HAR file!
continue
for v in harFile['log']['entries']:
+ v_headers = get_headers(v)
+ try:
+ if "openai-sentinel-proof-token" in v_headers:
+ proofTokens.append(json.loads(base64.b64decode(
+ v_headers["openai-sentinel-proof-token"].split("gAAAAAB", 1)[-1].encode()
+ ).decode()))
+ except Exception as e:
+ if debug.logging:
+ print(f"Read proof token: {e}")
if arkPreURL in v['request']['url']:
chatArks.append(parseHAREntry(v))
elif v['request']['url'] == sessionUrl:
@@ -61,13 +72,13 @@ def readHAR():
accessToken = json.loads(v["response"]["content"]["text"]).get("accessToken")
except KeyError:
continue
- cookies = {c['name']: c['value'] for c in v['request']['cookies']}
- headers = get_headers(v)
+ cookies = {c['name']: c['value'] for c in v['request']['cookies'] if c['name'] != "oai-did"}
+ headers = v_headers
if not accessToken:
raise NoValidHarFileError("No accessToken found in .har files")
if not chatArks:
- return None, accessToken, cookies, headers
- return chatArks.pop(), accessToken, cookies, headers
+ return None, accessToken, cookies, headers, proofTokens
+ return chatArks.pop(), accessToken, cookies, headers, proofTokens
def get_headers(entry) -> dict:
return {h['name'].lower(): h['value'] for h in entry['request']['headers'] if h['name'].lower() not in ['content-length', 'cookie'] and not h['name'].startswith(':')}
@@ -101,7 +112,8 @@ def genArkReq(chatArk: arkReq) -> arkReq:
async def sendRequest(tmpArk: arkReq, proxy: str = None):
async with StreamSession(headers=tmpArk.arkHeader, cookies=tmpArk.arkCookies, proxies={"https": proxy}) as session:
async with session.post(tmpArk.arkURL, data=tmpArk.arkBody) as response:
- arkose = (await response.json()).get("token")
+ data = await response.json()
+ arkose = data.get("token")
if "sup=1|rid=" not in arkose:
return RuntimeError("No valid arkose token generated")
return arkose
@@ -131,10 +143,10 @@ def getN() -> str:
return base64.b64encode(timestamp.encode()).decode()
async def getArkoseAndAccessToken(proxy: str) -> tuple[str, str, dict, dict]:
- global chatArk, accessToken, cookies, headers
+ global chatArk, accessToken, cookies, headers, proofTokens
if chatArk is None or accessToken is None:
- chatArk, accessToken, cookies, headers = readHAR()
+ chatArk, accessToken, cookies, headers, proofTokens = readHAR()
if chatArk is None:
- return None, accessToken, cookies, headers
+ return None, accessToken, cookies, headers, proofTokens
newReq = genArkReq(chatArk)
- return await sendRequest(newReq, proxy), accessToken, cookies, headers
+ return await sendRequest(newReq, proxy), accessToken, cookies, headers, proofTokens
diff --git a/g4f/Provider/openai/proofofwork.py b/g4f/Provider/openai/proofofwork.py
index 51d96bc4..cbce153f 100644
--- a/g4f/Provider/openai/proofofwork.py
+++ b/g4f/Provider/openai/proofofwork.py
@@ -2,35 +2,51 @@ import random
import hashlib
import json
import base64
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone
-def generate_proof_token(required: bool, seed: str, difficulty: str, user_agent: str):
+proof_token_cache: dict = {}
+
+def generate_proof_token(required: bool, seed: str = None, difficulty: str = None, user_agent: str = None, proofTokens: list = None):
if not required:
return
-
- cores = [8, 12, 16, 24]
- screens = [3000, 4000, 6000]
-
- core = random.choice(cores)
- screen = random.choice(screens)
+ if seed is not None and seed in proof_token_cache:
+ return proof_token_cache[seed]
# Get current UTC time
now_utc = datetime.now(timezone.utc)
parse_time = now_utc.strftime('%a, %d %b %Y %H:%M:%S GMT')
- config = [core + screen, parse_time, None, 0, user_agent, "https://tcr9i.chat.openai.com/v2/35536E1E-65B4-4D96-9D97-6ADB7EFF8147/api.js","dpl=53d243de46ff04dadd88d293f088c2dd728f126f","en","en-US",442,"plugins−[object PluginArray]","","alert"]
-
- diff_len = len(difficulty) // 2
-
+ if proofTokens:
+ config = random.choice(proofTokens)
+ else:
+ screen = random.choice([3008, 4010, 6000]) * random.choice([1, 2, 4])
+ config = [
+ screen, parse_time,
+ None, 0, user_agent,
+ "https://tcr9i.chat.openai.com/v2/35536E1E-65B4-4D96-9D97-6ADB7EFF8147/api.js",
+ "dpl=1440a687921de39ff5ee56b92807faaadce73f13","en","en-US",
+ None,
+ "plugins−[object PluginArray]",
+ random.choice(["_reactListeningcfilawjnerp", "_reactListening9ne2dfo1i47", "_reactListening410nzwhan2a"]),
+ random.choice(["alert", "ontransitionend", "onprogress"])
+ ]
+
+ config[1] = parse_time
+ config[4] = user_agent
+ config[7] = random.randint(101, 2100)
+
+ diff_len = None if difficulty is None else len(difficulty)
for i in range(100000):
config[3] = i
json_data = json.dumps(config)
base = base64.b64encode(json_data.encode()).decode()
- hash_value = hashlib.sha3_512((seed + base).encode()).digest()
+ hash_value = hashlib.sha3_512((seed or "" + base).encode()).digest()
- if hash_value.hex()[:diff_len] <= difficulty:
- result = "gAAAAAB" + base
- return result
+ if difficulty is None or hash_value.hex()[:diff_len] <= difficulty:
+ if seed is None:
+ return "gAAAAAC" + base
+ proof_token_cache[seed] = "gAAAAAB" + base
+ return proof_token_cache[seed]
fallback_base = base64.b64encode(f'"{seed}"'.encode()).decode()
return "gAAAAABwQ8Lk5FbGpA2NcR9dShT6gYjU7VxZ4D" + fallback_base
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-05 01:49:01 +0200