From 3a23e81de93c4c9a83aa22b70ea13066f06541e3 Mon Sep 17 00:00:00 2001 From: Heiner Lohaus Date: Sun, 21 Apr 2024 22:39:00 +0200 Subject: Add Replicate Provider Fix Bug in OpenaiChat and MetaAI Add read cookie and .har files Use factory for api app --- g4f/Provider/MetaAI.py | 2 +- g4f/Provider/Replicate.py | 84 +++++++++++++++++++++++++++++++++++ g4f/Provider/__init__.py | 2 +- g4f/Provider/needs_auth/OpenaiChat.py | 15 ++++--- g4f/Provider/unfinished/Replicate.py | 78 -------------------------------- g4f/api/__init__.py | 46 ++++++++++--------- g4f/cli.py | 34 +++++++++----- g4f/cookies.py | 58 ++++++++++++++++++++++++ g4f/gui/__init__.py | 3 -- g4f/gui/gui_parser.py | 1 + g4f/gui/run.py | 6 +++ 11 files changed, 208 insertions(+), 121 deletions(-) create mode 100644 g4f/Provider/Replicate.py delete mode 100644 g4f/Provider/unfinished/Replicate.py (limited to 'g4f') diff --git a/g4f/Provider/MetaAI.py b/g4f/Provider/MetaAI.py index 045255e7..caed7778 100644 --- a/g4f/Provider/MetaAI.py +++ b/g4f/Provider/MetaAI.py @@ -89,7 +89,7 @@ class MetaAI(AsyncGeneratorProvider): headers = {} headers = { 'content-type': 'application/x-www-form-urlencoded', - 'cookie': format_cookies(cookies), + 'cookie': format_cookies(self.cookies), 'origin': 'https://www.meta.ai', 'referer': 'https://www.meta.ai/', 'x-asbd-id': '129477', diff --git a/g4f/Provider/Replicate.py b/g4f/Provider/Replicate.py new file mode 100644 index 00000000..593fd04d --- /dev/null +++ b/g4f/Provider/Replicate.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from .base_provider import AsyncGeneratorProvider, ProviderModelMixin +from .helper import format_prompt, filter_none +from ..typing import AsyncResult, Messages +from ..requests import raise_for_status +from ..requests.aiohttp import StreamSession +from ..errors import ResponseError, MissingAuthError + +class Replicate(AsyncGeneratorProvider, ProviderModelMixin): + url = "https://replicate.com" + working = True + default_model = "meta/meta-llama-3-70b-instruct" + + @classmethod + async def create_async_generator( + cls, + model: str, + messages: Messages, + api_key: str = None, + proxy: str = None, + timeout: int = 180, + system_prompt: str = None, + max_new_tokens: int = None, + temperature: float = None, + top_p: float = None, + top_k: float = None, + stop: list = None, + extra_data: dict = {}, + headers: dict = { + "accept": "application/json", + }, + **kwargs + ) -> AsyncResult: + model = cls.get_model(model) + if cls.needs_auth and api_key is None: + raise MissingAuthError("api_key is missing") + if api_key is not None: + headers["Authorization"] = f"Bearer {api_key}" + api_base = "https://api.replicate.com/v1/models/" + else: + api_base = "https://replicate.com/api/models/" + async with StreamSession( + proxy=proxy, + headers=headers, + timeout=timeout + ) as session: + data = { + "stream": True, + "input": { + "prompt": format_prompt(messages), + **filter_none( + system_prompt=system_prompt, + max_new_tokens=max_new_tokens, + temperature=temperature, + top_p=top_p, + top_k=top_k, + stop_sequences=",".join(stop) if stop else None + ), + **extra_data + }, + } + url = f"{api_base.rstrip('/')}/{model}/predictions" + async with session.post(url, json=data) as response: + message = "Model not found" if response.status == 404 else None + await raise_for_status(response, message) + result = await response.json() + if "id" not in result: + raise ResponseError(f"Invalid response: {result}") + async with session.get(result["urls"]["stream"], headers={"Accept": "text/event-stream"}) as response: + await raise_for_status(response) + event = None + async for line in response.iter_lines(): + if line.startswith(b"event: "): + event = line[7:] + if event == b"done": + break + elif event == b"output": + if line.startswith(b"data: "): + new_text = line[6:].decode() + if new_text: + yield new_text + else: + yield "\n" \ No newline at end of file diff --git a/g4f/Provider/__init__.py b/g4f/Provider/__init__.py index 27c14672..d2d9bfda 100644 --- a/g4f/Provider/__init__.py +++ b/g4f/Provider/__init__.py @@ -9,7 +9,6 @@ from .deprecated import * from .not_working import * from .selenium import * from .needs_auth import * -from .unfinished import * from .Aichatos import Aichatos from .Aura import Aura @@ -46,6 +45,7 @@ from .MetaAI import MetaAI from .MetaAIAccount import MetaAIAccount from .PerplexityLabs import PerplexityLabs from .Pi import Pi +from .Replicate import Replicate from .ReplicateImage import ReplicateImage from .Vercel import Vercel from .WhiteRabbitNeo import WhiteRabbitNeo diff --git a/g4f/Provider/needs_auth/OpenaiChat.py b/g4f/Provider/needs_auth/OpenaiChat.py index 7952d606..3d6e9858 100644 --- a/g4f/Provider/needs_auth/OpenaiChat.py +++ b/g4f/Provider/needs_auth/OpenaiChat.py @@ -340,9 +340,8 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin): Raises: RuntimeError: If an error occurs during processing. """ - async with StreamSession( - proxies={"all": proxy}, + proxy=proxy, impersonate="chrome", timeout=timeout ) as session: @@ -364,26 +363,27 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin): api_key = cls._api_key = None cls._create_request_args() if debug.logging: - print("OpenaiChat: Load default_model failed") + print("OpenaiChat: Load default model failed") print(f"{e.__class__.__name__}: {e}") arkose_token = None if cls.default_model is None: + error = None try: arkose_token, api_key, cookies, headers = await getArkoseAndAccessToken(proxy) cls._create_request_args(cookies, headers) cls._set_api_key(api_key) except NoValidHarFileError as e: - ... + error = e if cls._api_key is None: await cls.nodriver_access_token() if cls._api_key is None and cls.needs_auth: - raise e + raise error cls.default_model = cls.get_model(await cls.get_default_model(session, cls._headers)) async with session.post( f"{cls.url}/backend-anon/sentinel/chat-requirements" - if not cls._api_key else + if cls._api_key is None else f"{cls.url}/backend-api/sentinel/chat-requirements", json={"conversation_mode_kind": "primary_assistant"}, headers=cls._headers @@ -412,7 +412,8 @@ class OpenaiChat(AsyncGeneratorProvider, ProviderModelMixin): print("OpenaiChat: Upload image failed") print(f"{e.__class__.__name__}: {e}") - model = cls.get_model(model).replace("gpt-3.5-turbo", "text-davinci-002-render-sha") + model = cls.get_model(model) + model = "text-davinci-002-render-sha" if model == "gpt-3.5-turbo" else model if conversation is None: conversation = Conversation(conversation_id, str(uuid.uuid4()) if parent_id is None else parent_id) else: diff --git a/g4f/Provider/unfinished/Replicate.py b/g4f/Provider/unfinished/Replicate.py deleted file mode 100644 index aaaf31b3..00000000 --- a/g4f/Provider/unfinished/Replicate.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -import asyncio - -from ..base_provider import AsyncGeneratorProvider, ProviderModelMixin -from ..helper import format_prompt, filter_none -from ...typing import AsyncResult, Messages -from ...requests import StreamSession, raise_for_status -from ...image import ImageResponse -from ...errors import ResponseError, MissingAuthError - -class Replicate(AsyncGeneratorProvider, ProviderModelMixin): - url = "https://replicate.com" - working = True - default_model = "mistralai/mixtral-8x7b-instruct-v0.1" - api_base = "https://api.replicate.com/v1/models/" - - @classmethod - async def create_async_generator( - cls, - model: str, - messages: Messages, - api_key: str = None, - proxy: str = None, - timeout: int = 180, - system_prompt: str = None, - max_new_tokens: int = None, - temperature: float = None, - top_p: float = None, - top_k: float = None, - stop: list = None, - extra_data: dict = {}, - headers: dict = {}, - **kwargs - ) -> AsyncResult: - model = cls.get_model(model) - if api_key is None: - raise MissingAuthError("api_key is missing") - headers["Authorization"] = f"Bearer {api_key}" - async with StreamSession( - proxies={"all": proxy}, - headers=headers, - timeout=timeout - ) as session: - data = { - "stream": True, - "input": { - "prompt": format_prompt(messages), - **filter_none( - system_prompt=system_prompt, - max_new_tokens=max_new_tokens, - temperature=temperature, - top_p=top_p, - top_k=top_k, - stop_sequences=",".join(stop) if stop else None - ), - **extra_data - }, - } - url = f"{cls.api_base.rstrip('/')}/{model}/predictions" - async with session.post(url, json=data) as response: - await raise_for_status(response) - result = await response.json() - if "id" not in result: - raise ResponseError(f"Invalid response: {result}") - async with session.get(result["urls"]["stream"], headers={"Accept": "text/event-stream"}) as response: - await raise_for_status(response) - event = None - async for line in response.iter_lines(): - if line.startswith(b"event: "): - event = line[7:] - elif event == b"output": - if line.startswith(b"data: "): - yield line[6:].decode() - elif not line.startswith(b"id: "): - continue#yield "+"+line.decode() - elif event == b"done": - break \ No newline at end of file diff --git a/g4f/api/__init__.py b/g4f/api/__init__.py index ed39fc58..aaac1827 100644 --- a/g4f/api/__init__.py +++ b/g4f/api/__init__.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import json import uvicorn @@ -8,14 +10,19 @@ from fastapi.exceptions import RequestValidationError from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY from fastapi.encoders import jsonable_encoder from pydantic import BaseModel -from typing import List, Union, Optional +from typing import Union, Optional import g4f import g4f.debug from g4f.client import AsyncClient from g4f.typing import Messages -app = FastAPI() +def create_app() -> FastAPI: + app = FastAPI() + api = Api(app) + api.register_routes() + api.register_validation_exception_handler() + return app class ChatCompletionsConfig(BaseModel): messages: Messages @@ -29,16 +36,19 @@ class ChatCompletionsConfig(BaseModel): web_search: Optional[bool] = None proxy: Optional[str] = None +list_ignored_providers: list[str] = None + +def set_list_ignored_providers(ignored: list[str]): + global list_ignored_providers + list_ignored_providers = ignored + class Api: - def __init__(self, list_ignored_providers: List[str] = None) -> None: - self.list_ignored_providers = list_ignored_providers + def __init__(self, app: FastAPI) -> None: + self.app = app self.client = AsyncClient() - - def set_list_ignored_providers(self, list: list): - self.list_ignored_providers = list def register_validation_exception_handler(self): - @app.exception_handler(RequestValidationError) + @self.app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): details = exc.errors() modified_details = [] @@ -54,17 +64,17 @@ class Api: ) def register_routes(self): - @app.get("/") + @self.app.get("/") async def read_root(): return RedirectResponse("/v1", 302) - @app.get("/v1") + @self.app.get("/v1") async def read_root_v1(): return HTMLResponse('g4f API: Go to ' 'chat/completions ' 'or models.') - @app.get("/v1/models") + @self.app.get("/v1/models") async def models(): model_list = dict( (model, g4f.models.ModelUtils.convert[model]) @@ -78,7 +88,7 @@ class Api: } for model_id, model in model_list.items()] return JSONResponse(model_list) - @app.get("/v1/models/{model_name}") + @self.app.get("/v1/models/{model_name}") async def model_info(model_name: str): try: model_info = g4f.models.ModelUtils.convert[model_name] @@ -91,7 +101,7 @@ class Api: except: return JSONResponse({"error": "The model does not exist."}) - @app.post("/v1/chat/completions") + @self.app.post("/v1/chat/completions") async def chat_completions(config: ChatCompletionsConfig, request: Request = None, provider: str = None): try: config.provider = provider if config.provider is None else config.provider @@ -103,7 +113,7 @@ class Api: config.api_key = auth_header response = self.client.chat.completions.create( **config.dict(exclude_none=True), - ignored=self.list_ignored_providers + ignored=list_ignored_providers ) except Exception as e: logging.exception(e) @@ -125,14 +135,10 @@ class Api: return StreamingResponse(streaming(), media_type="text/event-stream") - @app.post("/v1/completions") + @self.app.post("/v1/completions") async def completions(): return Response(content=json.dumps({'info': 'Not working yet.'}, indent=4), media_type="application/json") -api = Api() -api.register_routes() -api.register_validation_exception_handler() - def format_exception(e: Exception, config: ChatCompletionsConfig) -> str: last_provider = g4f.get_last_provider(True) return json.dumps({ @@ -156,4 +162,4 @@ def run_api( host, port = bind.split(":") if debug: g4f.debug.logging = True - uvicorn.run("g4f.api:app", host=host, port=int(port), workers=workers, use_colors=use_colors)# \ No newline at end of file + uvicorn.run("g4f.api:create_app", host=host, port=int(port), workers=workers, use_colors=use_colors, factory=True)# \ No newline at end of file diff --git a/g4f/cli.py b/g4f/cli.py index 6b39091d..86fc2dbb 100644 --- a/g4f/cli.py +++ b/g4f/cli.py @@ -1,7 +1,11 @@ +from __future__ import annotations + import argparse from g4f import Provider from g4f.gui.run import gui_parser, run_gui_args +from g4f.cookies import read_cookie_files +from g4f import debug def main(): parser = argparse.ArgumentParser(description="Run gpt4free") @@ -10,28 +14,36 @@ def main(): api_parser.add_argument("--bind", default="0.0.0.0:1337", help="The bind string.") api_parser.add_argument("--debug", action="store_true", help="Enable verbose logging.") api_parser.add_argument("--workers", type=int, default=None, help="Number of workers.") - api_parser.add_argument("--disable_colors", action="store_true", help="Don't use colors.") + api_parser.add_argument("--disable-colors", action="store_true", help="Don't use colors.") + api_parser.add_argument("--ignore-cookie-files", action="store_true", help="Don't read .har and cookie files.") api_parser.add_argument("--ignored-providers", nargs="+", choices=[provider for provider in Provider.__map__], default=[], help="List of providers to ignore when processing request.") subparsers.add_parser("gui", parents=[gui_parser()], add_help=False) args = parser.parse_args() if args.mode == "api": - import g4f.api - g4f.api.api.set_list_ignored_providers( - args.ignored_providers - ) - g4f.api.run_api( - bind=args.bind, - debug=args.debug, - workers=args.workers, - use_colors=not args.disable_colors - ) + run_api_args(args) elif args.mode == "gui": run_gui_args(args) else: parser.print_help() exit(1) +def run_api_args(args): + if args.debug: + debug.logging = True + if not args.ignore_cookie_files: + read_cookie_files() + import g4f.api + g4f.api.set_list_ignored_providers( + args.ignored_providers + ) + g4f.api.run_api( + bind=args.bind, + debug=args.debug, + workers=args.workers, + use_colors=not args.disable_colors + ) + if __name__ == "__main__": main() diff --git a/g4f/cookies.py b/g4f/cookies.py index 578be8db..efd6e969 100644 --- a/g4f/cookies.py +++ b/g4f/cookies.py @@ -2,6 +2,7 @@ from __future__ import annotations import os import time +import json try: from platformdirs import user_config_dir @@ -38,6 +39,7 @@ def get_cookies(domain_name: str = '', raise_requirements_error: bool = True, si Returns: Dict[str, str]: A dictionary of cookie names and values. """ + global _cookies if domain_name in _cookies: return _cookies[domain_name] @@ -46,6 +48,7 @@ def get_cookies(domain_name: str = '', raise_requirements_error: bool = True, si return cookies def set_cookies(domain_name: str, cookies: Cookies = None) -> None: + global _cookies if cookies: _cookies[domain_name] = cookies elif domain_name in _cookies: @@ -84,6 +87,61 @@ def load_cookies_from_browsers(domain_name: str, raise_requirements_error: bool print(f"Error reading cookies from {cookie_fn.__name__} for {domain_name}: {e}") return cookies +def read_cookie_files(dirPath: str = "./har_and_cookies"): + global _cookies + harFiles = [] + cookieFiles = [] + for root, dirs, files in os.walk(dirPath): + for file in files: + if file.endswith(".har"): + harFiles.append(os.path.join(root, file)) + elif file.endswith(".json"): + cookieFiles.append(os.path.join(root, file)) + _cookies = {} + for path in harFiles: + with open(path, 'rb') as file: + try: + harFile = json.load(file) + except json.JSONDecodeError: + # Error: not a HAR file! + continue + if debug.logging: + print("Read .har file:", path) + new_cookies = {} + for v in harFile['log']['entries']: + v_cookies = {} + for c in v['request']['cookies']: + if c['domain'] not in v_cookies: + v_cookies[c['domain']] = {} + v_cookies[c['domain']][c['name']] = c['value'] + for domain, c in v_cookies.items(): + _cookies[domain] = c + new_cookies[domain] = len(c) + if debug.logging: + for domain, new_values in new_cookies.items(): + print(f"Cookies added: {new_values} from {domain}") + for path in cookieFiles: + with open(path, 'rb') as file: + try: + cookieFile = json.load(file) + except json.JSONDecodeError: + # Error: not a json file! + continue + if not isinstance(cookieFile, list): + continue + if debug.logging: + print("Read cookie file:", path) + new_cookies = {} + for c in cookieFile: + if isinstance(c, dict) and "domain" in c: + if c["domain"] not in new_cookies: + new_cookies[c["domain"]] = {} + new_cookies[c["domain"]][c["name"]] = c["value"] + for domain, new_values in new_cookies.items(): + if debug.logging: + print(f"Cookies added: {len(new_values)} from {domain}") + _cookies[domain] = new_values + def _g4f(domain_name: str) -> list: """ Load cookies from the 'g4f' browser (if exists). diff --git a/g4f/gui/__init__.py b/g4f/gui/__init__.py index f5e448ad..930a2aa0 100644 --- a/g4f/gui/__init__.py +++ b/g4f/gui/__init__.py @@ -12,9 +12,6 @@ def run_gui(host: str = '0.0.0.0', port: int = 8080, debug: bool = False) -> Non if import_error is not None: raise MissingRequirementsError(f'Install "gui" requirements | pip install -U g4f[gui]\n{import_error}') - if debug: - from g4f import debug - debug.logging = True config = { 'host' : host, 'port' : port, diff --git a/g4f/gui/gui_parser.py b/g4f/gui/gui_parser.py index ad458f5c..9fd70bef 100644 --- a/g4f/gui/gui_parser.py +++ b/g4f/gui/gui_parser.py @@ -5,4 +5,5 @@ def gui_parser(): parser.add_argument("-host", type=str, default="0.0.0.0", help="hostname") parser.add_argument("-port", type=int, default=8080, help="port") parser.add_argument("-debug", action="store_true", help="debug mode") + parser.add_argument("--ignore-cookie-files", action="store_true", help="Don't read .har and cookie files.") return parser \ No newline at end of file diff --git a/g4f/gui/run.py b/g4f/gui/run.py index 91314d2d..9b1c527c 100644 --- a/g4f/gui/run.py +++ b/g4f/gui/run.py @@ -1,6 +1,12 @@ from .gui_parser import gui_parser +from ..cookies import read_cookie_files +import g4f.debug def run_gui_args(args): + if args.debug: + g4f.debug.logging = True + if not args.ignore_cookie_files: + read_cookie_files() from g4f.gui import run_gui host = args.host port = args.port -- cgit v1.2.3