From 77697be33381a01350d0818ff069469faea2f4ac Mon Sep 17 00:00:00 2001 From: Arran Hobson Sayers Date: Thu, 12 Oct 2023 02:35:11 +0100 Subject: Unify g4f tools into one CLI --- g4f/api/__init__.py | 162 +++++++++++++++++++++++++++++++++++++++++++ g4f/api/run.py | 4 ++ g4f/cli.py | 28 ++++++++ g4f/gui/__init__.py | 2 +- g4f/gui/run.py | 30 ++++---- g4f/interference/__init__.py | 94 ------------------------- g4f/interference/run.py | 4 -- 7 files changed, 213 insertions(+), 111 deletions(-) create mode 100644 g4f/api/__init__.py create mode 100644 g4f/api/run.py create mode 100644 g4f/cli.py delete mode 100644 g4f/interference/__init__.py delete mode 100644 g4f/interference/run.py (limited to 'g4f') diff --git a/g4f/api/__init__.py b/g4f/api/__init__.py new file mode 100644 index 00000000..c52085dc --- /dev/null +++ b/g4f/api/__init__.py @@ -0,0 +1,162 @@ +import json +import random +import string +import time + +import requests +from flask import Flask, request +from flask_cors import CORS +from transformers import AutoTokenizer + +from g4f import ChatCompletion + +app = Flask(__name__) +CORS(app) + + +@app.route("/") +def index(): + return "interference api, url: http://127.0.0.1:1337" + + +@app.route("/chat/completions", methods=["POST"]) +def chat_completions(): + model = request.get_json().get("model", "gpt-3.5-turbo") + stream = request.get_json().get("stream", False) + messages = request.get_json().get("messages") + + response = ChatCompletion.create(model=model, stream=stream, messages=messages) + + completion_id = "".join(random.choices(string.ascii_letters + string.digits, k=28)) + completion_timestamp = int(time.time()) + + if not stream: + return { + "id": f"chatcmpl-{completion_id}", + "object": "chat.completion", + "created": completion_timestamp, + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": response, + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": None, + "completion_tokens": None, + "total_tokens": None, + }, + } + + def streaming(): + for chunk in response: + completion_data = { + "id": f"chatcmpl-{completion_id}", + "object": "chat.completion.chunk", + "created": completion_timestamp, + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": chunk, + }, + "finish_reason": None, + } + ], + } + + content = json.dumps(completion_data, separators=(",", ":")) + yield f"data: {content}\n\n" + time.sleep(0.1) + + end_completion_data = { + "id": f"chatcmpl-{completion_id}", + "object": "chat.completion.chunk", + "created": completion_timestamp, + "model": model, + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": "stop", + } + ], + } + content = json.dumps(end_completion_data, separators=(",", ":")) + yield f"data: {content}\n\n" + + return app.response_class(streaming(), mimetype="text/event-stream") + + +# Get the embedding from huggingface +def get_embedding(input_text, token): + huggingface_token = token + embedding_model = "sentence-transformers/all-mpnet-base-v2" + max_token_length = 500 + + # Load the tokenizer for the 'all-mpnet-base-v2' model + tokenizer = AutoTokenizer.from_pretrained(embedding_model) + # Tokenize the text and split the tokens into chunks of 500 tokens each + tokens = tokenizer.tokenize(input_text) + token_chunks = [ + tokens[i : i + max_token_length] + for i in range(0, len(tokens), max_token_length) + ] + + # Initialize an empty list + embeddings = [] + + # Create embeddings for each chunk + for chunk in token_chunks: + # Convert the chunk tokens back to text + chunk_text = tokenizer.convert_tokens_to_string(chunk) + + # Use the Hugging Face API to get embeddings for the chunk + api_url = f"https://api-inference.huggingface.co/pipeline/feature-extraction/{embedding_model}" + headers = {"Authorization": f"Bearer {huggingface_token}"} + chunk_text = chunk_text.replace("\n", " ") + + # Make a POST request to get the chunk's embedding + response = requests.post( + api_url, + headers=headers, + json={"inputs": chunk_text, "options": {"wait_for_model": True}}, + ) + + # Parse the response and extract the embedding + chunk_embedding = response.json() + # Append the embedding to the list + embeddings.append(chunk_embedding) + + # averaging all the embeddings + # this isn't very effective + # someone a better idea? + num_embeddings = len(embeddings) + average_embedding = [sum(x) / num_embeddings for x in zip(*embeddings)] + embedding = average_embedding + return embedding + + +@app.route("/embeddings", methods=["POST"]) +def embeddings(): + input_text_list = request.get_json().get("input") + input_text = " ".join(map(str, input_text_list)) + token = request.headers.get("Authorization").replace("Bearer ", "") + embedding = get_embedding(input_text, token) + + return { + "data": [{"embedding": embedding, "index": 0, "object": "embedding"}], + "model": "text-embedding-ada-002", + "object": "list", + "usage": {"prompt_tokens": None, "total_tokens": None}, + } + + +def run_api(): + app.run(host="0.0.0.0", port=1337) diff --git a/g4f/api/run.py b/g4f/api/run.py new file mode 100644 index 00000000..6e9b63f3 --- /dev/null +++ b/g4f/api/run.py @@ -0,0 +1,4 @@ +from g4f.api import run_api + +if __name__ == "__main__": + run_api() diff --git a/g4f/cli.py b/g4f/cli.py new file mode 100644 index 00000000..42401cc8 --- /dev/null +++ b/g4f/cli.py @@ -0,0 +1,28 @@ +import argparse + +from g4f.api import run_api +from g4f.gui.run import gui_parser, run_gui_args + + +def run_gui(args): + print("Running GUI...") + + +def main(): + parser = argparse.ArgumentParser(description="Run gpt4free") + subparsers = parser.add_subparsers(dest="mode", help="Mode to run the g4f in.") + subparsers.add_parser("api") + subparsers.add_parser("gui", parents=[gui_parser()], add_help=False) + + args = parser.parse_args() + if args.mode == "api": + run_api() + elif args.mode == "gui": + run_gui_args(args) + else: + parser.print_help() + exit(1) + + +if __name__ == "__main__": + main() diff --git a/g4f/gui/__init__.py b/g4f/gui/__init__.py index 48b78881..a8000e71 100644 --- a/g4f/gui/__init__.py +++ b/g4f/gui/__init__.py @@ -27,4 +27,4 @@ def run_gui(host: str = '0.0.0.0', port: int = 80, debug: bool = False) -> None: print(f"Running on port {config['port']}") app.run(**config) - print(f"Closing port {config['port']}") \ No newline at end of file + print(f"Closing port {config['port']}") diff --git a/g4f/gui/run.py b/g4f/gui/run.py index 731c7cbf..0f94814c 100644 --- a/g4f/gui/run.py +++ b/g4f/gui/run.py @@ -1,18 +1,24 @@ -from g4f.gui import run_gui from argparse import ArgumentParser +from g4f.gui import run_gui -if __name__ == '__main__': - - parser = ArgumentParser(description='Run the GUI') - - parser.add_argument('-host', type=str, default='0.0.0.0', help='hostname') - parser.add_argument('-port', type=int, default=80, help='port') - parser.add_argument('-debug', action='store_true', help='debug mode') - args = parser.parse_args() - port = args.port +def gui_parser(): + parser = ArgumentParser(description="Run the GUI") + parser.add_argument("-host", type=str, default="0.0.0.0", help="hostname") + parser.add_argument("-port", type=int, default=80, help="port") + parser.add_argument("-debug", action="store_true", help="debug mode") + return parser + + +def run_gui_args(args): host = args.host + port = args.port debug = args.debug - - run_gui(host, port, debug) \ No newline at end of file + run_gui(host, port, debug) + + +if __name__ == "__main__": + parser = gui_parser() + args = parser.parse_args() + run_gui_args(args) diff --git a/g4f/interference/__init__.py b/g4f/interference/__init__.py deleted file mode 100644 index d756faa7..00000000 --- a/g4f/interference/__init__.py +++ /dev/null @@ -1,94 +0,0 @@ -import json -import time -import random -import string - -from typing import Any -from flask import Flask, request -from flask_cors import CORS -from g4f import ChatCompletion - -app = Flask(__name__) -CORS(app) - -@app.route('/') -def index(): - return 'interference api, url: http://127.0.0.1:1337' - -@app.route('/chat/completions', methods=['POST']) -def chat_completions(): - model = request.get_json().get('model', 'gpt-3.5-turbo') - stream = request.get_json().get('stream', False) - messages = request.get_json().get('messages') - - response = ChatCompletion.create(model = model, - stream = stream, messages = messages) - - completion_id = ''.join(random.choices(string.ascii_letters + string.digits, k=28)) - completion_timestamp = int(time.time()) - - if not stream: - return { - 'id': f'chatcmpl-{completion_id}', - 'object': 'chat.completion', - 'created': completion_timestamp, - 'model': model, - 'choices': [ - { - 'index': 0, - 'message': { - 'role': 'assistant', - 'content': response, - }, - 'finish_reason': 'stop', - } - ], - 'usage': { - 'prompt_tokens': None, - 'completion_tokens': None, - 'total_tokens': None, - }, - } - - def streaming(): - for chunk in response: - completion_data = { - 'id': f'chatcmpl-{completion_id}', - 'object': 'chat.completion.chunk', - 'created': completion_timestamp, - 'model': model, - 'choices': [ - { - 'index': 0, - 'delta': { - 'content': chunk, - }, - 'finish_reason': None, - } - ], - } - - content = json.dumps(completion_data, separators=(',', ':')) - yield f'data: {content}\n\n' - time.sleep(0.1) - - end_completion_data: dict[str, Any] = { - 'id': f'chatcmpl-{completion_id}', - 'object': 'chat.completion.chunk', - 'created': completion_timestamp, - 'model': model, - 'choices': [ - { - 'index': 0, - 'delta': {}, - 'finish_reason': 'stop', - } - ], - } - content = json.dumps(end_completion_data, separators=(',', ':')) - yield f'data: {content}\n\n' - - return app.response_class(streaming(), mimetype='text/event-stream') - -def run_interference(): - app.run(host='0.0.0.0', port=1337, debug=True) \ No newline at end of file diff --git a/g4f/interference/run.py b/g4f/interference/run.py deleted file mode 100644 index e527ce11..00000000 --- a/g4f/interference/run.py +++ /dev/null @@ -1,4 +0,0 @@ -from g4f.interference import run_interference - -if __name__ == '__main__': - run_interference() \ No newline at end of file -- cgit v1.2.3