From 479ef89f4c3bc284cf50395481f2ef807c2f90ee Mon Sep 17 00:00:00 2001 From: abc <98614666+xtekky@users.noreply.github.com> Date: Mon, 11 Mar 2024 18:31:13 +0000 Subject: ~ | support local llm inference --- g4f/local/__init__.py | 73 ++++----------------------------------------------- 1 file changed, 5 insertions(+), 68 deletions(-) (limited to 'g4f/local/__init__.py') diff --git a/g4f/local/__init__.py b/g4f/local/__init__.py index 626643fc..cc678dc6 100644 --- a/g4f/local/__init__.py +++ b/g4f/local/__init__.py @@ -1,75 +1,13 @@ -import random, string, time, re - from ..typing import Union, Iterator, Messages from ..stubs import ChatCompletion, ChatCompletionChunk from .core.engine import LocalProvider from .core.models import models - -IterResponse = Iterator[Union[ChatCompletion, ChatCompletionChunk]] - -def read_json(text: str) -> dict: - match = re.search(r"```(json|)\n(?P[\S\s]+?)\n```", text) - if match: - return match.group("code") - return text - -def iter_response( - response: Iterator[str], - stream: bool, - response_format: dict = None, - max_tokens: int = None, - stop: list = None -) -> IterResponse: - - content = "" - finish_reason = None - completion_id = ''.join(random.choices(string.ascii_letters + string.digits, k=28)) - for idx, chunk in enumerate(response): - content += str(chunk) - if max_tokens is not None and idx + 1 >= max_tokens: - finish_reason = "length" - first = -1 - word = None - if stop is not None: - for word in list(stop): - first = content.find(word) - if first != -1: - content = content[:first] - break - if stream and first != -1: - first = chunk.find(word) - if first != -1: - chunk = chunk[:first] - else: - first = 0 - if first != -1: - finish_reason = "stop" - if stream: - yield ChatCompletionChunk(chunk, None, completion_id, int(time.time())) - if finish_reason is not None: - break - finish_reason = "stop" if finish_reason is None else finish_reason - if stream: - yield ChatCompletionChunk(None, finish_reason, completion_id, int(time.time())) - else: - if response_format is not None and "type" in response_format: - if response_format["type"] == "json_object": - content = read_json(content) - yield ChatCompletion(content, finish_reason, completion_id, int(time.time())) - -def filter_none(**kwargs): - for key in list(kwargs.keys()): - if kwargs[key] is None: - del kwargs[key] - return kwargs +from ..client import iter_response, filter_none, IterResponse class LocalClient(): - def __init__( - self, - **kwargs - ) -> None: + def __init__(self, **kwargs) -> None: self.chat: Chat = Chat(self) - + @staticmethod def list_models(): return list(models.keys()) @@ -100,10 +38,9 @@ class Completions(): ) response = iter_response(response, stream, response_format, max_tokens, stop) return response if stream else next(response) - + class Chat(): completions: Completions def __init__(self, client: LocalClient): - self.completions = Completions(client) - \ No newline at end of file + self.completions = Completions(client) \ No newline at end of file -- cgit v1.2.3