diff --git a/src/chap/backends/anthropic.py b/src/chap/backends/anthropic.py index 486db4d..f3d8f41 100644 --- a/src/chap/backends/anthropic.py +++ b/src/chap/backends/anthropic.py @@ -17,7 +17,7 @@ class Anthropic(AutoAskMixin): @dataclass class Parameters: url: str = "https://api.anthropic.com" - model: str = "claude-3-sonnet-20240229" + model: str = "claude-3-5-sonnet-20240620" max_new_tokens: int = 1000 def __init__(self) -> None: @@ -94,5 +94,5 @@ Answer each question accurately and thoroughly. def factory() -> Backend: - """Uses the huggingface text-generation-interface web API""" + """Uses the anthropic text-generation-interface web API""" return Anthropic() diff --git a/src/chap/backends/llama_cpp.py b/src/chap/backends/llama_cpp.py index 99ddece..5cb92cb 100644 --- a/src/chap/backends/llama_cpp.py +++ b/src/chap/backends/llama_cpp.py @@ -18,10 +18,16 @@ class LlamaCpp(AutoAskMixin): url: str = "http://localhost:8080/completion" """The URL of a llama.cpp server's completion endpoint.""" - start_prompt: str = "" - system_format: str = "<>{}<>" - user_format: str = " [INST] {} [/INST]" - assistant_format: str = " {}" + start_prompt: str = "<|begin_of_text|>" + system_format: str = ( + "<|start_header_id|>system<|end_header_id|>\n\n{}<|eot_id|>" + ) + user_format: str = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + assistant_format: str = ( + "<|start_header_id|>assistant<|end_header_id|>\n\n{}<|eot_id|>" + ) + end_prompt: str = "<|start_header_id|>assistant<|end_header_id|>\n\n" + stop: str | None = None def __init__(self) -> None: super().__init__() @@ -59,7 +65,7 @@ A dialog, where USER interacts with AI. AI is helpful, kind, obedient, honest, a params = { "prompt": self.make_full_query(session + [User(query)], max_query_size), "stream": True, - "stop": ["", "", "[INST]"], + "stop": ["", "", "[INST]", "<|eot_id|>"], } new_content: list[str] = [] try: diff --git a/src/chap/core.py b/src/chap/core.py index 9a3df7f..67ed9f7 100644 --- a/src/chap/core.py +++ b/src/chap/core.py @@ -129,7 +129,11 @@ def configure_api_from_environment( setattr(api.parameters, field.name, tv) -def get_api(ctx: click.Context, name: str = "openai_chatgpt") -> Backend: +def get_api(ctx: click.Context | None = None, name: str | None = None) -> Backend: + if ctx is None: + ctx = click.Context(click.Command("chap")) + if name is None: + name = os.environ.get("CHAP_BACKEND", "openai_chatgpt") name = name.replace("-", "_") backend = cast( Backend, importlib.import_module(f"{__package__}.backends.{name}").factory()