From b4893b9bdd161cc2e421183d4eaf3c4094656cd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jason=20K=C3=B6lker?= Date: Wed, 11 Dec 2024 20:07:33 +0000 Subject: fix(utils): improve response mapping Make the reponse mapping more robust by checking for an empty (or missing) `choices` list and substitute a list with an empty dictionary. Use `.get` to access the `message` or `delta` object again returning an empty dictionary if they are not found. When using `hermes3-405b` on lambda cloud's inference (based on openrouter) a final response was returned with an empty list for choices causing a traceback on completion. Debug log: ``` [2024-12-11 19:49:11.925592] [engine-chat] response: {'id': 'chatcmpl-140a7a938d2149c8a750f47af6a11be1', 'object': 'chat.completion.chunk', 'created': 1733946550, 'model': 'hermes3-405b', 'choices': [{'index': 0, 'delta': {'content': ' today'}, 'finish_reason': None, 'content_filter_results': {'hate': {'filtered': False}, 'self_harm': {'filtered': False}, 'sexual': {'filtered': False}, 'violence': {'filtered': False}, 'jailbreak': {'filtered': False, 'detected': False}, 'profanity': {'filtered': False, 'detected': False}}}], 'system_fingerprint': '', 'usage': {'prompt_tokens': 40, 'completion_tokens': 18, 'total_tokens': 58, 'prompt_tokens_details': None, 'completion_tokens_details': None}} [2024-12-11 19:49:11.975457] [engine-chat] response: {'id': 'chatcmpl-140a7a938d2149c8a750f47af6a11be1', 'object': 'chat.completion.chunk', 'created': 1733946550, 'model': 'hermes3-405b', 'choices': [{'index': 0, 'delta': {'content': '?'}, 'finish_reason': None, 'content_filter_results': {'hate': {'filtered': False}, 'self_harm': {'filtered': False}, 'sexual': {'filtered': False}, 'violence': {'filtered': False}, 'jailbreak': {'filtered': False, 'detected': False}, 'profanity': {'filtered': False, 'detected': False}}}], 'system_fingerprint': '', 'usage': {'prompt_tokens': 40, 'completion_tokens': 19, 'total_tokens': 59, 'prompt_tokens_details': None, 'completion_tokens_details': None}} [2024-12-11 19:49:12.008987] [engine-chat] response: {'id': 'chatcmpl-140a7a938d2149c8a750f47af6a11be1', 'object': 'chat.completion.chunk', 'created': 1733946550, 'model': 'hermes3-405b', 'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'stop', 'content_filter_results': {'hate': {'filtered': False}, 'self_harm': {'filtered': False}, 'sexual': {'filtered': False}, 'violence': {'filtered': False}, 'jailbreak': {'filtered': False, 'detected': False}, 'profanity': {'filtered': False, 'detected': False}}}], 'system_fingerprint': '', 'usage': {'prompt_tokens': 40, 'completion_tokens': 20, 'total_tokens': 60, 'prompt_tokens_details': None, 'completion_tokens_details': None}} [2024-12-11 19:49:12.009400] [engine-chat] response: {'id': 'chatcmpl-140a7a938d2149c8a750f47af6a11be1', 'object': 'chat.completion.chunk', 'created': 1733946550, 'model': 'hermes3-405b', 'choices': [], 'system_fingerprint': '', 'usage': {'prompt_tokens': 40, 'completion_tokens': 19, 'total_tokens': 59, 'prompt_tokens_details': None, 'completion_tokens_details': None}} ``` --- py/utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'py/utils.py') diff --git a/py/utils.py b/py/utils.py index c58529c..f81eeb8 100644 --- a/py/utils.py +++ b/py/utils.py @@ -362,13 +362,22 @@ def make_chat_text_chunks(messages, config_options): url = config_options['endpoint_url'] response = openai_request(url, request, http_options) + def _choices(resp): + choices = resp.get('choices', [{}]) + + # NOTE choices may exist in the response, but be an empty list. + if not choices: + return [{}] + + return choices + def map_chunk_no_stream(resp): printDebug("[engine-chat] response: {}", resp) - return resp['choices'][0]['message'].get('content', '') + return _choices(resp)[0].get('message', {}).get('content', '') def map_chunk_stream(resp): printDebug("[engine-chat] response: {}", resp) - return resp['choices'][0]['delta'].get('content', '') + return _choices(resp)[0].get('delta', {}).get('content', '') map_chunk = map_chunk_stream if openai_options['stream'] else map_chunk_no_stream -- cgit v1.2.3