def _tokenize_object()

in aidial_adapter_openai/utils/tokenizer.py [0:0]

• 9 lines of code
• 5 McCabe index (conditional complexity)

    def _tokenize_object(self, obj: Any) -> int:
        if not obj:
            return 0

        # OpenAI doesn't reveal tokenization algorithm for tools calls and function calls.
        # An approximation is used instead - token count in the string repr of the objects.
        text = (
            obj
            if isinstance(obj, str)
            else json.dumps(obj, separators=(",", ":"))
        )
        return self.tokenize_text(text)