def tokenize_request_message()

in aidial_adapter_openai/utils/tokenizer.py [0:0]

• 16 lines of code
• 2 McCabe index (conditional complexity)

    def tokenize_request_message(self, message: MultiModalMessage) -> int:
        tokens = self._tokens_per_request_message
        raw_message = message.raw_message

        tokens += _process_raw_message(
            raw_message=raw_message,
            tokens_per_name=self._tokens_per_request_message_name,
            calculate_text_tokens=self.tokenize_text,
            handle_custom_content_part=lambda content_part: None,
        )

        # Processing image parts of message
        for metadata in message.image_metadatas:
            tokens += self.image_tokenizer.tokenize(
                width=metadata.width,
                height=metadata.height,
                detail=metadata.detail,
            )
        return tokens