in server/src/main/java/com/epam/aidial/core/server/util/ModelCostCalculator.java [81:125]
private static int getResponseContentLength(ModelType modelType, Buffer responseBody, boolean isStreamingResponse) {
if (modelType == ModelType.EMBEDDING) {
return 0;
}
if (isStreamingResponse) {
try (Scanner scanner = new Scanner(new ByteBufInputStream(responseBody.getByteBuf()))) {
// each chunk is separated by one or multiple new lines with the prefix: 'data:' (except the first chunk)
// chunks may contain `data:` inside chunk data, which may lead to incorrect parsing
scanner.useDelimiter("(^data: *|\n+data: *)");
int len = 0;
while (scanner.hasNext()) {
String chunk = scanner.next();
if (chunk.startsWith("[DONE]")) {
break;
}
ObjectNode tree = (ObjectNode) ProxyUtil.MAPPER.readTree(chunk);
ArrayNode choices = (ArrayNode) tree.get("choices");
if (choices == null) {
// skip error message
continue;
}
JsonNode contentNode = choices.get(0).get("delta").get("content");
if (contentNode != null) {
len += getLengthWithoutWhitespace(contentNode.textValue());
}
}
return len;
} catch (Throwable e) {
throw new RuntimeException(e);
}
} else {
try (InputStream stream = new ByteBufInputStream(responseBody.getByteBuf())) {
ObjectNode tree = (ObjectNode) ProxyUtil.MAPPER.readTree(stream);
ArrayNode choices = (ArrayNode) tree.get("choices");
if (choices == null) {
// skip error message
return 0;
}
JsonNode contentNode = choices.get(0).get("message").get("content");
return getLengthWithoutWhitespace(contentNode.textValue());
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
}