in tweetypie/server/src/main/scala/com/twitter/tweetypie/service/observer/Observer.scala [260:363]
def countTweetAttributes(stats: StatsReceiver, byClient: Boolean): Effect[Tweet] = {
val ageStat = stats.stat("age")
val tweetCounter = stats.counter("tweets")
val retweetCounter = stats.counter("retweets")
val repliesCounter = stats.counter("replies")
val inReplyToTweetCounter = stats.counter("in_reply_to_tweet")
val selfRepliesCounter = stats.counter("self_replies")
val directedAtCounter = stats.counter("directed_at")
val mentionsCounter = stats.counter("mentions")
val mentionsStat = stats.stat("mentions")
val urlsCounter = stats.counter("urls")
val urlsStat = stats.stat("urls")
val hashtagsCounter = stats.counter("hashtags")
val hashtagsStat = stats.stat("hashtags")
val mediaCounter = stats.counter("media")
val mediaStat = stats.stat("media")
val photosCounter = stats.counter("media", "photos")
val gifsCounter = stats.counter("media", "animated_gifs")
val videosCounter = stats.counter("media", "videos")
val cardsCounter = stats.counter("cards")
val card2Counter = stats.counter("card2")
val geoCoordsCounter = stats.counter("geo_coordinates")
val placeCounter = stats.counter("place")
val quotedTweetCounter = stats.counter("quoted_tweet")
val selfRetweetCounter = stats.counter("self_retweet")
val languageScope = stats.scope("language")
val textLengthStat = stats.stat("text_length")
val selfThreadCounter = stats.counter("self_thread")
val communitiesTweetCounter = stats.counter("communities")
observeAdditionalFields(stats).also {
Effect[Tweet] { tweet =>
def coreDataField[T](f: TweetCoreData => T): Option[T] =
tweet.coreData.map(f)
def coreDataOptionField[T](f: TweetCoreData => Option[T]) =
coreDataField(f).flatten
(SnowflakeId.isSnowflakeId(tweet.id) match {
case true => Some(SnowflakeId.timeFromId(tweet.id))
case false => coreDataField(_.createdAtSecs.seconds.afterEpoch)
}).foreach { createdAt => ageStat.add(createdAt.untilNow.inSeconds) }
if (!byClient) {
val mentions = getMentions(tweet)
val urls = getUrls(tweet)
val hashtags = getHashtags(tweet)
val media = getMedia(tweet)
val mediaKeys = media.flatMap(_.mediaKey)
val share = coreDataOptionField(_.share)
val selfThreadMetadata = getSelfThreadMetadata(tweet)
val communities = getCommunities(tweet)
tweetCounter.incr()
if (share.isDefined) retweetCounter.incr()
if (coreDataOptionField(_.directedAtUser).isDefined) directedAtCounter.incr()
coreDataOptionField(_.reply).foreach { reply =>
repliesCounter.incr()
if (reply.inReplyToStatusId.nonEmpty) {
// repliesCounter counts all Tweets with a Reply struct,
// but that includes both directed-at Tweets and
// conversational replies. Only conversational replies
// have inReplyToStatusId present, so this counter lets
// us split apart those two cases.
inReplyToTweetCounter.incr()
}
// Not all Tweet objects have CoreData yet isSelfReply() requires it. Thus, this
// invocation is guarded by the `coreDataOptionField(_.reply)` above.
if (isSelfReply(tweet)) selfRepliesCounter.incr()
}
if (mentions.nonEmpty) mentionsCounter.incr()
if (urls.nonEmpty) urlsCounter.incr()
if (hashtags.nonEmpty) hashtagsCounter.incr()
if (media.nonEmpty) mediaCounter.incr()
if (selfThreadMetadata.nonEmpty) selfThreadCounter.incr()
if (communities.nonEmpty) communitiesTweetCounter.incr()
mentionsStat.add(mentions.size)
urlsStat.add(urls.size)
hashtagsStat.add(hashtags.size)
mediaStat.add(media.size)
if (mediaKeys.exists(MediaKeyClassifier.isImage(_))) photosCounter.incr()
if (mediaKeys.exists(MediaKeyClassifier.isGif(_))) gifsCounter.incr()
if (mediaKeys.exists(MediaKeyClassifier.isVideo(_))) videosCounter.incr()
if (tweet.cards.exists(_.nonEmpty)) cardsCounter.incr()
if (tweet.card2.nonEmpty) card2Counter.incr()
if (coreDataOptionField(_.coordinates).nonEmpty) geoCoordsCounter.incr()
if (TweetLenses.place.get(tweet).nonEmpty) placeCounter.incr()
if (TweetLenses.quotedTweet.get(tweet).nonEmpty) quotedTweetCounter.incr()
if (share.exists(_.sourceUserId == getUserId(tweet))) selfRetweetCounter.incr()
tweet.language
.map(_.language)
.foreach(lang => languageScope.counter(lang).incr())
coreDataField(_.text).foreach(text => textLengthStat.add(codePointLength(text)))
}
}
}
}