in src/java/com/twitter/search/common/schema/earlybird/EarlybirdSchemaCreateTool.java [180:412]
static {
// Add individual tweet encoded features as views on top of
// EarlybirdFieldConstant.ENCODED_TWEET_FEATURES_FIELD
// int intIndex, int bitStartPos, int bitLength
newEarlybirdFeatureConfiguration(IS_RETWEET_FLAG, ThriftCSFType.BOOLEAN, 0, 0, 1);
newEarlybirdFeatureConfiguration(IS_OFFENSIVE_FLAG, ThriftCSFType.BOOLEAN, 0, 1, 1);
newEarlybirdFeatureConfiguration(HAS_LINK_FLAG, ThriftCSFType.BOOLEAN, 0, 2, 1);
newEarlybirdFeatureConfiguration(HAS_TREND_FLAG, ThriftCSFType.BOOLEAN, 0, 3, 1);
newEarlybirdFeatureConfiguration(IS_REPLY_FLAG, ThriftCSFType.BOOLEAN, 0, 4, 1);
newEarlybirdFeatureConfiguration(IS_SENSITIVE_CONTENT, ThriftCSFType.BOOLEAN, 0, 5, 1);
newEarlybirdFeatureConfiguration(HAS_MULTIPLE_HASHTAGS_OR_TRENDS_FLAG,
ThriftCSFType.BOOLEAN, 0, 6, 1);
newEarlybirdFeatureConfiguration(FROM_VERIFIED_ACCOUNT_FLAG, ThriftCSFType.BOOLEAN, 0, 7, 1);
newEarlybirdFeatureConfiguration(TEXT_SCORE, ThriftCSFType.INT, 0, 8, 8);
newEarlybirdFeatureConfiguration(LANGUAGE, ThriftCSFType.INT, 0, 16, 8);
newEarlybirdFeatureConfiguration(LINK_LANGUAGE, ThriftCSFType.INT, 0, 24, 8);
newEarlybirdFeatureConfiguration(HAS_IMAGE_URL_FLAG, ThriftCSFType.BOOLEAN, 1, 0, 1);
newEarlybirdFeatureConfiguration(HAS_VIDEO_URL_FLAG, ThriftCSFType.BOOLEAN, 1, 1, 1);
newEarlybirdFeatureConfiguration(HAS_NEWS_URL_FLAG, ThriftCSFType.BOOLEAN, 1, 2, 1);
newEarlybirdFeatureConfiguration(HAS_EXPANDO_CARD_FLAG, ThriftCSFType.BOOLEAN, 1, 3, 1);
newEarlybirdFeatureConfiguration(HAS_MULTIPLE_MEDIA_FLAG, ThriftCSFType.BOOLEAN, 1, 4, 1);
newEarlybirdFeatureConfiguration(PROFILE_IS_EGG_FLAG, ThriftCSFType.BOOLEAN, 1, 5, 1);
newEarlybirdFeatureConfiguration(NUM_MENTIONS, ThriftCSFType.INT, 1, 6, 2); // 0, 1, 2, 3+
newEarlybirdFeatureConfiguration(NUM_HASHTAGS, ThriftCSFType.INT, 1, 8, 2); // 0, 1, 2, 3+
newEarlybirdFeatureConfiguration(HAS_CARD_FLAG, ThriftCSFType.BOOLEAN, 1, 10, 1);
newEarlybirdFeatureConfiguration(HAS_VISIBLE_LINK_FLAG, ThriftCSFType.BOOLEAN, 1, 11, 1);
newEarlybirdFeatureConfiguration(USER_REPUTATION, ThriftCSFType.INT, 1, 12, 8);
newEarlybirdFeatureConfiguration(IS_USER_SPAM_FLAG, ThriftCSFType.BOOLEAN, 1, 20, 1);
newEarlybirdFeatureConfiguration(IS_USER_NSFW_FLAG, ThriftCSFType.BOOLEAN, 1, 21, 1);
newEarlybirdFeatureConfiguration(IS_USER_BOT_FLAG, ThriftCSFType.BOOLEAN, 1, 22, 1);
newEarlybirdFeatureConfiguration(IS_USER_NEW_FLAG, ThriftCSFType.BOOLEAN, 1, 23, 1);
newEarlybirdFeatureConfiguration(PREV_USER_TWEET_ENGAGEMENT, ThriftCSFType.INT, 1, 24, 6);
newEarlybirdFeatureConfiguration(COMPOSER_SOURCE_IS_CAMERA_FLAG,
ThriftCSFType.BOOLEAN, 1, 30, 1);
newEarlybirdFeatureConfiguration(IS_NULLCAST_FLAG, ThriftCSFType.BOOLEAN, 1, 31, 1);
newEarlybirdFeatureConfiguration(RETWEET_COUNT, ThriftCSFType.DOUBLE, 2, 0, 8,
ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(FAVORITE_COUNT, ThriftCSFType.DOUBLE, 2, 8, 8,
ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(REPLY_COUNT, ThriftCSFType.DOUBLE, 2, 16, 8,
ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(PARUS_SCORE, ThriftCSFType.DOUBLE, 2, 24, 8);
newEarlybirdFeatureConfiguration(HAS_CONSUMER_VIDEO_FLAG, ThriftCSFType.BOOLEAN, 3, 0, 1);
newEarlybirdFeatureConfiguration(HAS_PRO_VIDEO_FLAG, ThriftCSFType.BOOLEAN, 3, 1, 1);
newEarlybirdFeatureConfiguration(HAS_VINE_FLAG, ThriftCSFType.BOOLEAN, 3, 2, 1);
newEarlybirdFeatureConfiguration(HAS_PERISCOPE_FLAG, ThriftCSFType.BOOLEAN, 3, 3, 1);
newEarlybirdFeatureConfiguration(HAS_NATIVE_IMAGE_FLAG, ThriftCSFType.BOOLEAN, 3, 4, 1);
// NOTE: There are 3 bits left in the first byte of INT 3, if possible, please reserve them
// for future media types (SEARCH-9131)
// newEarlybirdFeatureConfiguration(FUTURE_MEDIA_BITS, ThriftCSFType.INT, 3, 5, 3);
newEarlybirdFeatureConfiguration(VISIBLE_TOKEN_RATIO, ThriftCSFType.INT, 3, 8, 4);
newEarlybirdFeatureConfiguration(HAS_QUOTE_FLAG, ThriftCSFType.BOOLEAN, 3, 12, 1);
newEarlybirdFeatureConfiguration(FROM_BLUE_VERIFIED_ACCOUNT_FLAG,
ThriftCSFType.BOOLEAN, 3, 13, 1);
// Unused bits from bit 14 to bit 31 (18 bits)
// newEarlybirdFeatureConfiguration(UNUSED_BITS, ThriftCSFType.INT, 3, 14, 18);
newEarlybirdFeatureConfiguration(TWEET_SIGNATURE, ThriftCSFType.INT, 4, 0, 32);
newEarlybirdFeatureConfiguration(EMBEDS_IMPRESSION_COUNT,
ThriftCSFType.DOUBLE, 0, 0, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(EMBEDS_URL_COUNT,
ThriftCSFType.DOUBLE, 0, 8, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(VIDEO_VIEW_COUNT,
ThriftCSFType.DOUBLE, 0, 16, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
// Unused bits from bit 24 to bit 31 (8 bits).
// This used to be a feature that was decommissioned (SEARCHQUAL-10321)
newEarlybirdFeatureConfiguration(EXTENDED_FEATURE_UNUSED_BITS_0_24_8,
ThriftCSFType.INT, 0, 24, 8);
newEarlybirdFeatureConfiguration(REFERENCE_AUTHOR_ID_LEAST_SIGNIFICANT_INT,
ThriftCSFType.INT, 1, 0, 32, ThriftFeatureUpdateConstraint.IMMUTABLE);
newEarlybirdFeatureConfiguration(REFERENCE_AUTHOR_ID_MOST_SIGNIFICANT_INT,
ThriftCSFType.INT, 2, 0, 32, ThriftFeatureUpdateConstraint.IMMUTABLE);
newEarlybirdFeatureConfiguration(RETWEET_COUNT_V2,
ThriftCSFType.DOUBLE, 3, 0, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(FAVORITE_COUNT_V2,
ThriftCSFType.DOUBLE, 3, 8, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(REPLY_COUNT_V2,
ThriftCSFType.DOUBLE, 3, 16, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(EMBEDS_IMPRESSION_COUNT_V2,
ThriftCSFType.DOUBLE, 3, 24, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(EMBEDS_URL_COUNT_V2,
ThriftCSFType.DOUBLE, 4, 0, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(VIDEO_VIEW_COUNT_V2,
ThriftCSFType.DOUBLE, 4, 8, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(QUOTE_COUNT,
ThriftCSFType.DOUBLE, 4, 16, 8);
newEarlybirdFeatureConfiguration(LABEL_ABUSIVE_FLAG, ThriftCSFType.BOOLEAN, 4, 24, 1);
newEarlybirdFeatureConfiguration(LABEL_ABUSIVE_HI_RCL_FLAG, ThriftCSFType.BOOLEAN, 4, 25, 1);
newEarlybirdFeatureConfiguration(LABEL_DUP_CONTENT_FLAG, ThriftCSFType.BOOLEAN, 4, 26, 1);
newEarlybirdFeatureConfiguration(LABEL_NSFW_HI_PRC_FLAG, ThriftCSFType.BOOLEAN, 4, 27, 1);
newEarlybirdFeatureConfiguration(LABEL_NSFW_HI_RCL_FLAG, ThriftCSFType.BOOLEAN, 4, 28, 1);
newEarlybirdFeatureConfiguration(LABEL_SPAM_FLAG, ThriftCSFType.BOOLEAN, 4, 29, 1);
newEarlybirdFeatureConfiguration(LABEL_SPAM_HI_RCL_FLAG, ThriftCSFType.BOOLEAN, 4, 30, 1);
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_4_31_1,
ThriftCSFType.INT, 4, 31, 1);
newEarlybirdFeatureConfiguration(WEIGHTED_RETWEET_COUNT,
ThriftCSFType.DOUBLE, 5, 0, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(WEIGHTED_REPLY_COUNT,
ThriftCSFType.DOUBLE, 5, 8, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(WEIGHTED_FAVORITE_COUNT,
ThriftCSFType.DOUBLE, 5, 16, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(WEIGHTED_QUOTE_COUNT,
ThriftCSFType.DOUBLE, 5, 24, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(PERISCOPE_EXISTS,
ThriftCSFType.BOOLEAN, 6, 0, 1);
newEarlybirdFeatureConfiguration(PERISCOPE_HAS_BEEN_FEATURED,
ThriftCSFType.BOOLEAN, 6, 1, 1);
newEarlybirdFeatureConfiguration(PERISCOPE_IS_CURRENTLY_FEATURED,
ThriftCSFType.BOOLEAN, 6, 2, 1);
newEarlybirdFeatureConfiguration(PERISCOPE_IS_FROM_QUALITY_SOURCE,
ThriftCSFType.BOOLEAN, 6, 3, 1);
newEarlybirdFeatureConfiguration(PERISCOPE_IS_LIVE,
ThriftCSFType.BOOLEAN, 6, 4, 1);
newEarlybirdFeatureConfiguration(IS_TRENDING_NOW_FLAG,
ThriftCSFType.BOOLEAN, 6, 5, 1);
// remaining bits for integer 6
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_7_6_26,
ThriftCSFType.INT, 6, 6, 26);
// The decaying counters can become smaller
newEarlybirdFeatureConfiguration(DECAYED_RETWEET_COUNT,
ThriftCSFType.DOUBLE, 7, 0, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(DECAYED_REPLY_COUNT,
ThriftCSFType.DOUBLE, 7, 8, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(DECAYED_FAVORITE_COUNT,
ThriftCSFType.DOUBLE, 7, 16, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(DECAYED_QUOTE_COUNT,
ThriftCSFType.DOUBLE, 7, 24, 8, ThriftFeatureUpdateConstraint.POSITIVE);
// The fake engagement counters.
newEarlybirdFeatureConfiguration(FAKE_RETWEET_COUNT,
ThriftCSFType.DOUBLE, 8, 0, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(FAKE_REPLY_COUNT,
ThriftCSFType.DOUBLE, 8, 8, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(FAKE_FAVORITE_COUNT,
ThriftCSFType.DOUBLE, 8, 16, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(FAKE_QUOTE_COUNT,
ThriftCSFType.DOUBLE, 8, 24, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(LAST_RETWEET_SINCE_CREATION_HRS,
ThriftCSFType.INT, 9, 0, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(LAST_REPLY_SINCE_CREATION_HRS,
ThriftCSFType.INT, 9, 8, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(LAST_FAVORITE_SINCE_CREATION_HRS,
ThriftCSFType.INT, 9, 16, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(LAST_QUOTE_SINCE_CREATION_HRS,
ThriftCSFType.INT, 9, 24, 8, ThriftFeatureUpdateConstraint.INC_ONLY);
newEarlybirdFeatureConfiguration(NUM_HASHTAGS_V2,
ThriftCSFType.INT, 10, 0, 4);
newEarlybirdFeatureConfiguration(NUM_MENTIONS_V2,
ThriftCSFType.INT, 10, 4, 4);
newEarlybirdFeatureConfiguration(NUM_STOCKS,
ThriftCSFType.INT, 10, 8, 4);
// Remaining bits for integer 10
// Production Toxicity and PBlock score from HML (go/toxicity, go/pblock)
newEarlybirdFeatureConfiguration(TOXICITY_SCORE,
ThriftCSFType.DOUBLE, 10, 12, 10);
newEarlybirdFeatureConfiguration(PBLOCK_SCORE,
ThriftCSFType.DOUBLE, 10, 22, 10);
// The blink engagement counters
newEarlybirdFeatureConfiguration(BLINK_RETWEET_COUNT,
ThriftCSFType.DOUBLE, 11, 0, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(BLINK_REPLY_COUNT,
ThriftCSFType.DOUBLE, 11, 8, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(BLINK_FAVORITE_COUNT,
ThriftCSFType.DOUBLE, 11, 16, 8, ThriftFeatureUpdateConstraint.POSITIVE);
newEarlybirdFeatureConfiguration(BLINK_QUOTE_COUNT,
ThriftCSFType.DOUBLE, 11, 24, 8, ThriftFeatureUpdateConstraint.POSITIVE);
// Experimental health model scores from HML
newEarlybirdFeatureConfiguration(EXPERIMENTAL_HEALTH_MODEL_SCORE_1,
ThriftCSFType.DOUBLE, 12, 0, 10);
newEarlybirdFeatureConfiguration(EXPERIMENTAL_HEALTH_MODEL_SCORE_2,
ThriftCSFType.DOUBLE, 12, 10, 10);
newEarlybirdFeatureConfiguration(EXPERIMENTAL_HEALTH_MODEL_SCORE_3,
ThriftCSFType.DOUBLE, 12, 20, 10);
// remaining bits for integer 12
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_12_30_2,
ThriftCSFType.INT, 12, 30, 2);
// Experimental health model scores from HML (cont.)
newEarlybirdFeatureConfiguration(EXPERIMENTAL_HEALTH_MODEL_SCORE_4,
ThriftCSFType.DOUBLE, 13, 0, 10);
// Production pSpammyTweet score from HML (go/pspammytweet)
newEarlybirdFeatureConfiguration(P_SPAMMY_TWEET_SCORE,
ThriftCSFType.DOUBLE, 13, 10, 10);
// Production pReportedTweet score from HML (go/preportedtweet)
newEarlybirdFeatureConfiguration(P_REPORTED_TWEET_SCORE,
ThriftCSFType.DOUBLE, 13, 20, 10);
// remaining bits for integer 13
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_13_30_2,
ThriftCSFType.INT, 13, 30, 2);
// Experimental health model scores from HML (cont.)
// Prod Spammy Tweet Content model score from Platform Manipulation (go/spammy-tweet-content)
newEarlybirdFeatureConfiguration(SPAMMY_TWEET_CONTENT_SCORE,
ThriftCSFType.DOUBLE, 14, 0, 10);
// remaining bits for integer 14
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_14_10_22,
ThriftCSFType.INT, 14, 10, 22);
// Note that the integer index below is 0-based, but the index j in UNUSED_BITS_{j} below
// is 1-based.
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_16,
ThriftCSFType.INT, 15, 0, 32);
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_17,
ThriftCSFType.INT, 16, 0, 32);
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_18,
ThriftCSFType.INT, 17, 0, 32);
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_19,
ThriftCSFType.INT, 18, 0, 32);
newEarlybirdFeatureConfiguration(EXTENDED_TEST_FEATURE_UNUSED_BITS_20,
ThriftCSFType.INT, 19, 0, 32);
}