TwitterImagePipeline/Project/TIPImageDiskCache.m (1,518 lines of code) (raw):
//
// TIPImageDiskCache.m
// TwitterImagePipeline
//
// Created on 3/3/15.
// Copyright (c) 2015 Twitter, Inc. All rights reserved.
//
#include <pthread.h>
#import "TIP_Project.h"
#import "TIPError.h"
#import "TIPFileUtils.h"
#import "TIPGlobalConfiguration+Project.h"
#import "TIPImageCacheEntry.h"
#import "TIPImageDiskCache.h"
#import "TIPImageDiskCacheTemporaryFile.h"
#import "TIPImagePipelineInspectionResult+Project.h"
#import "TIPPartialImage.h"
#import "TIPTiming.h"
NS_ASSUME_NONNULL_BEGIN
static NSString * const kPartialImageExtension = @"tmp";
static NSString * const kXAttributeContextTTLKey = @"TTL";
static NSString * const kXAttributeContextUpdateTLLOnAccessKey = @"uTTL";
static NSString * const kXAttributeContextTreatAsPlaceholderKey = @"pl";
static NSString * const kXAttributeContextURLKey = @"URL";
static NSString * const kXAttributeContextLastAccessKey = @"LAD";
static NSString * const kXAttributeContextLastModifiedKey = @"LMD";
static NSString * const kXAttributeContextExpectedSizeKey = @"clen";
static NSString * const kXAttributeContextDimensionXKey = @"dX";
static NSString * const kXAttributeContextDimensionYKey = @"dY";
static NSString * const kXAttributeContextAnimated = @"ANI";
static NSDictionary<NSString *, Class> *_XAttributesKeysToKindsMap(void);
static NSDictionary<NSString *, Class> *_XAttributesKeysToKindsMap()
{
static NSDictionary *sMap;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
sMap = @{
kXAttributeContextTTLKey : [NSNumber class],
kXAttributeContextUpdateTLLOnAccessKey : [NSNumber class], // BOOL
kXAttributeContextTreatAsPlaceholderKey : [NSNumber class], // BOOL
kXAttributeContextURLKey : [NSURL class],
kXAttributeContextLastAccessKey : [NSDate class],
kXAttributeContextLastModifiedKey : [NSString class], // Only used for partial entries
kXAttributeContextExpectedSizeKey : [NSNumber class], // Only used for partial entries
kXAttributeContextDimensionXKey : [NSNumber class],
kXAttributeContextDimensionYKey : [NSNumber class],
kXAttributeContextAnimated : [NSNumber class], // BOOL
};
});
return sMap;
}
static NSDictionary<NSString *, Class> *_XAttributesKeysToKindsMapForCompleteEntry(void);
static NSDictionary<NSString *, Class> *_XAttributesKeysToKindsMapForCompleteEntry()
{
static NSDictionary *sMap;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
NSMutableDictionary *attributes = [_XAttributesKeysToKindsMap() mutableCopy];
[attributes removeObjectsForKeys:@[kXAttributeContextLastModifiedKey, kXAttributeContextExpectedSizeKey]];
sMap = [attributes copy];
});
return sMap;
}
static NSDictionary * __nullable _XAttributesFromContext(TIPImageCacheEntryContext * __nullable context);
static TIPImageCacheEntryContext * __nullable _ContextFromXAttributes(NSDictionary *xattrs,
BOOL notYetComplete);
static NSOperation *
_ImageDiskCacheManifestLoadOperation(NSMutableDictionary<NSString *, TIPImageDiskCacheEntry *> *manifest,
NSMutableArray<NSString *> *falseEntryPaths,
NSMutableArray<TIPImageDiskCacheEntry *> *entries,
unsigned long long *totalSizeInOut,
NSURL *entryURL,
NSString *cachePath,
NSDate *timestamp,
NSOperationQueue *manifestCacheQueue,
NSOperation *finalCacheOperation);
static BOOL _UpdateImageConditionCheck(const BOOL force,
const BOOL oldWasPlaceholder,
const BOOL newIsPlaceholder,
const BOOL extraCondition,
const CGSize newDimensions,
const CGSize oldDimensions,
NSURL * __nullable oldURL,
NSURL * __nullable newURL);
static void _SortEntries(NSMutableArray<TIPImageDiskCacheEntry *> *entries);
NS_INLINE NSString *_CreateTempFilePath()
{
return [NSTemporaryDirectory() stringByAppendingPathComponent:[[NSUUID UUID] UUIDString]];
}
static NSOperationQueue *_ImageDiskCacheManifestCacheQueue(void); // serial
static NSOperationQueue *_ImageDiskCacheManifestIOQueue(void); // concurrent
static dispatch_queue_t _ImageDiskCacheManifestAccessQueue(void); // serial
@interface TIPImageDiskCache () <TIPLRUCacheDelegate>
@property (tip_atomic_direct) SInt64 atomicTotalSize;
- (NSString *)filePathForSafeIdentifier:(NSString *)safeIdentifier TIP_OBJC_DIRECT;
@end
TIP_OBJC_DIRECT_MEMBERS
@interface TIPImageDiskCache (Background)
- (nullable NSString *)_diskCache_copyImageEntryToTemporaryFile:(NSString *)unsafeIdentifier
error:(out NSError * __nullable * __nullable)errorOut;
- (nullable TIPImageDiskCacheEntry *)_diskCache_getImageEntry:(NSString *)unsafeIdentifier
options:(TIPImageDiskCacheFetchOptions)options
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap;
- (nullable TIPImageDiskCacheEntry *)_diskCache_getImageEntryDirectlyFromDisk:(NSString *)unsafeIdentifier
options:(TIPImageDiskCacheFetchOptions)options
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap;
- (nullable TIPImageDiskCacheEntry *)_diskCache_getImageEntryFromManifest:(NSString *)unsafeIdentifier
options:(TIPImageDiskCacheFetchOptions)options
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap;
- (void)_diskCache_updateImageEntry:(TIPImageCacheEntry *)entry
forciblyReplaceExisting:(BOOL)forciblyReplaceExisting
safeIdentifier:(NSString *)safeIdentifier;
- (BOOL)_diskCache_touchImage:(NSString *)safeIdentifier
forced:(BOOL)forced;
- (void)_diskCache_touchEntry:(nullable TIPImageDiskCacheEntry *)entry
forced:(BOOL)forced
partial:(BOOL)partial;
- (void)_diskCache_finalizeTemporaryFile:(TIPImageDiskCacheTemporaryFile *)tempFile
context:(TIPImageCacheEntryContext *)context;
- (void)_diskCache_clearAllImages;
- (void)_diskCache_ensureCacheDirectoryExists;
- (void)_diskCache_updateByteCountsAdded:(UInt64)bytesAdded
removed:(UInt64)bytesRemoved;
- (BOOL)_diskCache_renameImageEntryWithOldIdentifier:(NSString *)oldIdentifier
newIdentifier:(NSString *)newIdentifier
error:(out NSError * __nullable * __nullable)errorOut;
- (void)_diskCache_populateEntryWithCompleteImage:(TIPImageDiskCacheEntry *)entry
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap;
- (void)_diskCache_populateEntryWithPartialImage:(TIPImageDiskCacheEntry *)entry
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap;
- (void)_diskCache_populateEntryWithTemporaryFile:(TIPImageDiskCacheEntry *)entry;
- (void)_diskCache_inspect:(TIPInspectableCacheCallback)callback;
@end
typedef void(^TIPImageDiskCacheManifestPopulateEntriesCompletionBlock)(unsigned long long totalSize,
NSArray<TIPImageDiskCacheEntry *> * __nullable entries,
NSArray<NSString *> * __nullable falseEntryPaths);
TIP_OBJC_DIRECT_MEMBERS
@interface TIPImageDiskCache (Manifest)
- (void)_manifest_populateManifestWithCachePath:(NSString *)cachePath;
- (void)_manifest_populateEntriesWithCachePath:(NSString *)cachePath
completion:(TIPImageDiskCacheManifestPopulateEntriesCompletionBlock)completionBlock;
- (void)_manifest_finalizePopulateManifest:(NSArray<TIPImageDiskCacheEntry *> *)entries
totalSize:(unsigned long long)totalSize;
@end
@implementation TIPImageDiskCache
{
TIPGlobalConfiguration *_globalConfig;
dispatch_queue_t _manifestQueue;
UInt64 _earlyRemovedBytesSize;
TIPLRUCache *_manifest;
pthread_mutex_t _manifestMutex;
struct {
BOOL manifestIsLoading:1;
} _diskCache_flags;
}
- (TIPLRUCache *)manifest
{
__block TIPLRUCache *manifest = nil;
// Perform a thread safe double-NULL check.
// This should keep perf up for the common case
// with a slowdown in the rare case of accessing
// manifest while it is still loading.
// 1) thread safe get the manfiest
dispatch_sync(_manifestQueue, ^{
manifest = self->_manifest;
});
// nil manifest?
if (!manifest) {
// 2) thread safe wait until the loading completes via mutex
//
// This mutex will be locked by manifest loading which is
// kicked of at "init" time and always ends with the mutex
// being unlocked.
// Performing a lock/unlock here ensures we wait for the manifest
// before continuing
pthread_mutex_lock(&_manifestMutex);
pthread_mutex_unlock(&_manifestMutex);
// 3) loading completed, thread safe get the non-nil manifest
// (unless there was an error, then we'll have an invalid disk cache w/ nil manifest)
dispatch_sync(_manifestQueue, ^{
manifest = self->_manifest;
});
}
TIPAssert(manifest != nil);
return (TIPLRUCache * _Nonnull)manifest; // TIPAssert() performed 1 line above
}
- (NSUInteger)totalCost
{
return (NSUInteger)self.atomicTotalSize;
}
- (TIPImageCacheType)cacheType
{
return TIPImageCacheTypeDisk;
}
- (instancetype)initWithPath:(NSString *)cachePath
{
if (self = [super init]) {
TIPAssert(cachePath != nil);
_cachePath = [cachePath copy];
_globalConfig = [TIPGlobalConfiguration sharedInstance];
_manifestQueue = _ImageDiskCacheManifestAccessQueue();
_diskCache_flags.manifestIsLoading = YES;
pthread_mutex_init(&_manifestMutex, NULL);
pthread_mutex_lock(&_manifestMutex);
cachePath = _cachePath; // reassign local var to immutable ivar for async usage
tip_dispatch_async_autoreleasing(_manifestQueue, ^{
[self _manifest_populateManifestWithCachePath:cachePath];
});
}
return self;
}
- (void)dealloc
{
pthread_mutex_destroy(&_manifestMutex);
// Don't delete the on disk cache, but do remove the cache's total bytes from our global count of total bytes
const SInt64 totalSize = self.atomicTotalSize;
const SInt16 totalCount = (SInt16)_manifest.numberOfEntries;
TIPGlobalConfiguration *config = _globalConfig;
tip_dispatch_async_autoreleasing(config.queueForDiskCaches, ^{
config.internalTotalBytesForAllDiskCaches -= totalSize;
config.internalTotalCountForAllDiskCaches -= totalCount;
});
}
- (BOOL)renameImageEntryWithIdentifier:(NSString *)oldIdentifier
toIdentifier:(NSString *)newIdentifier
error:(NSError * __nullable * __nullable)error
{
__block BOOL success = NO;
__block NSError *outerError;
tip_dispatch_sync_autoreleasing(_globalConfig.queueForDiskCaches, ^{
NSError *innerError;
success = [self _diskCache_renameImageEntryWithOldIdentifier:oldIdentifier
newIdentifier:newIdentifier
error:&innerError];
outerError = innerError;
});
if (error) {
*error = outerError;
}
return success;
}
- (nullable NSString *)copyImageEntryFileForIdentifier:(NSString *)identifier
error:(out NSError * __autoreleasing __nullable * __nullable)error
{
TIPAssert(identifier != nil);
if (!identifier) {
return nil;
}
__block NSError *outerError;
__block NSString *tempFilePath;
tip_dispatch_sync_autoreleasing(_globalConfig.queueForDiskCaches, ^{
NSError *innerError;
tempFilePath = [self _diskCache_copyImageEntryToTemporaryFile:identifier
error:&innerError];
outerError = innerError;
});
if (error) {
*error = outerError;
}
return tempFilePath;
}
- (nullable TIPImageDiskCacheEntry *)imageEntryForIdentifier:(NSString *)identifier
options:(TIPImageDiskCacheFetchOptions)options
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap
{
if (!identifier) {
return nil;
}
__block TIPImageDiskCacheEntry *entry;
tip_dispatch_sync_autoreleasing(_globalConfig.queueForDiskCaches, ^{
entry = [self _diskCache_getImageEntry:identifier
options:options
targetDimensions:targetDimensions
targetContentMode:targetContentMode
decoderConfigMap:decoderConfigMap];
});
return entry;
}
- (void)updateImageEntry:(TIPImageCacheEntry *)entry forciblyReplaceExisting:(BOOL)force
{
TIPAssert(entry.identifier != nil);
if (!entry.identifier) {
return;
}
tip_dispatch_async_autoreleasing(_globalConfig.queueForDiskCaches, ^{
[self _diskCache_updateImageEntry:entry
forciblyReplaceExisting:force
safeIdentifier:TIPSafeFromRaw(entry.identifier)];
});
}
- (void)clearImageWithIdentifier:(NSString *)identifier
{
if (!identifier) {
return;
}
tip_dispatch_async_autoreleasing(_globalConfig.queueForDiskCaches, ^{
TIPLRUCache *manifest = [self diskCache_syncAccessManifest];
TIPImageDiskCacheEntry *entry = (TIPImageDiskCacheEntry *)[manifest entryWithIdentifier:TIPSafeFromRaw(identifier)];
[manifest removeEntry:entry];
});
}
- (void)clearAllImages:(nullable void (^)(void))completion
{
tip_dispatch_async_autoreleasing(_globalConfig.queueForDiskCaches, ^{
[self _diskCache_clearAllImages];
if (completion) {
completion();
}
});
}
- (void)prune
{
tip_dispatch_async_autoreleasing(_globalConfig.queueForDiskCaches, ^{
[self->_globalConfig pruneAllCachesOfType:self.cacheType
withPriorityCache:nil];
});
}
- (void)touchImageWithIdentifier:(NSString *)imageIdentifier
orSaveImageEntry:(nullable TIPImageDiskCacheEntry *)entry
{
if (entry) {
TIPAssert(entry && [imageIdentifier isEqualToString:entry.identifier]);
if (![imageIdentifier isEqualToString:entry.identifier]) {
return;
}
} else {
TIPAssert(!entry && imageIdentifier != nil);
if (!imageIdentifier) {
return;
}
}
tip_dispatch_async_autoreleasing(_globalConfig.queueForDiskCaches, ^{
NSString *safeIdentifier = TIPSafeFromRaw(imageIdentifier);
if (![self _diskCache_touchImage:safeIdentifier forced:NO] && entry) {
[self _diskCache_updateImageEntry:entry
forciblyReplaceExisting:NO
safeIdentifier:safeIdentifier];
}
});
}
- (TIPImageDiskCacheTemporaryFile *)openTemporaryFileForImageIdentifier:(NSString *)imageIdentifier
{
TIPAssert(imageIdentifier != nil);
if (!imageIdentifier) {
return nil;
}
NSString *finalPath = [self filePathForSafeIdentifier:TIPSafeFromRaw(imageIdentifier)];
TIPImageDiskCacheTemporaryFile *tempFile;
tempFile = [[TIPImageDiskCacheTemporaryFile alloc] initWithIdentifier:imageIdentifier
temporaryPath:_CreateTempFilePath()
finalPath:finalPath
diskCache:self];
return tempFile;
}
- (void)finalizeTemporaryFile:(TIPImageDiskCacheTemporaryFile *)tempFile
withContext:(TIPImageCacheEntryContext *)context
{
TIPAssert(tempFile.imageIdentifier != nil);
if (!tempFile.imageIdentifier) {
return;
}
tip_dispatch_async_autoreleasing(_globalConfig.queueForDiskCaches, ^{
[self _diskCache_finalizeTemporaryFile:tempFile
context:context];
});
}
- (void)clearTemporaryFilePath:(NSString *)filePath
{
if (!filePath) {
return;
}
tip_dispatch_async_autoreleasing(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_LOW, 0), ^{
[[NSFileManager defaultManager] removeItemAtPath:filePath error:NULL];
});
}
- (NSString *)filePathForSafeIdentifier:(NSString *)safeIdentifier
{
TIPAssert(safeIdentifier != nil);
if (!safeIdentifier) {
return nil;
}
TIPAssert(_cachePath != nil);
return [_cachePath stringByAppendingPathComponent:safeIdentifier];
}
#pragma mark TIPLRUCacheDelegate
- (void)tip_cache:(TIPLRUCache *)manifest didEvictEntry:(TIPImageDiskCacheEntry *)entry
{
const NSUInteger size = entry.completeFileSize + entry.partialFileSize;
_globalConfig.internalTotalCountForAllDiskCaches -= 1;
[self _diskCache_updateByteCountsAdded:0 removed:size];
NSFileManager *fm = [NSFileManager defaultManager];
NSString *filePath = [self filePathForSafeIdentifier:entry.safeIdentifier];
NSString *partialFilePath = [filePath stringByAppendingPathExtension:kPartialImageExtension];
[fm removeItemAtPath:filePath error:NULL];
[fm removeItemAtPath:partialFilePath error:NULL];
TIPLogDebug(@"%@ Evicted '%@', complete:'%@', partial:'%@'", NSStringFromClass([self class]), entry.safeIdentifier, entry.completeImageContext.URL, entry.partialImageContext.URL);
}
#pragma mark Inspect
- (void)inspect:(TIPInspectableCacheCallback)callback
{
tip_dispatch_async_autoreleasing(_globalConfig.queueForDiskCaches, ^{
[self _diskCache_inspect:callback];
});
}
@end
@implementation TIPImageDiskCache (Background)
- (void)_diskCache_updateByteCountsAdded:(UInt64)bytesAdded
removed:(UInt64)bytesRemoved
{
// are we decrementing our byte count before the manifest has finished loading?
if (bytesRemoved > bytesAdded && _diskCache_flags.manifestIsLoading) {
// this would cause the manifest to become negative
// instead, delay the decrement until later and just deal with the increment
_earlyRemovedBytesSize += bytesRemoved;
TIPLogWarning(@"Decrementing disk cache size before the Manifest finished loading! It's OK though, we'll delay the subtracting until later. Added: %llu, Sub'd: %llu", bytesAdded, bytesRemoved);
bytesRemoved = 0;
}
TIP_UPDATE_BYTES(self.atomicTotalSize, bytesAdded, bytesRemoved, @"Disk Cache Size");
TIP_UPDATE_BYTES(_globalConfig.internalTotalBytesForAllDiskCaches, bytesAdded, bytesRemoved, @"All Disk Caches Size");
}
- (void)_diskCache_ensureCacheDirectoryExists
{
[[NSFileManager defaultManager] createDirectoryAtPath:_cachePath
withIntermediateDirectories:YES
attributes:nil
error:NULL];
}
- (nullable NSString *)_diskCache_copyImageEntryToTemporaryFile:(NSString *)unsafeIdentifier
error:(out NSError * __nullable * __nullable)errorOut
{
NSString *temporaryFilePath = nil;
NSError *fileCopyError = nil;
NSString *filePath = [self diskCache_imageEntryFilePathForIdentifier:unsafeIdentifier
hitShouldMoveEntryToHead:YES
context:NULL];
if (filePath) {
NSFileManager *fm = [NSFileManager defaultManager];
temporaryFilePath = [NSTemporaryDirectory() stringByAppendingPathComponent:[NSUUID UUID].UUIDString];
[fm createDirectoryAtPath:temporaryFilePath.stringByDeletingLastPathComponent
withIntermediateDirectories:YES
attributes:NULL
error:NULL];
if (![fm copyItemAtPath:filePath toPath:temporaryFilePath error:&fileCopyError]) {
temporaryFilePath = nil;
}
}
if (!temporaryFilePath && !fileCopyError) {
fileCopyError = [NSError errorWithDomain:NSPOSIXErrorDomain
code:ENOENT
userInfo:nil];
}
if (errorOut) {
*errorOut = fileCopyError;
}
return temporaryFilePath;
}
- (nullable TIPImageDiskCacheEntry *)_diskCache_getImageEntry:(NSString *)unsafeIdentifier
options:(TIPImageDiskCacheFetchOptions)options
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap
{
TIPImageDiskCacheEntry *entry = nil;
if (_diskCache_flags.manifestIsLoading) {
entry = [self _diskCache_getImageEntryDirectlyFromDisk:unsafeIdentifier
options:options
targetDimensions:targetDimensions
targetContentMode:targetContentMode
decoderConfigMap:decoderConfigMap];
} else {
entry = [self _diskCache_getImageEntryFromManifest:unsafeIdentifier
options:options
targetDimensions:targetDimensions
targetContentMode:targetContentMode
decoderConfigMap:decoderConfigMap];
}
return entry;
}
- (nullable TIPImageDiskCacheEntry *)_diskCache_getImageEntryDirectlyFromDisk:(NSString *)unsafeIdentifier
options:(TIPImageDiskCacheFetchOptions)options
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap
{
TIPImageDiskCacheEntry *entry = nil;
NSFileManager *fm = [NSFileManager defaultManager];
NSString *safeIdentifer = TIPSafeFromRaw(unsafeIdentifier);
NSString *filePath = [self filePathForSafeIdentifier:safeIdentifer];
if ([fm fileExistsAtPath:filePath]) {
const NSUInteger size = TIPFileSizeAtPath(filePath, NULL);
if (size) {
NSDictionary *xattributes = TIPGetXAttributesForFile(filePath, _XAttributesKeysToKindsMap());
TIPImageCacheEntryContext *context = _ContextFromXAttributes(xattributes, NO);
if ([context isKindOfClass:[TIPCompleteImageEntryContext class]]) {
entry = [[TIPImageDiskCacheEntry alloc] init];
entry.identifier = unsafeIdentifier;
entry.completeImageContext = (id)context;
entry.completeFileSize = size;
if (TIP_BITMASK_HAS_SUBSET_FLAGS(options, TIPImageDiskCacheFetchOptionCompleteImage)) {
NSData *data = [NSData dataWithContentsOfURL:[NSURL fileURLWithPath:filePath]
options:(context.isAnimated) ? NSDataReadingMappedIfSafe : 0
error:NULL];
TIPImageContainer *image = [TIPImageContainer imageContainerWithData:data
targetDimensions:targetDimensions
targetContentMode:targetContentMode
decoderConfigMap:decoderConfigMap
codecCatalogue:nil];
if (image) {
entry.completeImage = image;
entry.completeImageData = data;
} else {
entry = nil;
}
}
}
}
}
return entry;
}
- (nullable TIPImageDiskCacheEntry *)_diskCache_getImageEntryFromManifest:(NSString *)unsafeIdentifier
options:(TIPImageDiskCacheFetchOptions)options
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap
{
TIPLRUCache *manifest = [self diskCache_syncAccessManifest];
NSString *safeIdentifer = TIPSafeFromRaw(unsafeIdentifier);
TIPImageDiskCacheEntry *entry = (TIPImageDiskCacheEntry *)[manifest entryWithIdentifier:safeIdentifer];
if (entry) {
// Validate TTL
NSDate *now = [NSDate date];
NSDate *lastAccess = nil;
const NSUInteger oldCost = entry.completeFileSize + entry.partialFileSize;
lastAccess = entry.partialImageContext.lastAccess;
if (lastAccess && [now timeIntervalSinceDate:lastAccess] > entry.partialImageContext.TTL) {
entry.partialImageContext = nil;
entry.partialImage = nil;
entry.partialFileSize = 0;
}
lastAccess = entry.completeImageContext.lastAccess;
if (lastAccess && [now timeIntervalSinceDate:lastAccess] > entry.completeImageContext.TTL) {
entry.completeImageContext = nil;
entry.completeImage = nil;
entry.completeFileSize = 0;
}
// Resolve changes to entry
const NSUInteger newCost = entry.completeFileSize + entry.partialFileSize;
if (!newCost) {
[manifest removeEntry:entry];
entry = nil;
} else {
[self _diskCache_updateByteCountsAdded:newCost removed:oldCost];
TIPAssert(newCost <= oldCost); // removing the cache image and/or partial image only ever removes bytes
}
if (entry) {
// Update entry
if (![entry.identifier isEqualToString:unsafeIdentifier]) {
// Entries read from disk can have hashed identifiers.
// If the safe identifiers match but the unsafe ones don't,
// we can safely update the existing entry's identifier.
entry.identifier = unsafeIdentifier;
}
[self _diskCache_touchImage:safeIdentifer forced:NO];
// Mutate and return a copy
entry = [entry copy];
if (TIP_BITMASK_HAS_SUBSET_FLAGS(options, TIPImageDiskCacheFetchOptionCompleteImage)) {
[self _diskCache_populateEntryWithCompleteImage:entry
targetDimensions:targetDimensions
targetContentMode:targetContentMode
decoderConfigMap:decoderConfigMap];
}
if (TIP_BITMASK_HAS_SUBSET_FLAGS(options, TIPImageDiskCacheFetchOptionPartialImage) || (TIP_BITMASK_HAS_SUBSET_FLAGS(options, TIPImageDiskCacheFetchOptionPartialImageIfNoCompleteImage) && !entry.completeImageContext)) {
[self _diskCache_populateEntryWithPartialImage:entry
decoderConfigMap:decoderConfigMap];
}
if (TIP_BITMASK_HAS_SUBSET_FLAGS(options, TIPImageDiskCacheFetchOptionTemporaryFile) || (TIP_BITMASK_HAS_SUBSET_FLAGS(options, TIPImageDiskCacheFetchOptionTemporaryFileIfNoCompleteImage) && !entry.completeImageContext)) {
[self _diskCache_populateEntryWithTemporaryFile:entry];
}
}
}
return entry;
}
- (void)_diskCache_populateEntryWithCompleteImage:(TIPImageDiskCacheEntry *)entry
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap
{
if (entry.completeImageContext) {
NSString *filePath = [self filePathForSafeIdentifier:entry.safeIdentifier];
TIPAssertMessage(filePath != nil, @"entry.identifier = %@", entry.identifier);
if (filePath) {
const BOOL memoryMap = entry.completeImageContext.isAnimated;
NSData *data = [NSData dataWithContentsOfURL:[NSURL fileURLWithPath:filePath]
options:(memoryMap) ? NSDataReadingMappedIfSafe : 0
error:NULL];
entry.completeImage = [TIPImageContainer imageContainerWithData:data
targetDimensions:targetDimensions
targetContentMode:targetContentMode
decoderConfigMap:decoderConfigMap
codecCatalogue:nil];
entry.completeImageData = data;
}
}
}
- (void)_diskCache_populateEntryWithPartialImage:(TIPImageDiskCacheEntry *)entry
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap
{
if (entry.partialImageContext) {
NSString *filePath = [self filePathForSafeIdentifier:entry.safeIdentifier];
filePath = [filePath stringByAppendingPathExtension:kPartialImageExtension];
TIPAssertMessage(filePath != nil, @"entry.identifier = %@", entry.identifier);
if (filePath) {
NSData *data = [NSData dataWithContentsOfFile:filePath];
if (data.length > 0) {
TIPPartialImage *partialImage;
partialImage = [[TIPPartialImage alloc] initWithExpectedContentLength:entry.partialImageContext.expectedContentLength];
[partialImage updateDecoderConfigMap:decoderConfigMap];
[partialImage appendData:data final:NO];
entry.partialImage = partialImage;
}
}
}
}
- (void)_diskCache_populateEntryWithTemporaryFile:(TIPImageDiskCacheEntry *)entry
{
if (entry.partialImageContext) {
NSString *finalPath = [self filePathForSafeIdentifier:entry.safeIdentifier];
NSString *partialPath = [finalPath stringByAppendingPathExtension:kPartialImageExtension];
NSString *tempPath = _CreateTempFilePath();
TIPAssertMessage(tempPath != nil, @"entry.identifier = %@", entry.identifier);
TIPAssertMessage(partialPath != nil, @"entry.identifier = %@", entry.identifier);
if (tempPath && partialPath && [[NSFileManager defaultManager] copyItemAtPath:partialPath toPath:tempPath error:NULL]) {
entry.tempFile = [[TIPImageDiskCacheTemporaryFile alloc] initWithIdentifier:entry.identifier
temporaryPath:tempPath
finalPath:finalPath
diskCache:self];
}
}
}
- (void)_diskCache_updateImageEntry:(TIPImageCacheEntry *)entry
forciblyReplaceExisting:(BOOL)forciblyReplaceExisting
safeIdentifier:(NSString *)safeIdentifier
{
// Validate entry first
if (!entry.identifier) {
return;
}
if (!entry.partialImageContext ^ !entry.partialImage) {
return;
}
if (!entry.completeImageContext ^ (!entry.completeImage && !entry.completeImageData && !entry.completeImageFilePath)) {
return;
}
[self _diskCache_ensureCacheDirectoryExists];
// Get the "existing" entry
TIPLRUCache *manifest = [self diskCache_syncAccessManifest];
TIPImageDiskCacheEntry *existingEntry = (TIPImageDiskCacheEntry *)[manifest entryWithIdentifier:safeIdentifier];
const BOOL hasPreviousEntry = (existingEntry != nil);
if (!existingEntry) {
if (forciblyReplaceExisting || entry.completeImageContext || entry.partialImageContext) {
existingEntry = [[TIPImageDiskCacheEntry alloc] init];
existingEntry.identifier = entry.identifier;
}
} else { // existingEntry
if (![existingEntry.identifier isEqualToString:entry.identifier]) {
// Entries read from disk can have hashed identifiers.
// If the safe identifiers match but the unsafe ones don't,
// we can safely update the existing entry's identifier.
existingEntry.identifier = entry.identifier;
}
}
// Set up variables
BOOL didChangePartial = NO, didChangeComplete = NO;
NSFileManager *fm = [NSFileManager defaultManager];
NSString *filePath = [self filePathForSafeIdentifier:safeIdentifier];
NSString *partialFilePath = [filePath stringByAppendingPathExtension:kPartialImageExtension];
// Check file path was generated
if (!filePath) {
NSMutableDictionary *userInfo = [[NSMutableDictionary alloc] init];
if (entry.identifier) {
userInfo[TIPProblemInfoKeyImageIdentifier] = entry.identifier;
}
if (safeIdentifier) {
userInfo[TIPProblemInfoKeySafeImageIdentifier] = safeIdentifier;
}
NSURL *contextURL = entry.completeImageContext.URL ?: entry.partialImageContext.URL;
if (contextURL) {
userInfo[TIPProblemInfoKeyImageURL] = contextURL;
}
if (_cachePath) {
// Context is helpful, but needn't expose it with a constant.
userInfo[@"cachePath"] = _cachePath;
}
[_globalConfig postProblem:TIPProblemDiskCacheUpdateImageEntryCouldNotGenerateFileName
userInfo:userInfo];
}
const NSUInteger oldCost = existingEntry.partialFileSize + existingEntry.completeFileSize;
CGSize oldDimensions;
CGSize newDimensions;
BOOL oldWasPlaceholder;
BOOL newIsPlaceholder;
BOOL conditionMetToUpdate;
// Update complete image
oldDimensions = existingEntry.completeImageContext.dimensions;
newDimensions = entry.completeImageContext.dimensions;
oldWasPlaceholder = existingEntry.completeImageContext.treatAsPlaceholder;
newIsPlaceholder = entry.completeImageContext.treatAsPlaceholder;
conditionMetToUpdate = _UpdateImageConditionCheck(forciblyReplaceExisting,
oldWasPlaceholder,
newIsPlaceholder,
NO /*extra*/,
newDimensions,
oldDimensions,
existingEntry.completeImageContext.URL,
entry.completeImageContext.URL);
if (conditionMetToUpdate) {
existingEntry.completeImageContext = nil;
existingEntry.completeFileSize = 0;
if (filePath) {
[fm removeItemAtPath:filePath error:NULL];
}
if (entry.completeImage || entry.completeImageData || entry.completeImageFilePath) {
BOOL success = NO;
NSError *error = nil;
if (filePath) {
if (entry.completeImage) {
success = [entry.completeImage saveToFilePath:filePath
type:entry.completeImageContext.imageType
codecCatalogue:nil
options:TIPImageEncodingNoOptions
quality:kTIPAppleQualityValueRepresentingJFIFQuality85
atomic:YES
error:&error];
} else if (entry.completeImageData) {
success = [entry.completeImageData writeToFile:filePath
options:NSDataWritingAtomic
error:&error];
} else {
success = [fm copyItemAtPath:entry.completeImageFilePath
toPath:filePath
error:&error];
}
}
if (success) {
existingEntry.completeImageContext = [entry.completeImageContext copy];
existingEntry.completeFileSize = (NSUInteger)TIPFileSizeAtPath(filePath, NULL);
// Clear partial on new entry since we set the complete image
entry.partialImage = nil;
entry.partialImageContext = nil;
} else {
NSString *key = nil;
id value = nil;
if (entry.completeImage) {
key = @"image";
value = entry.completeImage;
} else if (entry.completeImageData) {
key = @"imageData";
value = [NSString stringWithFormat:@"<Data: length=%tu>", entry.completeImageData.length];
} else {
key = @"imageFilePath";
value = entry.completeImageFilePath;
}
TIPLogWarning(@"Failed to update disk cache entry! %@", @{
@"filePath" : (filePath) ?: @"<null>",
key : value,
@"URL" : entry.completeImageContext.URL,
@"id" : entry.identifier,
@"error" : (error) ?: @"???"
});
}
}
didChangeComplete = YES;
}
// Update partial image
oldDimensions = existingEntry.partialImageContext.dimensions;
oldWasPlaceholder = existingEntry.partialImageContext.treatAsPlaceholder;
newIsPlaceholder = entry.partialImageContext.treatAsPlaceholder;
if (!didChangeComplete) {
newDimensions = entry.partialImageContext.dimensions;
}
conditionMetToUpdate = NO;
if (existingEntry.partialImageContext != nil || entry.partialImageContext != nil) {
// only both if there is a partial image to care about
conditionMetToUpdate = _UpdateImageConditionCheck(forciblyReplaceExisting,
oldWasPlaceholder,
newIsPlaceholder,
(oldWasPlaceholder && didChangeComplete) /*extra*/,
newDimensions,
oldDimensions,
existingEntry.partialImageContext.URL,
entry.partialImageContext.URL);
}
if (conditionMetToUpdate) {
existingEntry.partialImageContext = nil;
existingEntry.partialFileSize = 0;
if (partialFilePath) {
[fm removeItemAtPath:partialFilePath error:NULL];
if (entry.partialImage && !newIsPlaceholder) {
NSError *error = nil;
if ([entry.partialImage.data writeToFile:partialFilePath options:NSDataWritingAtomic error:&error]) {
existingEntry.partialImageContext = [entry.partialImageContext copy];
existingEntry.partialFileSize = entry.partialImage.byteCount;
} else {
TIPLogError(@"Failed to write partial image! %@", @{ @"data.length" : @(entry.partialImage.data.length), @"filePath" : partialFilePath, @"error" : (error) ?: @"???" });
}
}
}
didChangePartial = YES;
}
// Nothing changed
if (!didChangePartial && !didChangeComplete) {
return;
}
// Cap our entry size
const SInt64 max = [_globalConfig internalMaxBytesForCacheEntryOfType:self.cacheType];
if (existingEntry && (SInt64)existingEntry.partialFileSize > max) {
NSDictionary *userInfo = @{
TIPProblemInfoKeyImageIdentifier : existingEntry.identifier,
TIPProblemInfoKeySafeImageIdentifier : existingEntry.safeIdentifier,
TIPProblemInfoKeyImageURL : existingEntry.partialImageContext.URL,
TIPProblemInfoKeyImageDimensions : [NSValue valueWithCGSize:existingEntry.partialImageContext.dimensions],
@"expectedSize" : @(existingEntry.partialImageContext.expectedContentLength),
@"partialSize" : @(existingEntry.partialFileSize),
};
[_globalConfig postProblem:TIPProblemImageTooLargeToStoreInDiskCache userInfo:userInfo];
if (partialFilePath) {
[fm removeItemAtPath:partialFilePath error:NULL];
}
existingEntry.partialImage = nil;
existingEntry.partialImageContext = nil;
existingEntry.partialFileSize = 0;
didChangePartial = YES;
}
if (existingEntry && (SInt64)existingEntry.completeFileSize > max) {
NSValue *dimensionsValue = [NSValue valueWithCGSize:existingEntry.completeImageContext.dimensions];
NSDictionary *userInfo = @{
TIPProblemInfoKeyImageIdentifier : existingEntry.identifier,
TIPProblemInfoKeySafeImageIdentifier : existingEntry.safeIdentifier,
TIPProblemInfoKeyImageURL : existingEntry.completeImageContext.URL,
TIPProblemInfoKeyImageDimensions : dimensionsValue,
@"size" : @(existingEntry.completeFileSize),
};
[_globalConfig postProblem:TIPProblemImageTooLargeToStoreInDiskCache userInfo:userInfo];
[fm removeItemAtPath:filePath error:NULL];
existingEntry.completeImage = nil;
existingEntry.completeImageContext = nil;
existingEntry.completeFileSize = 0;
didChangeComplete = YES;
}
if (!existingEntry.partialImageContext && !existingEntry.completeImageContext) {
// shoot... the save cannot complete -- purge the entry
if (hasPreviousEntry) {
[manifest removeEntry:existingEntry];
}
} else {
// Update xattrs and LRU
const NSUInteger newCost = existingEntry.partialFileSize + existingEntry.completeFileSize;
[self _diskCache_updateByteCountsAdded:newCost removed:oldCost];
if (!hasPreviousEntry && existingEntry) {
_globalConfig.internalTotalCountForAllDiskCaches += 1;
}
[manifest addEntry:existingEntry];
if (didChangePartial) {
[self _diskCache_touchEntry:existingEntry
forced:forciblyReplaceExisting
partial:YES];
}
if (didChangeComplete) {
[self _diskCache_touchEntry:existingEntry
forced:forciblyReplaceExisting
partial:NO];
}
if (gTwitterImagePipelineAssertEnabled) {
if (existingEntry.partialImageContext && 0 == existingEntry.partialFileSize) {
NSDictionary *info = @{
@"dimension" : NSStringFromCGSize(existingEntry.partialImageContext.dimensions),
@"URL" : existingEntry.partialImageContext.URL,
@"id" : existingEntry.identifier,
};
TIPLogError(@"Cached zero cost partial image to disk cache %@", info);
}
if (existingEntry.completeImageContext && 0 == existingEntry.completeFileSize) {
NSDictionary *info = @{
@"dimension" : NSStringFromCGSize(existingEntry.completeImageContext.dimensions),
@"URL" : existingEntry.completeImageContext.URL,
@"id" : existingEntry.identifier,
};
TIPLogError(@"Cached zero cost complete image to disk cache %@", info);
}
}
}
[_globalConfig pruneAllCachesOfType:self.cacheType withPriorityCache:self];
}
- (BOOL)_diskCache_touchImage:(NSString *)safeIdentifier
forced:(BOOL)forced
{
TIPLRUCache *manifest = [self diskCache_syncAccessManifest];
TIPImageDiskCacheEntry *entry = (TIPImageDiskCacheEntry *)[manifest entryWithIdentifier:safeIdentifier];
if (entry) {
[self _diskCache_touchEntry:entry
forced:forced
partial:YES];
[self _diskCache_touchEntry:entry
forced:forced
partial:NO];
}
return entry != nil;
}
- (void)_diskCache_touchEntry:(nullable TIPImageDiskCacheEntry *)entry
forced:(BOOL)forced
partial:(BOOL)partial
{
TIPImageCacheEntryContext *context = (partial) ? entry.partialImageContext : entry.completeImageContext;
if (!context) {
return;
}
if (context.updateExpiryOnAccess || !context.lastAccess) {
context.lastAccess = [NSDate date];
} else if (!forced) {
return;
}
NSDictionary *xattrs = _XAttributesFromContext(context);
NSString *filePath = [self filePathForSafeIdentifier:entry.safeIdentifier];
if (partial) {
filePath = [filePath stringByAppendingPathExtension:kPartialImageExtension];
}
TIPAssertMessage(filePath != nil, @"entry.identifier = %@", entry.identifier);
if (!filePath) {
return;
}
const NSUInteger numberOfSetXAttributes = TIPSetXAttributesForFile(xattrs, filePath);
if (numberOfSetXAttributes != xattrs.count) {
NSDictionary *info = @{
@"filePath" : filePath,
@"id" : entry.identifier,
@"safeId" : entry.safeIdentifier,
@"xattrs" : xattrs
};
TIPLogError(@"Error writing xattrs! (wrote %tu of %tu)\n%@", numberOfSetXAttributes, xattrs.count, info);
}
#if DEBUG
NSDictionary *xattrsRoundTrip = TIPGetXAttributesForFile(filePath, _XAttributesKeysToKindsMap());
TIPAssertMessage([xattrs isEqualToDictionary:xattrsRoundTrip], @"xattrs differ!\nSet: %@\nGet: %@", xattrs, xattrsRoundTrip);
#endif
}
- (void)_diskCache_clearAllImages
{
TIPStartMethodScopedBackgroundTask(ClearAllImages);
TIPLRUCache *manifest = [self diskCache_syncAccessManifest];
const SInt16 totalCount = (SInt16)manifest.numberOfEntries;
[manifest clearAllEntries];
[self _diskCache_updateByteCountsAdded:0 removed:(UInt64)self.atomicTotalSize];
_globalConfig.internalTotalCountForAllDiskCaches -= totalCount;
[[NSFileManager defaultManager] removeItemAtPath:_cachePath error:NULL];
TIPLogInformation(@"Cleared all images in %@", self);
}
- (void)_diskCache_finalizeTemporaryFile:(TIPImageDiskCacheTemporaryFile *)tempFile
context:(TIPImageCacheEntryContext *)context
{
NSString * const finalPath = tempFile.finalPath;
if (!finalPath) {
NSString *message = [NSString stringWithFormat:@"%@ has a nil finalPath. identifier: %@", NSStringFromClass([tempFile class]), tempFile.imageIdentifier];
TIPAssertMessage(finalPath != nil, @"%@", message);
TIPLogError(@"%@", message);
return;
}
NSString * const tempPath = tempFile.temporaryPath;
if (!tempPath) {
NSString *message = [NSString stringWithFormat:@"%@ has a nil temporaryPath. identifier: %@", NSStringFromClass([tempFile class]), tempFile.imageIdentifier];
TIPAssertMessage(tempPath != nil, @"%@", message);
TIPLogError(@"%@", message);
return;
}
NSString * const partialPath = [finalPath stringByAppendingPathExtension:kPartialImageExtension];
NSString * const safeIdentifier = [finalPath lastPathComponent];
TIPAssert([safeIdentifier isEqualToString:TIPSafeFromRaw(tempFile.imageIdentifier)]);
[self _diskCache_ensureCacheDirectoryExists];
BOOL const isPartial = [context isKindOfClass:[TIPPartialImageEntryContext class]];
if (!isPartial) {
if (![context isKindOfClass:[TIPCompleteImageEntryContext class]]) {
TIPAssertMessage(NO, @"Invalid or nil context provided!");
return;
}
}
if (isPartial && context.treatAsPlaceholder) {
// don't cache incomplete placeholders
[self clearTemporaryFilePath:tempFile.temporaryPath];
return;
}
NSUInteger const size = (NSUInteger)TIPFileSizeAtPath(tempFile.temporaryPath, NULL);
if (!size) {
[self clearTemporaryFilePath:tempFile.temporaryPath];
return;
}
NSFileManager * const fm = [NSFileManager defaultManager];
TIPLRUCache * const manifest = [self diskCache_syncAccessManifest];
TIPImageDiskCacheEntry *entry = (TIPImageDiskCacheEntry *)[manifest entryWithIdentifier:safeIdentifier];
TIPImageCacheEntryContext * const oldPartialContext = entry.partialImageContext;
TIPImageCacheEntryContext * const oldCompleteContext = entry.completeImageContext;
CGSize const newDimensions = context.dimensions;
CGSize const oldPartialDimensions = oldPartialContext.dimensions;
CGSize const oldCompleteDimensions = oldCompleteContext.dimensions;
// 1) Remove lowest fidelity entries where appropriate
if (entry) {
if (!isPartial) {
// This is a complete entry...
if (oldPartialContext) {
if ((oldPartialDimensions.width * oldPartialDimensions.height) <= (newDimensions.width * newDimensions.height)) {
// if the old partial image is smaller (or equal), remove it
[self _diskCache_updateByteCountsAdded:0
removed:entry.partialFileSize];
entry.partialFileSize = 0;
entry.partialImageContext = nil;
[fm removeItemAtPath:partialPath error:NULL];
}
}
if (oldCompleteContext) {
const BOOL oldSizeTooSmall = (oldCompleteDimensions.width * oldCompleteDimensions.height) < (newDimensions.width * newDimensions.height);
if (oldSizeTooSmall || oldCompleteContext.treatAsPlaceholder) {
// if the old complete image is smaller, remove it
[self _diskCache_updateByteCountsAdded:0
removed:entry.completeFileSize];
entry.completeFileSize = 0;
entry.completeImageContext = nil;
[fm removeItemAtPath:finalPath error:NULL];
} else {
// otherwise, clear ourself
[self clearTemporaryFilePath:tempFile.temporaryPath];
return;
}
}
} else {
// This is a partial entry...
if (oldPartialContext) {
if ((oldPartialDimensions.width * oldPartialDimensions.height) <= (newDimensions.width * newDimensions.height)) {
// if the old partial image is smaller (or equal), remove it
[self _diskCache_updateByteCountsAdded:0
removed:entry.partialFileSize];
entry.partialFileSize = 0;
entry.partialImageContext = nil;
[fm removeItemAtPath:partialPath error:NULL];
}
}
if (oldCompleteContext) {
if ((oldCompleteDimensions.width * oldCompleteDimensions.height) >= (newDimensions.width * newDimensions.height)) {
// if the old complete image is larger (or equal), clear ourselves
[self clearTemporaryFilePath:tempFile.temporaryPath];
return;
}
}
}
}
// 2) Move our new bytes into the disk cache
NSError *error;
if ([fm moveItemAtPath:tempFile.temporaryPath toPath:(isPartial) ? partialPath : finalPath error:&error]) {
context = [context copy];
const BOOL newEntry = !entry;
if (!entry) {
entry = [[TIPImageDiskCacheEntry alloc] init];
entry.identifier = tempFile.imageIdentifier;
}
if (isPartial) {
entry.partialFileSize = size;
entry.partialImageContext = (id)context;
} else {
entry.completeFileSize = size;
entry.completeImageContext = (id)context;
}
[self _diskCache_updateByteCountsAdded:size removed:0];
if (newEntry) {
_globalConfig.internalTotalCountForAllDiskCaches += 1;
}
if (gTwitterImagePipelineAssertEnabled) {
if (entry.partialImageContext && 0 == entry.partialFileSize) {
TIPLogError(@"Cached zero cost partial image to disk cache %@", @{
@"dimension" : NSStringFromCGSize(entry.partialImageContext.dimensions),
@"URL" : entry.partialImageContext.URL,
@"id" : entry.identifier,
});
}
if (entry.completeImageContext && 0 == entry.completeFileSize) {
TIPLogError(@"Cached zero cost complete image to disk cache %@", @{
@"dimension" : NSStringFromCGSize(entry.completeImageContext.dimensions),
@"URL" : entry.completeImageContext.URL,
@"id" : entry.identifier,
});
}
}
[manifest addEntry:entry];
[self _diskCache_touchEntry:entry
forced:YES
partial:isPartial];
[_globalConfig pruneAllCachesOfType:self.cacheType withPriorityCache:self];
} else {
TIPLogWarning(@"%@", error);
}
}
- (void)_diskCache_inspect:(TIPInspectableCacheCallback)callback
{
NSMutableArray *completedEntries = [[NSMutableArray alloc] init];
NSMutableArray *partialEntries = [[NSMutableArray alloc] init];
TIPLRUCache *manifest = [self diskCache_syncAccessManifest];
for (TIPImageDiskCacheEntry *cacheEntry in manifest) {
TIPImagePipelineInspectionResultEntry *entry;
Class resultClass;
resultClass = [TIPImagePipelineInspectionResultCompleteDiskEntry class];
entry = [TIPImagePipelineInspectionResultEntry entryWithCacheEntry:cacheEntry
class:resultClass];
if (entry) {
[completedEntries addObject:entry];
}
resultClass = [TIPImagePipelineInspectionResultPartialDiskEntry class];
entry = [TIPImagePipelineInspectionResultEntry entryWithCacheEntry:cacheEntry
class:resultClass];
if (entry) {
[partialEntries addObject:entry];
}
}
callback(completedEntries, partialEntries);
}
- (BOOL)_diskCache_renameImageEntryWithOldIdentifier:(NSString *)oldIdentifier
newIdentifier:(NSString *)newIdentifier
error:(out NSError * __nullable * __nullable)errorOut
{
NSString *oldSafeID = TIPSafeFromRaw(oldIdentifier);
TIPLRUCache *manifest = [self diskCache_syncAccessManifest];
TIPImageDiskCacheEntry *oldEntry = (TIPImageDiskCacheEntry *)[manifest entryWithIdentifier:oldSafeID
canMutate:NO];
if (!oldEntry) {
if (errorOut) {
*errorOut = [NSError errorWithDomain:NSPOSIXErrorDomain
code:ENOENT
userInfo:nil];
}
return NO;
}
NSString *newSafeID = TIPSafeFromRaw(newIdentifier);
TIPCompleteImageEntryContext *completeContext = oldEntry.completeImageContext;
NSString *oldCompleteFilePath = (completeContext) ? [self filePathForSafeIdentifier:oldSafeID] : nil;
TIPPartialImageEntryContext *partialContext = oldEntry.partialImageContext;
NSString *oldPartialFilePath = (partialContext) ? [[self filePathForSafeIdentifier:oldSafeID] stringByAppendingPathExtension:kPartialImageExtension] : nil;
NSError *error = nil;
BOOL fail = NO;
if (oldCompleteFilePath) {
NSString *newCompleteFilePath = [self filePathForSafeIdentifier:newSafeID];
fail = ![[NSFileManager defaultManager] moveItemAtPath:oldCompleteFilePath
toPath:newCompleteFilePath
error:&error];
}
if (!fail && oldPartialFilePath) {
NSString *newPartialFilePath = [[self filePathForSafeIdentifier:newSafeID] stringByAppendingPathExtension:kPartialImageExtension];
fail = ![[NSFileManager defaultManager] moveItemAtPath:oldPartialFilePath
toPath:newPartialFilePath
error:&error];
if (fail) {
if (oldCompleteFilePath) {
// complete images take precedence over partial
fail = NO;
error = nil;
[[NSFileManager defaultManager] removeItemAtPath:oldPartialFilePath error:NULL];
}
}
}
if (!fail) {
TIPImageDiskCacheEntry *newEntry = [oldEntry copy];
newEntry.identifier = newIdentifier;
[manifest removeEntry:oldEntry];
[manifest addEntry:newEntry];
[self _diskCache_updateByteCountsAdded:newEntry.completeFileSize + newEntry.partialFileSize
removed:0];
_globalConfig.internalTotalCountForAllDiskCaches += 1;
}
TIPAssert(fail ^ !error);
if (errorOut) {
*errorOut = error;
}
return !fail;
}
@end
@implementation TIPImageDiskCache (PrivateExposed)
- (TIPLRUCache *)diskCache_syncAccessManifest
{
if (!_diskCache_flags.manifestIsLoading) {
// quick - unsynchronized...
// ...safe since _diskCache_flags.manifestIsLoading is sync'd on diskCache queue
return _manifest;
}
// slow - synchronized
return [self manifest];
}
- (void)diskCache_updateImageEntry:(TIPImageCacheEntry *)entry forciblyReplaceExisting:(BOOL)force
{
@autoreleasepool {
[self _diskCache_updateImageEntry:entry
forciblyReplaceExisting:force
safeIdentifier:TIPSafeFromRaw(entry.identifier)];
}
}
- (nullable NSString *)diskCache_imageEntryFilePathForIdentifier:(NSString *)identifier
hitShouldMoveEntryToHead:(BOOL)hitToHead
context:(out TIPImageCacheEntryContext * __autoreleasing __nullable * __nullable)contextOut
{
TIPCompleteImageEntryContext *context = nil;
NSFileManager *fm = [NSFileManager defaultManager];
NSString *safeIdentifer = TIPSafeFromRaw(identifier);
NSString *filePath = [self filePathForSafeIdentifier:safeIdentifer];
if (_diskCache_flags.manifestIsLoading) {
if ([fm fileExistsAtPath:filePath]) {
const NSUInteger size = TIPFileSizeAtPath(filePath, NULL);
if (size) {
NSDictionary *xattributes = TIPGetXAttributesForFile(filePath, _XAttributesKeysToKindsMapForCompleteEntry());
context = (id)_ContextFromXAttributes(xattributes, NO);
if (![context isKindOfClass:[TIPCompleteImageEntryContext class]]) {
context = nil;
}
}
}
} else {
TIPImageCacheEntry *entry = (TIPImageCacheEntry *)[_manifest entryWithIdentifier:safeIdentifer
canMutate:hitToHead];
context = [entry.completeImageContext copy];
}
if (!context) {
filePath = nil;
}
if (contextOut) {
*contextOut = context;
}
return filePath;
}
- (nullable TIPImageDiskCacheEntry *)diskCache_imageEntryForIdentifier:(NSString *)identifier
options:(TIPImageDiskCacheFetchOptions)options
targetDimensions:(CGSize)targetDimensions
targetContentMode:(UIViewContentMode)targetContentMode
decoderConfigMap:(nullable NSDictionary<NSString *, id> *)decoderConfigMap
{
return [self _diskCache_getImageEntry:identifier
options:options
targetDimensions:targetDimensions
targetContentMode:targetContentMode
decoderConfigMap:decoderConfigMap];
}
@end
@implementation TIPImageDiskCache (Manifest)
- (void)_manifest_populateManifestWithCachePath:(NSString *)cachePath
{
const uint64_t machStart = mach_absolute_time();
[self _manifest_populateEntriesWithCachePath:cachePath
completion:^(unsigned long long totalSize,
NSArray<TIPImageDiskCacheEntry *> *entries,
NSArray<NSString *> *falseEntryPaths) {
// remove files on background queue BEFORE updating the manifest
// to avoid race condition with earily read path
NSFileManager *fm = [NSFileManager defaultManager];
tip_dispatch_async_autoreleasing(self->_globalConfig.queueForDiskCaches, ^{
for (NSString *falseEntryPath in falseEntryPaths) {
[fm removeItemAtPath:falseEntryPath error:NULL];
}
});
[self _manifest_finalizePopulateManifest:entries
totalSize:totalSize];
const uint64_t machEnd = mach_absolute_time();
TIPLogInformation(@"%@('%@') took %.3fs to populate its manifest", NSStringFromClass([self class]), self.cachePath.lastPathComponent, TIPComputeDuration(machStart, machEnd));
[self prune]; // goes to the background queue
}];
}
- (void)_manifest_populateEntriesWithCachePath:(NSString *)cachePath
completion:(TIPImageDiskCacheManifestPopulateEntriesCompletionBlock)completionBlock
{
tip_dispatch_async_autoreleasing(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
NSError *error;
NSArray<NSURL *> *entryURLs = cachePath ? TIPContentsAtPath(cachePath, &error) : nil;
if (!entryURLs) {
TIPLogError(@"%@ could not load its cache entries from path '%@'. %@", NSStringFromClass([self class]), cachePath, error);
tip_dispatch_async_autoreleasing(self->_manifestQueue, ^{
completionBlock(0, nil, nil);
});
return;
}
__block unsigned long long totalSize = 0;
NSDate *now = [NSDate date];
NSMutableArray<TIPImageDiskCacheEntry *> *entries = [[NSMutableArray alloc] init];
NSMutableArray<NSString *> *falseEntryPaths = [[NSMutableArray alloc] init];
NSMutableDictionary<NSString *, TIPImageDiskCacheEntry *> *manifest = [[NSMutableDictionary alloc] initWithCapacity:entryURLs.count];
NSOperationQueue *manifestCacheQueue = _ImageDiskCacheManifestCacheQueue();
NSOperationQueue *manifestIOQueue = _ImageDiskCacheManifestIOQueue();
dispatch_queue_t manifestAccessQueue = self->_manifestQueue;
__block TIPImageDiskCacheManifestPopulateEntriesCompletionBlock clearableCompletionBlock = [completionBlock copy];
NSOperation *finalIOOperation = [NSBlockOperation blockOperationWithBlock:^{
// nothing, just an operation for dependency ordering
}];
NSOperation *finalCacheOperation = [NSBlockOperation blockOperationWithBlock:^{
// sort entries
_SortEntries(entries);
// assert that we don't dupe entries
if (gTwitterImagePipelineAssertEnabled) {
NSSet *entrySet = [NSSet setWithArray:entries];
TIPAssertMessage(entrySet.count == entries.count, @"Manifest load yielded the same entry (or entries) to be counted more than once!!!");
}
tip_dispatch_async_autoreleasing(manifestAccessQueue, ^{
// Call the completion block
clearableCompletionBlock(totalSize, entries, falseEntryPaths);
// MUST clear the block since not clearing it can lead to a retain cycle
clearableCompletionBlock = nil;
});
}];
[finalCacheOperation addDependency:finalIOOperation];
for (NSURL *entryURL in entryURLs) {
// putting the construction of the operation to load a manifest entry
// in a function to avoid risking capturing self which can lead to a
// retain cycle.
NSOperation *ioOp = _ImageDiskCacheManifestLoadOperation(manifest,
falseEntryPaths,
entries,
&totalSize,
entryURL,
cachePath,
now,
manifestCacheQueue,
finalCacheOperation);
[finalIOOperation addDependency:ioOp];
[manifestIOQueue addOperation:ioOp];
}
[manifestIOQueue addOperation:finalIOOperation];
[manifestCacheQueue addOperation:finalCacheOperation];
});
}
- (void)_manifest_finalizePopulateManifest:(NSArray<TIPImageDiskCacheEntry *> *)entries
totalSize:(unsigned long long)totalSize
{
const BOOL didLoadEntries = entries != nil;
const SInt16 count = (didLoadEntries) ? (SInt16)entries.count : 0;
_manifest = (didLoadEntries) ? [[TIPLRUCache alloc] initWithEntries:entries delegate:self] : nil;
pthread_mutex_unlock(&_manifestMutex);
tip_dispatch_async_autoreleasing(_globalConfig.queueForDiskCaches, ^{
self->_diskCache_flags.manifestIsLoading = NO;
if (didLoadEntries) {
const UInt64 removeSize = self->_earlyRemovedBytesSize;
self->_earlyRemovedBytesSize = 0;
self->_globalConfig.internalTotalCountForAllDiskCaches += count;
[self _diskCache_updateByteCountsAdded:totalSize
removed:removeSize];
}
});
}
@end
static NSDictionary * __nullable _XAttributesFromContext(TIPImageCacheEntryContext * __nullable context)
{
if (!context || !context.URL) {
return nil;
}
if (!context.lastAccess) {
context.lastAccess = [NSDate date];
}
NSMutableDictionary *d = [[NSMutableDictionary alloc] initWithCapacity:_XAttributesKeysToKindsMap().count];
TIPAssert(context.TTL > 0.0);
// Alwasy set ALL values
d[kXAttributeContextURLKey] = context.URL;
d[kXAttributeContextLastAccessKey] = context.lastAccess;
d[kXAttributeContextTTLKey] = @(context.TTL);
d[kXAttributeContextUpdateTLLOnAccessKey] = @(context.updateExpiryOnAccess);
d[kXAttributeContextDimensionXKey] = @(context.dimensions.width);
d[kXAttributeContextDimensionYKey] = @(context.dimensions.height);
d[kXAttributeContextAnimated] = @(context.isAnimated);
if ([context isKindOfClass:[TIPPartialImageEntryContext class]]) {
TIPPartialImageEntryContext *partialContext = (id)context;
d[kXAttributeContextLastModifiedKey] = partialContext.lastModified ?: @"!";
d[kXAttributeContextExpectedSizeKey] = @(partialContext.expectedContentLength);
} else {
d[kXAttributeContextLastModifiedKey] = @"!";
d[kXAttributeContextExpectedSizeKey] = @0;
}
if (context.treatAsPlaceholder) {
d[kXAttributeContextTreatAsPlaceholderKey] = @YES;
}
return d;
}
static TIPImageCacheEntryContext * __nullable _ContextFromXAttributes(NSDictionary *xattrs, BOOL notYetComplete)
{
id val;
TIPImageCacheEntryContext *context = nil;
if (!notYetComplete) {
context = [[TIPCompleteImageEntryContext alloc] init];
} else {
context = [[TIPPartialImageEntryContext alloc] init];
TIPPartialImageEntryContext *partialContext = (id)context;
val = xattrs[kXAttributeContextLastModifiedKey];
partialContext.lastModified = [(NSString *)val length] < 4 ? nil : val;
val = xattrs[kXAttributeContextExpectedSizeKey];
partialContext.expectedContentLength = [(NSNumber *)val unsignedIntegerValue];
}
val = xattrs[kXAttributeContextURLKey];
if (!val) {
return nil;
}
context.URL = val;
val = xattrs[kXAttributeContextLastAccessKey];
if (!val) {
return nil;
}
context.lastAccess = val;
val = xattrs[kXAttributeContextTTLKey];
if (!val) {
return nil;
}
context.TTL = [(NSNumber *)val doubleValue];
val = xattrs[kXAttributeContextAnimated];
if (!val) {
// Don't fail on missing "animated" property
val = @NO;
}
context.animated = [(NSNumber *)val boolValue];
CGSize dimensions = CGSizeZero;
dimensions.width = (CGFloat)[xattrs[kXAttributeContextDimensionXKey] doubleValue];
dimensions.height = (CGFloat)[xattrs[kXAttributeContextDimensionYKey] doubleValue];
if (dimensions.width < 1.0 || dimensions.height < 1.0) {
return nil;
}
context.dimensions = dimensions;
val = xattrs[kXAttributeContextUpdateTLLOnAccessKey];
context.updateExpiryOnAccess = [val boolValue];
val = xattrs[kXAttributeContextTreatAsPlaceholderKey];
context.treatAsPlaceholder = [val boolValue];
return context;
}
static NSOperation *
_ImageDiskCacheManifestLoadOperation(NSMutableDictionary<NSString *, TIPImageDiskCacheEntry *> *manifest,
NSMutableArray<NSString *> *falseEntryPaths,
NSMutableArray<TIPImageDiskCacheEntry *> *entries,
unsigned long long *totalSizeInOut,
NSURL *entryURL,
NSString *cachePath,
NSDate *timestamp,
NSOperationQueue *manifestCacheQueue,
NSOperation *finalCacheOperation)
{
__weak typeof(finalCacheOperation) weakFinalCacheOperation = finalCacheOperation;
return [NSBlockOperation blockOperationWithBlock:^{
NSString *path = entryURL.lastPathComponent;
TIPImageCacheEntryContext *context = nil;
NSString *rawIdentifier = nil;
NSError *error = nil;
const BOOL isTmp = [[path pathExtension] isEqualToString:kPartialImageExtension];
NSString * const safeIdentifier = isTmp ? [path stringByDeletingPathExtension] : path;
NSString *entryPath = [entryURL path];
const NSUInteger size = [[entryURL resourceValuesForKeys:@[NSURLFileSizeKey] error:&error][NSURLFileSizeKey] unsignedIntegerValue];
if (!size) {
TIPLogError(@"Could not get filesize of '%@': %@", entryURL, error);
} else {
rawIdentifier = TIPRawFromSafe(safeIdentifier);
NSDictionary *xattrMap = isTmp ? _XAttributesKeysToKindsMap() : _XAttributesKeysToKindsMapForCompleteEntry();
context = (rawIdentifier) ? _ContextFromXAttributes(TIPGetXAttributesForFile(entryPath, xattrMap), isTmp) : nil;
if (isTmp && ![context isKindOfClass:[TIPPartialImageEntryContext class]]) {
context = nil;
} else if (!isTmp && [context isKindOfClass:[TIPPartialImageEntryContext class]]) {
context = nil;
}
}
NSBlockOperation *cacheOp = [NSBlockOperation blockOperationWithBlock:^{
if (!context || ([timestamp timeIntervalSinceDate:context.lastAccess] > context.TTL)) {
[falseEntryPaths addObject:entryPath];
return;
}
BOOL manifestCacheHit = NO;
TIPImageDiskCacheEntry *entry = nil;
entry = manifest[safeIdentifier];
if (!entry) {
entry = [[TIPImageDiskCacheEntry alloc] init];
entry.identifier = rawIdentifier;
manifest[safeIdentifier] = entry;
[entries addObject:entry];
} else {
manifestCacheHit = YES;
}
if (manifestCacheHit) {
TIPAssertMessage([entry.identifier isEqualToString:rawIdentifier], @"\n\tentry.identifier = %@\n\trawIdentifier = %@", entry.identifier, rawIdentifier);
}
if (isTmp) {
TIPAssertMessage(!entry.partialImageContext, @"\n\tentry.identifier = %@\n\trawIdentifier = %@", entry.identifier, rawIdentifier);
entry.partialImageContext = (id)context;
entry.partialFileSize = size;
} else {
TIPAssertMessage(!entry.completeImageContext, @"\n\tentry.identifier = %@\n\trawIdentifier = %@", entry.identifier, rawIdentifier);
entry.completeImageContext = (id)context;
entry.completeFileSize = size;
}
*totalSizeInOut = *totalSizeInOut + size;
if (entry.partialImageContext && entry.completeImageContext) {
const CGSize partialDimensions = entry.partialImageContext.dimensions;
const CGSize completeDimensions = entry.completeImageContext.dimensions;
if ((partialDimensions.width * partialDimensions.height) <= (completeDimensions.width * completeDimensions.height)) {
// We have a partial image that is lower fidelity than a completed image...
// remove the partial image from our disk cache
NSString * const partialEntryPath = [entryPath stringByAppendingPathExtension:kPartialImageExtension];
*totalSizeInOut = *totalSizeInOut - entry.partialFileSize;
entry.partialFileSize = 0;
entry.partialImageContext = nil;
TIPLogWarning(@"Partial image in disk cache is lower fidelity than complete image counterpart, removing: %@", partialEntryPath);
[falseEntryPaths addObject:partialEntryPath];
}
}
}];
[weakFinalCacheOperation addDependency:cacheOp];
[manifestCacheQueue addOperation:cacheOp];
}];
}
static BOOL _UpdateImageConditionCheck(const BOOL force,
const BOOL oldWasPlaceholder,
const BOOL newIsPlaceholder,
const BOOL extraCondition,
const CGSize newDimensions,
const CGSize oldDimensions,
NSURL * __nullable oldURL,
NSURL * __nullable newURL)
{
if (force) {
// forced
return YES;
}
if (oldWasPlaceholder && !newIsPlaceholder) {
// are we replacing a placeholder w/ a non-placeholder?
return YES;
}
if (extraCondition) {
// extra condition
return YES;
}
if (oldWasPlaceholder != newIsPlaceholder) {
// placeholderness missmatch
return NO;
}
// IMPORTANT: We use "last in wins" logic.
// It is easier for clients to detect larger varients matching smaller varients
// than smaller variants matching larger variants.
// This way, clients can load the smaller variant first, load the larger variant second and
// (next time they access smaller or larger variant) the larger variant is cached.
if ((newDimensions.width * newDimensions.height) >= (oldDimensions.width * oldDimensions.height)) {
// we're replacing based on size, is the image identical?
// Be sure we aren't replacing the identical image (by URL)
const BOOL isIdenticalImage = CGSizeEqualToSize(oldDimensions, newDimensions) && [oldURL isEqual:newURL];
if (!isIdenticalImage) {
return YES;
}
}
return NO;
}
static NSOperationQueue *_ImageDiskCacheManifestCacheQueue()
{
static NSOperationQueue *sQueue;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
sQueue = [[NSOperationQueue alloc] init];
sQueue.name = @"com.twitter.tip.disk.manifest.cache.queue";
sQueue.maxConcurrentOperationCount = 1;
sQueue.qualityOfService = NSQualityOfServiceUtility;
});
return sQueue;
}
static NSOperationQueue *_ImageDiskCacheManifestIOQueue()
{
static NSOperationQueue *sQueue;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
sQueue = [[NSOperationQueue alloc] init];
sQueue.name = @"com.twitter.tip.disk.manifest.io.queue";
sQueue.maxConcurrentOperationCount = 4; // parallelized
sQueue.qualityOfService = NSQualityOfServiceUtility;
});
return sQueue;
}
static dispatch_queue_t _ImageDiskCacheManifestAccessQueue()
{
static dispatch_queue_t sQueue;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
sQueue = dispatch_queue_create("com.twitter.tip.disk.manifest.access.queue", DISPATCH_QUEUE_SERIAL);
});
return sQueue;
}
static void _SortEntries(NSMutableArray<TIPImageDiskCacheEntry *> *entries)
{
[entries sortUsingComparator:^NSComparisonResult(TIPImageDiskCacheEntry *entry1, TIPImageDiskCacheEntry *entry2) {
NSDate *lastAccess1 = entry1.mostRecentAccess;
NSDate *lastAccess2 = entry2.mostRecentAccess;
// Simple check if both are nil (or identical)
if (lastAccess1 == lastAccess2) {
return NSOrderedSame;
}
// Put the missing access at the end
if (!lastAccess1) {
return NSOrderedDescending;
} else if (!lastAccess2) {
return NSOrderedAscending;
}
// Full compare
return [lastAccess2 compare:lastAccess1];
}];
}
NS_ASSUME_NONNULL_END