in pipe-cli/src/utilities/datastorage_operations.py [0:0]
def _filter_items(cls, items, manager, source_wrapper, destination_wrapper, permission_to_check,
include, exclude, force, quiet, skip_existing, sync_newer, verify_destination,
unsafe_chars, unsafe_chars_replacement, empty_files):
logging.debug(u'Preprocessing paths...')
filtered_items = []
for item in items:
full_path = item[1]
relative_path = item[2]
source_size = item[3]
logging.debug(u'Preprocessing path {}...'.format(full_path))
item = cls._process_unsafe_chars(item, quiet, unsafe_chars, unsafe_chars_replacement)
item = cls._process_empty_files(item, quiet, empty_files)
if not item:
continue
if relative_path.endswith('/'):
continue
# check that we have corresponding permission for the file before take action
if source_wrapper.is_local() and not os.access(to_string(full_path), permission_to_check):
continue
if source_wrapper.is_file() and not source_wrapper.path == full_path:
continue
if not include and not exclude and not skip_existing and not verify_destination:
if not source_wrapper.is_file():
possible_folder_name = source_wrapper.path_with_trailing_separator()
# if operation from source root
if possible_folder_name == source_wrapper.path_separator:
filtered_items.append(item)
continue
if not full_path.startswith(possible_folder_name):
continue
if not PatternMatcher.match_any(relative_path, include):
if not quiet:
click.echo(u"Skipping file {} since it doesn't match any of include patterns [{}]."
.format(full_path, ",".join(include)))
continue
if PatternMatcher.match_any(relative_path, exclude, default=False):
if not quiet:
click.echo(u"Skipping file {} since it matches exclude patterns [{}]."
.format(full_path, ",".join(exclude)))
continue
if relative_path.endswith(FOLDER_MARKER):
filtered_items.append(item)
continue
if not skip_existing and (force or not verify_destination):
filtered_items.append(item)
continue
destination_key = manager.get_destination_key(destination_wrapper, relative_path)
if skip_existing and sync_newer:
destination_size, destination_modification_datetime = \
manager.get_destination_object_head(destination_wrapper, destination_key)
else:
destination_size = manager.get_destination_size(destination_wrapper, destination_key)
destination_modification_datetime = None
destination_is_empty = destination_size is None
if destination_is_empty:
filtered_items.append(item)
continue
if skip_existing:
source_key = manager.get_source_key(source_wrapper, full_path)
source_modification_datetime = item[4] if sync_newer and len(item) >= 5 else None
if sync_newer and source_modification_datetime is None and source_wrapper.is_local():
source_modification_datetime = StorageOperations.get_local_file_modification_datetime(source_key)
need_to_overwrite = not manager.skip_existing(source_key, source_size, source_modification_datetime,
destination_key, destination_size,
destination_modification_datetime, sync_newer, quiet)
if need_to_overwrite and not force:
cls._force_required()
if need_to_overwrite:
filtered_items.append(item)
continue
if not force:
cls._force_required()
filtered_items.append(item)
return filtered_items