imagedatadupes

imagedatadupes (29885B)

---

 #!/usr/bin/env perl
 
 # License: See the bottom of this file.
 
 # NOTE: There might be various inaccuracies in the image decompression, but that's probably not an
 # issue for any realistic scenario that this program is used for:
 # https://stackoverflow.com/questions/25119285/weird-results-using-php-imagick-getimagesignature-method
 
 use strict;
 use warnings;
 use Image::ExifTool qw(ImageInfo);
 use Image::Magick;
 use File::MimeInfo::Magic;
 use Data::Compare;
 use File::Spec::Functions qw(catfile);
 use Pod::Usage;
 use Getopt::Long;
 use Cwd qw(realpath);
 
 # ignored in comparison
 # NOTE: If more common tags are added after a 1.0 release is tagged, they
 # should go in a different array that can be enabled with a separate option
 # to avoid breaking old commands that might have depended on the old list
 # of ignored tags.
 my @exif_ignore = (
 	"FileInodeChangeDate",
 	"Directory",
 	"FileModifyDate",
 	"FileAccessDate",
 	"FileName",
 	"FileSize",
 	"FilePermissions",
 );
 my $verbose = 0;
 my $skip_mime_check = 0;
 
 sub load_exif_data {
 	my $filename = shift;
 	my $info = ImageInfo($filename);
 	# Not too important since ImageMagick already should weed out unsupported filetypes
 	# ExifTool seems to print at least *something* (e.g. permissions, etc.) even for files
 	# that don't have any embedded tags (including directories), so there probably aren't
 	# any files that ImageMagick supports but ExifTool doesn't.
 	if (exists $info->{"Error"}) {
 		warn "WARNING: Ignoring $filename as ExifTool returned an error: $info->{Error}\n";
 		return undef;
 	} elsif (exists $info->{"Warning"}) {
 		warn "WARNING: ExifTool returned a warning for $filename: $info->{Warning}\n";
 	}
 
 	for my $ignore (@exif_ignore) {
 		delete $info->{$ignore};
 	}
 	# Remove all undefined/empty values since there are sometimes images
 	# that are identical, except that one contains just whitespace for
 	# a metadata value
 	for my $key (keys %$info) {
 		if (!defined $info->{$key} || $info->{$key} =~ /^\s*$/) {
 			delete $info->{$key};
 		}
 	}
 	return $info;
 }
 
 # -1: error
 #  0: doesn't matter which image is chosen
 #  1: image1 should be chosen (includes more metadata than image2)
 #  2: image2 should be chosen (includes more metadata than image1)
 sub compare_exif_data {
 	# filenames are just for debugging
 	my ($info1, $info2, $filename1, $filename2) = @_;
 	# Make a new hash containing just the keys so we can delete keys even if
 	# the original infos should not be destroyed (makes the logic a bit simpler).
 	sub copy_hash_keys {
 		my %new_hash;
 		my $hash = shift;
 		for my $key (keys %$hash) {
 			$new_hash{$key} = 1;
 		}
 		return \%new_hash;
 	}
 	my $info1_keys = copy_hash_keys($info1);
 	my $info2_keys = copy_hash_keys($info2);
 
 	my $info1_superset = 0;
 	for my $key (keys %$info1_keys) {
 		# Need Data::Compare because there may be references, etc.
 		# (e.g. ThumbnailImage is a reference)
 		if (exists $info2_keys->{$key} && Compare($info1->{$key}, $info2->{$key}) == 1) {
 			# NOP
 		} elsif (!exists $info2_keys->{$key}) {
 			if ($key eq "Warning") {
 				warn "WARNING: ExifTool gave a warning on $filename1 but not on $filename2\n";
 				warn "Treating these files as different to be on the safe side.\n";
 				return -1;
 			}
 			# $info1 contains a key that is empty or nonexistent in $info2
 			$info1_superset = 1;
 			warn "$filename1 includes $key which is empty/nonexistent in $filename2\n" if ($verbose);
 		} else {
 			warn "$key is \"$info1->{$key}\" for $filename1 but \"$info2->{$key}\" for $filename2\n" if ($verbose);
 			return -1;
 		}
 		delete $info1_keys->{$key};
 		delete $info2_keys->{$key};
 	}
 
 	if ($info1_superset && %$info2_keys) {
 		if ($verbose) {
 			for my $key (keys %$info2_keys) {
 				warn "$filename2 includes $key which is nonexistent in $filename1\n";
 			}
 			warn "$filename1 and $filename2 both contain EXIF data not included in the other file\n";
 		}
 		return -1;
 	} elsif ($info1_superset) {
 		return 1;
 	} elsif (!%$info2_keys) {
 		return 0;
 	} else {
 		if (exists $info2_keys->{"Warning"}) {
 			warn "WARNING: ExifTool gave a warning on $filename2 but not on $filename1\n";
 			warn "Treating these files as different to be on the safe side.\n";
 			return -1;
 		}
 		if ($verbose) {
 			for my $key (keys %$info2_keys) {
 				warn "$filename2 includes $key which is nonexistent in $filename1\n";
 			}
 		}
 		return 2;
 	}
 }
 
 sub build_signature_list {
 	my @queue = @_;
 	my @signatures;
 	while (@queue) {
 		my $filename = pop @queue;
 		# Just ignore all symlinks. This is the easiest way to
 		# prevent issues, and there shouldn't be any symlinks in
 		# the directories I use this on anyways.
 		if (-l $filename) {
 			warn "WARNING: Ignoring symlink $filename\n";
 			next;
 		}
 		if (-d $filename) {
 			my $dh;
 			if (!opendir $dh, $filename) {
 				warn "WARNING: Unable to open directory $filename\n";
 				next;
 			}
 			# FIXME: Maybe test for errors on readdir:
 			# https://github.com/Perl/perl5/issues/17907
 			my @new_files = map catfile($filename, $_), grep {$_ ne "." && $_ ne ".."} readdir $dh;
 			closedir $dh;
 			push @queue, @new_files;
 		} elsif (-f $filename) {
 			if (!-r $filename) {
 				warn "WARNING: File $filename is not readable\n";
 				next;
 			}
 			if (!$skip_mime_check) {
 				my $mime = mimetype($filename);
 				if (!defined($mime)) {
 					warn "Unable to determine MIME type for $filename\n" if ($verbose);
 					next;
 				} elsif ($mime !~ /^image/) {
 					warn "Unsupported MIME type for $filename: $mime\n" if ($verbose);
 					next;
 				}
 			}
 			my $image = Image::Magick->new;
 			my $err = $image->Read($filename);
 			if ($err) {
 				$err =~ /(\d+)/;
 				# 420 is "no decode delegate for this image format", we don't want
 				# to spam that for every file that isn't an image format
 				if ($1 != 420 || $verbose) {
 					warn "WARNING: Unable to open file $filename with ImageMagick: $err\n";
 				}
 				next;
 			}
 			my $sig = $image->Get("signature");
 			if (!defined($sig)) {
 				warn "WARNING: Unable to obtain signature for $filename\n";
 				next;
 			}
 			# This is needed so that it's possible to avoid the same file ending up
 			# in the same deletion group (if the same file/directory was added
 			# to the command-line arguments twice - not very realistic, but let's
 			# just be on the safe side).
 			my $fullpath = realpath($filename);
 			if (!defined($fullpath)) {
 				warn "WARNING: Unable to get absolute path for $filename\n";
 				next;
 			}
 			push @signatures, [$fullpath, $sig];
 		} else {
 			warn "WARNING: Ignoring non-existent or non-regular file $filename\n";
 		}
 	}
 	return @signatures;
 }
 
 sub handle_group_exif {
 	# At this point, there really shouldn't be any errors reading the EXIF data since
 	# ImageMagick already successfully read the images (unless there's a file format
 	# for which ImageMagick has a loader but ExifTool doesn't).
 	my @exif_arr = grep defined($_->[3]), map [@$_, load_exif_data($_->[1])], @_;
 	my @all_groups;
 	while (@exif_arr) {
 		my $first = $exif_arr[0];
 		# Strip signature and exif data for elements in final array since we don't need them anymore
 		push @all_groups, [[$first->[0], $first->[1]]];
 		my @tmp_arr;
 		for my $i (1..$#exif_arr) {
 			my $ret = compare_exif_data($first->[3], $exif_arr[$i]->[3], $first->[1], $exif_arr[$i]->[1]);
 			if ($ret == 0 || $ret == 1) {
 				push @{$all_groups[-1]}, [$exif_arr[$i]->[0], $exif_arr[$i]->[1]];
 			} elsif ($ret == 2) {
 				unshift @{$all_groups[-1]}, [$exif_arr[$i]->[0], $exif_arr[$i]->[1]];
 				$first = $exif_arr[$i];
 			} else {
 				push @tmp_arr, $exif_arr[$i];
 			}
 		}
 		@exif_arr = @tmp_arr;
 	}
 	# Filter out single-element groups
 	return grep {$#$_ > 0} @all_groups;
 }
 
 sub handle_group_image_data {
 	my @cur_imgs = @_;
 	my @all_groups;
 	while (@cur_imgs) {
 		my $first = shift @cur_imgs;
 		my $image1 = Image::Magick->new;
 		my $err = $image1->Read($first->[1]);
 		# There really shouldn't be any errors here since all images have been
 		# previously read by ImageMagick to calculate the signatures, but let's
 		# just be on the safe side.
 		# Error 420 isn't ignored here because it doesn't matter if it's shown
 		# in the extremely rare case that it occurs here even though all the
 		# files were already opened previously.
 		if ($err) {
 			warn "WARNING: ImageMagick error while reading $first->[1]: $err\n";
 			next;
 		}
 		push @all_groups, [$first];
 		my @tmp_arr;
 		for my $i (0..$#cur_imgs) {
 			my $image2 = Image::Magick->new;
 			$err = $image2->Read($cur_imgs[$i]->[1]);
 			if ($err) {
 				warn "WARNING: ImageMagick error while reading $cur_imgs[$i]->[1]: $err\n";
 				next;
 			}
 			my $diff = $image1->Compare(image=>$image2, metric=>"ae");
 			# NOTE: If the two images have different sizes, ImageMagick doesn't seem to throw
 			# an error, but the returned error should be non-zero.
 			my $img_error = $diff->Get("error");
 			if (!defined($img_error)) {
 				warn "WARNING: Unknown error during comparison of $first->[1] and $cur_imgs[$i]->[1]\n";
 				$img_error = 1;
 			}
 			if ($img_error == 0) {
 				push @{$all_groups[-1]}, $cur_imgs[$i];
 			} else {
 				push @tmp_arr, $cur_imgs[$i];
 			}
 		}
 		@cur_imgs = @tmp_arr;
 	}
 	# Filter out single-element groups
 	return grep {$#$_ > 0} @all_groups;
 }
 
 my @extra_exif_ignore;
 my $no_default_exif_ignore = 0;
 my $show_help = 0;
 my $isolate = 0;
 my $print_ignored = 0;
 my $skip_exact_equality = 0;
 my $delete_files = 0;
 my $no_prompt = 0;
 my $number_output = 0;
 my $default_choice = "";
 
 Getopt::Long::Configure("bundling");
 GetOptions (
 	"exif-ignore=s" => \@extra_exif_ignore,
 	"no-default-exif-ignore" => \$no_default_exif_ignore,
 	"h|help" => \$show_help,
 	"v|verbose" => \$verbose,
 	"I|isolate" => \$isolate,
 	"print-exif-ignore" => \$print_ignored,
 	"Q|quick" => \$skip_exact_equality,
 	"d|delete" => \$delete_files,
 	"N|no-prompt" => \$no_prompt,
 	"n|number" => \$number_output,
 	"default-choice=s" => \$default_choice,
 	"skip-mime-check" => \$skip_mime_check,
 ) || pod2usage(-exitval => 1, -verbose => 1);
 @extra_exif_ignore = split(/,/, join(',', @extra_exif_ignore));
 if ($no_default_exif_ignore) {
 	@exif_ignore = @extra_exif_ignore;
 } else {
 	push @exif_ignore, @extra_exif_ignore;
 }
 
 if ($print_ignored) {
 	print join(",", @exif_ignore) . "\n";
 	exit 0;
 }
 
 pod2usage(-exitval => 0, -verbose => 2) if $show_help;
 pod2usage(-exitval => 1, -verbose => 1) if @ARGV < 1;
 if ($default_choice ne "" && $default_choice !~ /^[yYnNp]$/) {
 	warn "ERROR: --default-choice must be y, Y, n, N, or p!\n";
 	pod2usage(-exitval => 1, -verbose => 1);
 }
 
 my @siglist;
 for my $i (0..$#ARGV) {
 	# Add index of command-line argument for precedence when outputting which
 	# files should be deleted
 	# Format is now [original cmd argument index, full path, signature]
 	push @siglist, map [$i, $_->[0], $_->[1]], build_signature_list($ARGV[$i]);
 }
 
 # Sort first by signature, then by original command-line argument index, then by file path
 # Last sort by file path is so that if, for instance, there are two directories within
 # the same cmd arg (and isolate isn't enabled), duplicates are always removed from the
 # same directory.
 @siglist = sort {
 	$a->[2] cmp $b->[2] || $a->[0] <=> $b->[0] || $a->[1] cmp $b->[1];
 } @siglist;
 
 my @group;
 my @final_groups;
 my $prev_sig = "";
 for my $file (@siglist) {
 	if ($file->[2] ne $prev_sig) {
 		if (scalar(@group) > 1) {
 			if ($skip_exact_equality) {
 				# Need to make a copy of @group when storing it in @final_groups
 				push @final_groups, [@group];
 			} else {
 				push @final_groups, handle_group_image_data(@group);
 			}
 		}
 		@group = ();
 		$prev_sig = $file->[2];
 	}
 	push @group, $file;
 }
 if (scalar(@group) > 1) {
 	if ($skip_exact_equality) {
 		push @final_groups, [@group];
 	} else {
 		push @final_groups, handle_group_image_data(@group);
 	}
 }
 
 @final_groups = map {handle_group_exif(@$_)} @final_groups;
 # Sort by length of group, then by filename of first file in group
 @final_groups = sort {$#$a <=> $#$b || $a->[0]->[1] cmp $b->[0]->[1]} @final_groups;
 
 # 0: Do not delete files
 # 1: Delete files
 # 2: Go to previous duplicate group
 sub prompt_delete {
 	if ($no_prompt) {
 		return $delete_files;
 	}
 	my $choice = "";
 	my $default_printstr = $default_choice eq "" ? "" : "[$default_choice]";
 	while ($choice !~ /^[yYnNp]$/) {
 		print STDERR "Delete files? (y/Y/n/N/p)$default_printstr ";
 		$choice = <STDIN>;
 		chomp $choice;
 		if ($choice eq "") {
 			$choice = $default_choice;
 		}
 	}
 	if ($choice eq "y") {
 		return 1;
 	} elsif ($choice eq "n") {
 		return 0;
 	} elsif ($choice eq "Y") {
 		$delete_files = 1; # should already be 1 anyways
 		$no_prompt = 1;
 		return 1;
 	} elsif ($choice eq "N") {
 		$delete_files = 0;
 		$no_prompt = 1;
 		return 0;
 	} else {
 		return 2;
 	}
 }
 
 # It doesn't matter that the same reference will be repeated here since the
 # array elements are replaced later anyways
 my @final_delete_files = ([]) x scalar(@final_groups);
 sub output_groups {
 	my $cur_group_id = shift;
 	while ($cur_group_id <= $#final_groups) {
 		# Reset so it is set correctly when going back to correct mistakes
 		$final_delete_files[$cur_group_id] = [];
 		my $group = $final_groups[$cur_group_id];
 		print "\n" if ($cur_group_id > 0);
 		if ($number_output) {
 			print "[" . ($cur_group_id + 1) . "/" . ($#final_groups + 1) . "]\n";
 		}
 		my $first = $group->[0];
 		print "[+] $first->[1]\n";
 		# Note: This may be a bit inefficient, but these hashes make it simple to avoid mistakes
 		# and give somewhat decent diagnostic messages.
 		my %keep_list;
 		my %delete_list;
 		$keep_list{$first->[1]} = 1;
 		for my $file (@$group[1..$#$group]) {
 			if ($isolate && $file->[0] == $first->[0]) {
 				print "[+] $file->[1]\n";
 				$keep_list{$file->[1]} = 1;
 			} else {
 				if (exists $keep_list{$file->[1]}) {
 					warn "WARNING: Ignoring $file->[1] as it was queued for deletion but refers " .
 					     "to the same file as a file in the list of files to keep\n";
 				} elsif (exists $delete_list{$file->[1]}) {
 					warn "WARNING: $file->[1] already queued for deletion\n";
 				} else {
 					print "[-] $file->[1]\n";
 					$delete_list{$file->[1]} = 1;
 				}
 			}
 		}
 		if ($delete_files && (my $choice = prompt_delete())) {
 			if ($choice == 1) {
 				$final_delete_files[$cur_group_id] = [keys %delete_list];
 			} else {
 				$cur_group_id--;
 				if ($cur_group_id <= 0) {
 					$cur_group_id = 0;
 					print "\n";
 				}
 				next;
 			}
 		}
 		$cur_group_id++;
 	}
 }
 
 output_groups(0);
 
 # Loop here so the user can still correct mistakes for the previous prompts when
 # we have already reached the final prompt.
 while (1) {
 	my $any_to_delete = 0;
 	for my $delete_group (@final_delete_files) {
 		if (@$delete_group) {
 			$any_to_delete = 1;
 			last;
 		}
 	}
 	if ($any_to_delete) {
 		print "\n" if !$no_prompt;
 		my $choice = $no_prompt ? "y" : "";
 		while ($choice !~ /^[ynp]$/) {
 			print "Delete all chosen files? (y/n/p) ";
 			$choice = <STDIN>;
 			chomp $choice;
 		}
 		if ($choice eq "y") {
 			for my $delete_group (@final_delete_files) {
 				for my $filename (@$delete_group) {
 					warn "Deleting $filename\n" if ($verbose);
 					if (!unlink($filename)) {
 						warn "WARNING: Unable to delete $filename\n";
 					}
 				}
 			}
 		} elsif ($choice eq "p") {
 			print "\n" if $#final_groups == 0;
 			# Go back to group output, starting at last group
 			output_groups($#final_groups);
 			next;
 		}
 	}
 	last;
 }
 
 __END__
 
 =head1 NAME
 
 imagedatadupes - find duplicate images
 
 =head1 SYNOPSIS
 
 B<imagedatadupes> [option ...] file/directory ...
 
    Options:
       -I, --isolate
       -d, --delete
       -N, --no-prompt
       -Q, --quick
       -n, --number
       -v, --verbose
       -h, --help
       --default-choice=CHOICE
       --exif-ignore=TAG,...
       --no-default-exif-ignore
       --print-exif-ignore
       --skip-mime-check
 
 =head1 DESCRIPTION
 
 B<imagedatadupes> compares image files to check for duplicates and optionally deletes them.
 Duplicates are found even if the files are not identical, as long as the image data is the same,
 and the metadata is similar enough based on certain criteria.
 
 The images are first compared using signatures of the image data calculated with ImageMagick(1).
 Then, images with matching signatures are compared for exact equality of the image data. Lastly,
 the metadata returned by ExifTool(1) is compared (this includes various metadata tags, not just
 EXIF data). If the metadata in one file is a superset of the metadata in another file, the
 former is preferred over the latter when deciding which files to keep. Note that a metadata value
 which is empty or only contains whitespace is treated as if it does not exist.
 
 The specific use-case for this is files where the EXIF data was modified to add a comment (e.g. where
 a photo was taken). If an old copy of the same image still exists, it won't be found as a duplicate
 by programs such as fdupes(1) and jdupes(1) since the EXIF data is different, but the new version
 only contains more data than the old one, so there's no point in keeping the old copy. In order for this
 to work in practice, some metadata tags that ExifTool returns, such as "FileModifyDate", need to be
 ignored (see B<--exif-ignore>, B<--no-default-exif-ignore>, and B<--print-exif-ignore>).
 
 The output format consists of groups of files separated by empty lines, with each file being preceded
 by [+] or [-], depending on whether it was automatically selected for keeping or deletion, respectively.
 If B<--delete> is not used, duplicates are only printed and no actual deletion is performed.
 
 If multiple files are identical (both image data and metadata), files that were included in earlier
 command-line parameters are preferred for keeping. If multiple files included in the same command-line
 parameter are identical and B<--isolate> is not used, the file whose absolute path comes first
 lexicographically is preferred for keeping.
 
 Before being output, the groups are sorted first based on the number of files in each group, then
 based on the lexicographical order of the absolute path of the first file in each group.
 This is mainly so that it is easier to spot outliers, such as a group where the file chosen
 for keeping (i.e. the first file in the group) is not in the expected directory.
 
 See B<CAVEATS> for some issues that can arise.
 
 B<WARNING:> Always make backups of your files before using this program to delete duplicates. There is
 a nonzero chance that bugs still exist which could cause the wrong files to be deleted. If you have
 directories containing files that should not be modified in any way, and you only want to check if
 there are duplicates of those files in other directories, you may want to change the permissions
 of those files so they are read-only and a malfunction of this program cannot cause any issues.
 At least that's what a paranoid person such as the author of this program would do.
 
 =head1 OPTIONS
 
 =over 8
 
 =item B<-I>, B<--isolate>
 
 Isolate the command-line parameters from each other. In other words, for each group of
 duplicates, all files belonging to the same command-line parameter as the file selected
 for keeping are also kept.
 
 =item B<-d>, B<--delete>
 
 Prompt user for deletion of duplicates. Note that the actual deletion is not performed until all
 duplicate groups have been processed and the user has been prompted once more to confirm that all
 the chosen files should really be deleted.
 The options are as follows:
 
 =over 4
 
 =item B<y>
 
 Delete the files that were automatically selected for deletion in the
 current duplicate group.
 
 =item B<Y>
 
 Delete the files that were automatically selected for deletion in the
 current duplicate group and in all subsequent duplicate groups.
 
 =item B<n>
 
 Do not delete the files that were automatically selected for deletion
 in the current duplicate group.
 
 =item B<N>
 
 Do not delete the files that were automatically selected for deletion
 in the current duplicate group and in all subsequent duplicate groups.
 
 =item B<p>
 
 Go back to the previous duplicate group. This is particularly useful
 together with B<--default-choice> so that choices can be made very
 quickly, but mistakes can still be corrected by going back to previous
 duplicate groups. Previously chosen options are not saved, so when
 B<p> is used to go back multiple groups, all choices have to be made
 again.
 
 =back
 
 Note that B<Y> and B<N> also cause the final prompt asking if all chosen
 files should be deleted to be skipped.
 
 =item B<-N>, B<--no-prompt>
 
 When used in conjunction with B<--delete>, automatically delete duplicates without prompting the user.
 This option should be used with care as it can lead to data loss. Make sure to always have backups
 of your data in case something goes wrong.
 
 =item B<-Q>, B<--quick>
 
 Only determine duplicates based on the data signature and metadata (skip the check for exact
 equality of the image data).
 
 =item B<-n>, B<--number>
 
 Print the number of the current group and the total number of groups for each duplicate group
 that is output. This is useful in interactive mode so there is an indication of how many
 groups are still left.
 
 =item B<-v>, B<--verbose>
 
 Print informational messages to stdout.
 This currently includes some file reading errors, as well as information on the metadata
 comparison (e.g. which tags were present in one file but not in the other one).
 Additionally, all filenames are printed once more right before they are actually deleted.
 
 =item B<-h>, B<--help>
 
 Print usage information, then exit.
 
 =item B<--default-choice>=CHOICE
 
 Set a default choice to use in interactive mode. This must be one of the characters that can be
 entered at a prompt during interactive mode. When this is set, it is possible to simply press
 enter at each prompt in order to use the default choice. Note that this is only used during
 interactive mode, it does not have any effect when B<--no-prompt> is in effect.
 
 =item B<--exif-ignore>=TAG,...
 
 Add additional metadata tags to the list of tags that should be ignored when comparing two files.
 This option can be given multiple times. Alternatively, multiple tags can be given at once
 as a comma-separated list. The B<--verbose> option might help to figure out why two images
 aren't being considered as duplicates. Additionally, ExifTool can be used to find the list
 of all tag names an image contains using the command "exiftool -s image.jpg". It is generally
 a good idea to start with the default ignore list and only add more tags later if needed.
 Adding too many tags to the ignore list can easily lead to data loss when files are considered
 to be duplicates even though the values for some ignored metadata tags were different.
 
 =item B<--no-default-exif-ignore>
 
 Do not use the default list of ignored metadata tags.
 
 WARNING: As this program has not reached version 1.0 yet, the default list of ignored metadata tags
 may still change without warning. Always use B<--exif-ignore> together with B<--no-default-exif-ignore>
 if you want to be certain that nothing unexpected happens.
 
 =item B<--print-exif-ignore>
 
 Print all metadata tags that will be ignored, including the default ones if they haven't been turned off
 using B<--no-default-exif-ignore>.
 
 =item B<--skip-mime-check>
 
 Do not check the MIME type of files. Instead, simply process anything that is supported by ImageMagick
 and ExifTool. By default, only MIME types starting with "image" are processed in order to avoid issues
 with ImageMagick trying to open large video files or similar.
 
 =back
 
 =head1 CAVEATS
 
 The ordering of the files within a duplicate group in the output can be a bit weird.
 In particular, if a file is chosen for keeping because it contains more metadata, it will be displayed
 at the beginning of the file list, regardless of the regular sorting. This can seem strange when
 there are multiple duplicates each in multiple command-line parameters since it can happen that the
 duplicates from one command-line parameter are not written right after each another if one of them was
 moved to the beginning of the output due to its metadata.
 
 Before printing the output, a final check is performed to make sure that none of the files in
 a duplicate group have the same path (this should only happen if the same directory/file was given
 multiple times). This can cause only one file to be displayed for a duplicate group.
 In a similar vein, when B<--isolate> is used, it is possible for a duplicate group to be printed
 even though none of the files are considered for deletion.
 
 If there are any errors while reading a file, the file is ignored during further processing.
 Errors that are only due to a filetype not being supported are normally not printed (except
 in rare circumstances) in order to avoid spamming the output with error messages for all
 non-image files. These are only printed when B<--verbose> is used.
 
 Originally, any warnings given by ExifTool also caused the file to be ignored. However, this
 caused issues because it meant that identical files which both included the same warning
 were not considered as duplicates. Thus, warnings are now treated almost the same as
 other metadata. The only difference is that warnings cannot be used to decide that one file
 contains a superset of another file's metadata. This is a safety feature as it is theoretically
 possible for two files to be identical, except that one is damaged in some way, leading to a
 warning. If warnings were treated exactly the same as other metadata, the file without the
 warning would be deleted as it contains less metadata, but this would probably not be the
 correct choice. In that case, a warning is printed and the files are not considered as
 duplicates so they can be examined manually later. If warnings should be disregarded
 entirely when comparing files, the "Warning" tag can be ignored using B<--exif-ignore>.
 
 If there is an unknown error during the exact comparison of the image data, this might be due to
 memory restrictions in the ImageMagick configuration. Try opening the ImageMagick configuration
 file "policy.xml" and changing the maximum memory. The configuration file is probably located
 at /etc/ImageMagick-6/policy.xml or /etc/ImageMagick-7/policy.xml, depending on the version of
 ImageMagick. There should be a line similar to
 '<policy domain="resource" name="memory" value="256MiB"/>' that can be changed to increase the
 maximum memory. There is also a similar line with "disk" instead of "memory" that can be used
 to control how much temporary disk space ImageMagick is allowed to use when there is not enough
 memory. The "map" and "area" options may also be of interest. See the ImageMagick documentation
 for a description of the other options in the policy.xml file. Note that, depending on how much
 memory is available, it might be a good idea to set the disk limit to 0 in order to fail more
 quickly and avoid a lot of disk writes when large files are opened.
 
 ImageMagick seems to sometimes open files without any error even though they don't contain any
 image data. For instance, it managed to open some JPEG files that had been corrupted and only
 contained zeroes, calculating the same signature for all of them. This should not be an issue
 normally since a MIME type check is now performed before trying to read any files with ImageMagick.
 Another interesting case found in the wild dealt with .exi files, which appeared to contain just
 the EXIF data from JPEG files. These passed the MIME type check, and ImageMagick calculated the
 same signature for all of them, but returned an error later on during the exact image data
 comparison. Such cases should generally not be a problem since the image data or metadata
 comparison should still fail, causing the files to be treated as different, but it's probably
 still a good idea to keep these edge cases in mind.
 
 The image data used for the comparisons is whatever ImageMagick decompresses the image
 files to, which might include various inaccuracies. However, that probably isn't an issue
 in any realistic scenario.
 
 For simplicity, all comparisons are performed even when B<--isolate> is used and it technically
 isn't necessary to compare files from the same command-line parameter.
 
 This program completely ignores all symbolic links in order to avoid all the issues that
 can be caused by them.
 
 =head1 EXAMPLES
 
 =over 8
 
 =item B<imagedatadupes -dnI --default-choice=y directory1 directory2 ...>
 
 Look for duplicates in the given directories, making sure that duplicate images belonging to the
 same command-line paramater are either all kept or all deleted. The user is prompted to confirm
 the deletion of each duplicate group, with the default choice being set to B<y> so that it is
 easy to always press enter (if a mistake is made, B<p> can be entered at the prompt to go back
 to the previous duplicate group). In order to give an indication of progress during the
 prompting for deletion, the output groups are numbered.
 
 =item B<imagedatadupes -dnI --default-choice=y --exif-ignore=ThumbnailOffset directory1 directory2 ...>
 
 Same as above, but additionally ignore the "ThumbnailOffset" tag when comparing image metadata.
 
 =back
 
 =head1 EXIT STATUS
 
 Always 0, unless the arguments given were invalid.
 
 =head1 SEE ALSO
 
 ExifTool(1), fdupes(1), findimagedupes(1p), ImageMagick(1), jdupes(1)
 
 =head1 LICENSE
 
 Copyright (c) 2026 lumidify <nobody@lumidify.org>
 
 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above
 copyright notice and this permission notice appear in all copies.
 
 THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.