commit a2653a9d8a70ee89088a617213a251bbff5d4ce5
Author: lumidify <nobody@lumidify.org>
Date: Thu, 9 Jul 2026 16:55:33 +0200
Add initial implementation
Diffstat:
| A | .gitignore | | | 1 | + |
| A | LICENSE | | | 13 | +++++++++++++ |
| A | Makefile | | | 37 | +++++++++++++++++++++++++++++++++++++ |
| A | README | | | 12 | ++++++++++++ |
| A | TODO | | | 42 | ++++++++++++++++++++++++++++++++++++++++++ |
| A | imagedatadupes | | | 781 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
6 files changed, 886 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+imagedatadupes.1
diff --git a/LICENSE b/LICENSE
@@ -0,0 +1,13 @@
+Copyright (c) 2026 lumidify <nobody@lumidify.org>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
diff --git a/Makefile b/Makefile
@@ -0,0 +1,37 @@
+.POSIX:
+
+NAME = imagedatadupes
+VERSION = 0.0
+
+PREFIX = /usr/local
+MANPREFIX = ${PREFIX}/man
+
+MAN1 = ${NAME:=.1}
+MISCFILES = Makefile README LICENSE TODO
+
+${MAN1}: ${NAME}
+ pod2man ${NAME} ${MAN1}
+
+install: ${MAN1}
+ mkdir -p "${DESTDIR}${PREFIX}/bin"
+ cp -f ${NAME} "${DESTDIR}${PREFIX}/bin"
+ chmod 755 "${DESTDIR}${PREFIX}/bin/${NAME}"
+ mkdir -p "${DESTDIR}${MANPREFIX}/man1"
+ cp -f ${MAN1} "${DESTDIR}${MANPREFIX}/man1"
+ chmod 644 "${DESTDIR}${MANPREFIX}/man1/${MAN1}"
+
+uninstall:
+ rm -f "${DESTDIR}${PREFIX}/bin/${NAME}"
+ rm -f "${DESTDIR}${MANPREFIX}/man1/${MAN1}"
+
+clean:
+ rm -f ${MAN1}
+
+dist:
+ rm -rf "${NAME}-${VERSION}"
+ mkdir -p "${NAME}-${VERSION}"
+ cp -rf ${NAME} ${MISCFILES} "${NAME}-${VERSION}"
+ tar cf - "${NAME}-${VERSION}" | gzip -c > "${NAME}-${VERSION}.tar.gz"
+ rm -rf "${NAME}-${VERSION}"
+
+.PHONY: all clean install uninstall dist
diff --git a/README b/README
@@ -0,0 +1,12 @@
+REQUIREMENTS: Perl 5 Image::ExifTool Image::Magick File::MimeInfo::Magic
+
+imagedatadupes compares image files to check for duplicates and optionally
+deletes them. Duplicates are found even if the files are not identical,
+as long as the image data is the same, and the metadata is similar enough
+based on certain criteria.
+
+See imagedatadupes -h for the full documentation.
+
+Alternatively, install it with 'make install' (as root; this just generates
+a man page and copies imagedatadupes and the generated man page to the
+appropriate system directories) and then run 'man imagedatadupes'.
diff --git a/TODO b/TODO
@@ -0,0 +1,42 @@
+# TODO
+
+* Maybe allow opening an external program with the duplicates as arguments.
+ Alternatively, create a proper output format that deals with filenames
+ including newlines, etc., so that a separate program can parse the output
+ properly and delete the files or perform other actions.
+* Add some sort of progress indication during signature calculation and
+ image data comparison. One idea would be to simply show the current
+ file being processed (all on the same line) during signature calculation
+ in order to give a general idea how far it has progressed. This would
+ also require the files to be sorted before adding them to the queue in
+ order for it to be useful. During the later stages, the number of the
+ current duplicate group can be printed, together with the total number
+ of groups (of course, the total number of groups can still become smaller
+ during processing).
+* Allow adding filters to exclude filenames.
+* Allow saving a backup of the metadata before deleting files. This would
+ also allow deleting files even if they contain metadata that is not
+ included in other files as long as the image data is duplicated somewhere.
+* Allow configuring verbosity properly to print only some informational
+ messages.
+* Add an option to compare metadata tags that are different between files,
+ for instance when both files contain a UserComment, but it is different
+ because it was edited at some point. Currently, that is considered to
+ be a mismatch, but it would be convenient to be able to easily choose
+ which file to keep instead of having to manually compare the tags using
+ ExifTool.
+* Add an option to ignore specific metadata values for a certain tag.
+ For instance, I have had files where one copy contained a UserComment
+ that was just a 4-character string, possibly added automatically by
+ some program. This caused them to not be considered as duplicates even
+ though other copies contained proper UserComments.
+* Maybe change the way ignored metadata tags are handled. Currently, they
+ are completely disregarded in the comparison, but it might be useful to
+ still consider them when deciding whether one file's metadata is a
+ superset of another file's metadata. For instance, if two images are
+ duplicates except that only one of them contains a UserComment tag,
+ and UserComment is being ignored, the current algorithm will not give
+ any preference which file should be kept and which should be deleted
+ since the UserComment tag is completely ignored. It might be better to
+ still take ignored tags into account unless doing so would cause the
+ files to not be considered as duplicates anymore.
diff --git a/imagedatadupes b/imagedatadupes
@@ -0,0 +1,781 @@
+#!/usr/bin/env perl
+
+# License: See the bottom of this file.
+
+# NOTE: There might be various inaccuracies in the image decompression, but that's probably not an
+# issue for any realistic scenario that this program is used for:
+# https://stackoverflow.com/questions/25119285/weird-results-using-php-imagick-getimagesignature-method
+
+use strict;
+use warnings;
+use Image::ExifTool qw(ImageInfo);
+use Image::Magick;
+use File::MimeInfo::Magic;
+use Data::Compare;
+use File::Spec::Functions qw(catfile);
+use Pod::Usage;
+use Getopt::Long;
+use Cwd qw(realpath);
+
+# ignored in comparison
+# NOTE: If more common tags are added after a 1.0 release is tagged, they
+# should go in a different array that can be enabled with a separate option
+# to avoid breaking old commands that might have depended on the old list
+# of ignored tags.
+my @exif_ignore = (
+ "FileInodeChangeDate",
+ "Directory",
+ "FileModifyDate",
+ "FileAccessDate",
+ "FileName",
+ "FileSize",
+ "FilePermissions",
+);
+my $verbose = 0;
+my $skip_mime_check = 0;
+
+sub load_exif_data {
+ my $filename = shift;
+ my $info = ImageInfo($filename);
+ # Not too important since ImageMagick already should weed out unsupported filetypes
+ # ExifTool seems to print at least *something* (e.g. permissions, etc.) even for files
+ # that don't have any embedded tags (including directories), so there probably aren't
+ # any files that ImageMagick supports but ExifTool doesn't.
+ if (exists $info->{"Error"}) {
+ warn "WARNING: Ignoring $filename as ExifTool returned an error: $info->{Error}\n";
+ return undef;
+ } elsif (exists $info->{"Warning"}) {
+ warn "WARNING: ExifTool returned a warning for $filename: $info->{Warning}\n";
+ }
+
+ for my $ignore (@exif_ignore) {
+ delete $info->{$ignore};
+ }
+ return $info;
+}
+
+# -1: error
+# 0: doesn't matter which image is chosen
+# 1: image1 should be chosen (includes more metadata than image2)
+# 2: image2 should be chosen (includes more metadata than image1)
+sub compare_exif_data {
+ # filenames are just for debugging
+ my ($info1, $info2, $filename1, $filename2) = @_;
+ # Make a new hash containing just the keys so we can delete keys even if
+ # the original infos should not be destroyed (makes the logic a bit simpler).
+ sub copy_hash_keys {
+ my %new_hash;
+ my $hash = shift;
+ for my $key (keys %$hash) {
+ $new_hash{$key} = 1;
+ }
+ return \%new_hash;
+ }
+ my $info1_keys = copy_hash_keys($info1);
+ my $info2_keys = copy_hash_keys($info2);
+
+ my $info1_superset = 0;
+ for my $key (keys %$info1_keys) {
+ # Need Data::Compare because there may be references, etc.
+ # (e.g. ThumbnailImage is a reference)
+ if (exists $info2_keys->{$key} && Compare($info1->{$key}, $info2->{$key}) == 1) {
+ # NOP
+ } elsif (!exists $info2_keys->{$key} || !defined $info2->{$key} || $info2->{$key} eq "") {
+ if ($key eq "Warning") {
+ warn "WARNING: ExifTool gave a warning on $filename1 but not on $filename2\n";
+ warn "Treating these files as different to be on the safe side.\n";
+ return -1;
+ }
+ # $info1 contains a key that is empty or nonexistent in $info2
+ $info1_superset = 1;
+ warn "$filename1 includes $key which is empty/nonexistent in $filename2\n" if ($verbose);
+ } else {
+ warn "$key is different for $filename1 and $filename2\n" if ($verbose);
+ return -1;
+ }
+ delete $info1_keys->{$key};
+ delete $info2_keys->{$key};
+ }
+
+ if ($info1_superset && %$info2_keys) {
+ if ($verbose) {
+ for my $key (keys %$info2_keys) {
+ warn "$filename2 includes $key which is nonexistent in $filename1\n";
+ }
+ warn "$filename1 and $filename2 both contain EXIF data not included in the other file\n";
+ }
+ return -1;
+ } elsif ($info1_superset) {
+ return 1;
+ } elsif (!%$info2_keys) {
+ return 0;
+ } else {
+ if (exists $info2_keys->{"Warning"}) {
+ warn "WARNING: ExifTool gave a warning on $filename2 but not on $filename1\n";
+ warn "Treating these files as different to be on the safe side.\n";
+ return -1;
+ }
+ if ($verbose) {
+ for my $key (keys %$info2_keys) {
+ warn "$filename2 includes $key which is nonexistent in $filename1\n";
+ }
+ }
+ return 2;
+ }
+}
+
+sub build_signature_list {
+ my @queue = @_;
+ my @signatures;
+ while (@queue) {
+ my $filename = pop @queue;
+ # Just ignore all symlinks. This is the easiest way to
+ # prevent issues, and there shouldn't be any symlinks in
+ # the directories I use this on anyways.
+ if (-l $filename) {
+ warn "WARNING: Ignoring symlink $filename\n";
+ next;
+ }
+ if (-d $filename) {
+ my $dh;
+ if (!opendir $dh, $filename) {
+ warn "WARNING: Unable to open directory $filename\n";
+ next;
+ }
+ # FIXME: Maybe test for errors on readdir:
+ # https://github.com/Perl/perl5/issues/17907
+ my @new_files = map catfile($filename, $_), grep {$_ ne "." && $_ ne ".."} readdir $dh;
+ closedir $dh;
+ push @queue, @new_files;
+ } elsif (-f $filename) {
+ if (!-r $filename) {
+ warn "WARNING: File $filename is not readable\n";
+ next;
+ }
+ if (!$skip_mime_check) {
+ my $mime = mimetype($filename);
+ if (!defined($mime)) {
+ warn "Unable to determine MIME type for $filename\n" if ($verbose);
+ next;
+ } elsif ($mime !~ /^image/) {
+ warn "Unsupported MIME type for $filename: $mime\n" if ($verbose);
+ next;
+ }
+ }
+ my $image = Image::Magick->new;
+ my $err = $image->Read($filename);
+ if ($err) {
+ $err =~ /(\d+)/;
+ # 420 is "no decode delegate for this image format", we don't want
+ # to spam that for every file that isn't an image format
+ if ($1 != 420 || $verbose) {
+ warn "WARNING: Unable to open file $filename with ImageMagick: $err\n";
+ }
+ next;
+ }
+ my $sig = $image->Get("signature");
+ if (!defined($sig)) {
+ warn "WARNING: Unable to obtain signature for $filename\n";
+ next;
+ }
+ # This is needed so that it's possible to avoid the same file ending up
+ # in the same deletion group (if the same file/directory was added
+ # to the command-line arguments twice - not very realistic, but let's
+ # just be on the safe side).
+ my $fullpath = realpath($filename);
+ if (!defined($fullpath)) {
+ warn "WARNING: Unable to get absolute path for $filename\n";
+ next;
+ }
+ push @signatures, [$fullpath, $sig];
+ } else {
+ warn "WARNING: Ignoring non-existent or non-regular file $filename\n";
+ }
+ }
+ return @signatures;
+}
+
+sub handle_group_exif {
+ # At this point, there really shouldn't be any errors reading the EXIF data since
+ # ImageMagick already successfully read the images (unless there's a file format
+ # for which ImageMagick has a loader but ExifTool doesn't).
+ my @exif_arr = grep defined($_->[3]), map [@$_, load_exif_data($_->[1])], @_;
+ my @all_groups;
+ while (@exif_arr) {
+ my $first = $exif_arr[0];
+ # Strip signature and exif data for elements in final array since we don't need them anymore
+ push @all_groups, [[$first->[0], $first->[1]]];
+ my @tmp_arr;
+ for my $i (1..$#exif_arr) {
+ my $ret = compare_exif_data($first->[3], $exif_arr[$i]->[3], $first->[1], $exif_arr[$i]->[1]);
+ if ($ret == 0 || $ret == 1) {
+ push @{$all_groups[-1]}, [$exif_arr[$i]->[0], $exif_arr[$i]->[1]];
+ } elsif ($ret == 2) {
+ unshift @{$all_groups[-1]}, [$exif_arr[$i]->[0], $exif_arr[$i]->[1]];
+ $first = $exif_arr[$i];
+ } else {
+ push @tmp_arr, $exif_arr[$i];
+ }
+ }
+ @exif_arr = @tmp_arr;
+ }
+ # Filter out single-element groups
+ return grep {$#$_ > 0} @all_groups;
+}
+
+sub handle_group_image_data {
+ my @cur_imgs = @_;
+ my @all_groups;
+ while (@cur_imgs) {
+ my $first = shift @cur_imgs;
+ my $image1 = Image::Magick->new;
+ my $err = $image1->Read($first->[1]);
+ # There really shouldn't be any errors here since all images have been
+ # previously read by ImageMagick to calculate the signatures, but let's
+ # just be on the safe side.
+ # Error 420 isn't ignored here because it doesn't matter if it's shown
+ # in the extremely rare case that it occurs here even though all the
+ # files were already opened previously.
+ if ($err) {
+ warn "WARNING: ImageMagick error while reading $first->[1]: $err\n";
+ next;
+ }
+ push @all_groups, [$first];
+ my @tmp_arr;
+ for my $i (0..$#cur_imgs) {
+ my $image2 = Image::Magick->new;
+ $err = $image2->Read($cur_imgs[$i]->[1]);
+ if ($err) {
+ warn "WARNING: ImageMagick error while reading $cur_imgs[$i]->[1]: $err\n";
+ next;
+ }
+ my $diff = $image1->Compare(image=>$image2, metric=>"ae");
+ # NOTE: If the two images have different sizes, ImageMagick doesn't seem to throw
+ # an error, but the returned error should be non-zero.
+ my $img_error = $diff->Get("error");
+ if (!defined($img_error)) {
+ warn "WARNING: Unknown error during comparison of $first->[1] and $cur_imgs[$i]->[1]\n";
+ $img_error = 1;
+ }
+ if ($img_error == 0) {
+ push @{$all_groups[-1]}, $cur_imgs[$i];
+ } else {
+ push @tmp_arr, $cur_imgs[$i];
+ }
+ }
+ @cur_imgs = @tmp_arr;
+ }
+ # Filter out single-element groups
+ return grep {$#$_ > 0} @all_groups;
+}
+
+my @extra_exif_ignore;
+my $no_default_exif_ignore = 0;
+my $show_help = 0;
+my $isolate = 0; # only find duplicates between different commandline arguments (i.e. not within one directory that is given)
+my $print_ignored = 0;
+my $skip_exact_equality = 0;
+my $delete_files = 0;
+my $no_prompt = 0;
+my $number_output = 0;
+my $default_choice = "";
+
+Getopt::Long::Configure("bundling");
+GetOptions (
+ "exif-ignore=s" => \@extra_exif_ignore,
+ "no-default-exif-ignore" => \$no_default_exif_ignore,
+ "h|help" => \$show_help,
+ "v|verbose" => \$verbose,
+ "I|isolate" => \$isolate,
+ "print-exif-ignore" => \$print_ignored,
+ "Q|quick" => \$skip_exact_equality,
+ "d|delete" => \$delete_files,
+ "N|no-prompt" => \$no_prompt,
+ "n|number" => \$number_output,
+ "default-choice=s" => \$default_choice,
+ "skip-mime-check" => \$skip_mime_check,
+) || pod2usage(-exitval => 1, -verbose => 1);
+@extra_exif_ignore = split(/,/, join(',', @extra_exif_ignore));
+if ($no_default_exif_ignore) {
+ @exif_ignore = @extra_exif_ignore;
+} else {
+ push @exif_ignore, @extra_exif_ignore;
+}
+
+if ($print_ignored) {
+ print join(",", @exif_ignore) . "\n";
+ exit 0;
+}
+
+pod2usage(-exitval => 0, -verbose => 2) if $show_help;
+pod2usage(-exitval => 1, -verbose => 1) if @ARGV < 1;
+if ($default_choice ne "" && $default_choice !~ /^[yYnNp]$/) {
+ warn "ERROR: --default-choice must be y, Y, n, N, or p!\n";
+ pod2usage(-exitval => 1, -verbose => 1);
+}
+
+my @siglist;
+for my $i (0..$#ARGV) {
+ # Add index of command-line argument for precedence when outputting which
+ # files should be deleted
+ # Format is now [original cmd argument index, full path, signature]
+ push @siglist, map [$i, $_->[0], $_->[1]], build_signature_list($ARGV[$i]);
+}
+
+# Sort first by signature, then by original command-line argument index, then by file path
+# Last sort by file path is so that if, for instance, there are two directories within
+# the same cmd arg (and isolate isn't enabled), duplicates are always removed from the
+# same directory.
+@siglist = sort {
+ $a->[2] cmp $b->[2] || $a->[0] <=> $b->[0] || $a->[1] cmp $b->[1];
+} @siglist;
+
+my @group;
+my @final_groups;
+my $prev_sig = "";
+for my $file (@siglist) {
+ if ($file->[2] ne $prev_sig) {
+ if (scalar(@group) > 1) {
+ if ($skip_exact_equality) {
+ # Need to make a copy of @group when storing it in @final_groups
+ push @final_groups, [@group];
+ } else {
+ push @final_groups, handle_group_image_data(@group);
+ }
+ }
+ @group = ();
+ $prev_sig = $file->[2];
+ }
+ push @group, $file;
+}
+if (scalar(@group) > 1) {
+ if ($skip_exact_equality) {
+ push @final_groups, [@group];
+ } else {
+ push @final_groups, handle_group_image_data(@group);
+ }
+}
+
+@final_groups = map {handle_group_exif(@$_)} @final_groups;
+# Sort by length of group, then by filename of first file in group
+@final_groups = sort {$#$a <=> $#$b || $a->[0]->[1] cmp $b->[0]->[1]} @final_groups;
+
+# 0: Do not delete files
+# 1: Delete files
+# 2: Go to previous duplicate group
+sub prompt_delete {
+ if ($no_prompt) {
+ return $delete_files;
+ }
+ my $choice = "";
+ my $default_printstr = $default_choice eq "" ? "" : "[$default_choice]";
+ while ($choice !~ /^[yYnNp]$/) {
+ print STDERR "Delete files? (y/Y/n/N/p)$default_printstr ";
+ $choice = <STDIN>;
+ chomp $choice;
+ if ($choice eq "") {
+ $choice = $default_choice;
+ }
+ }
+ if ($choice eq "y") {
+ return 1;
+ } elsif ($choice eq "n") {
+ return 0;
+ } elsif ($choice eq "Y") {
+ $delete_files = 1; # should already be 1 anyways
+ $no_prompt = 1;
+ return 1;
+ } elsif ($choice eq "N") {
+ $delete_files = 0;
+ $no_prompt = 1;
+ return 0;
+ } else {
+ return 2;
+ }
+}
+
+# It doesn't matter that the same reference will be repeated here since the
+# array elements are replaced later anyways
+my @final_delete_files = ([]) x scalar(@final_groups);
+sub output_groups {
+ my $cur_group_id = shift;
+ while ($cur_group_id <= $#final_groups) {
+ # Reset so it is set correctly when going back to correct mistakes
+ $final_delete_files[$cur_group_id] = [];
+ my $group = $final_groups[$cur_group_id];
+ print "\n" if ($cur_group_id > 0);
+ if ($number_output) {
+ print "[" . ($cur_group_id + 1) . "/" . ($#final_groups + 1) . "]\n";
+ }
+ my $first = $group->[0];
+ print "[+] $first->[1]\n";
+ # Note: This may be a bit inefficient, but these hashes make it simple to avoid mistakes
+ # and give somewhat decent diagnostic messages.
+ my %keep_list;
+ my %delete_list;
+ $keep_list{$first->[1]} = 1;
+ for my $file (@$group[1..$#$group]) {
+ if ($isolate && $file->[0] == $first->[0]) {
+ print "[+] $file->[1]\n";
+ $keep_list{$file->[1]} = 1;
+ } else {
+ if (exists $keep_list{$file->[1]}) {
+ warn "WARNING: Ignoring $file->[1] as it was queued for deletion but refers " .
+ "to the same file as a file in the list of files to keep\n";
+ } elsif (exists $delete_list{$file->[1]}) {
+ warn "WARNING: $file->[1] already queued for deletion\n";
+ } else {
+ print "[-] $file->[1]\n";
+ $delete_list{$file->[1]} = 1;
+ }
+ }
+ }
+ if ($delete_files && (my $choice = prompt_delete())) {
+ if ($choice == 1) {
+ $final_delete_files[$cur_group_id] = [keys %delete_list];
+ } else {
+ $cur_group_id--;
+ if ($cur_group_id <= 0) {
+ $cur_group_id = 0;
+ print "\n";
+ }
+ next;
+ }
+ }
+ $cur_group_id++;
+ }
+}
+
+output_groups(0);
+
+# Loop here so the user can still correct mistakes for the previous prompts when
+# we have already reached the final prompt.
+while (1) {
+ my $any_to_delete = 0;
+ for my $delete_group (@final_delete_files) {
+ if (@$delete_group) {
+ $any_to_delete = 1;
+ last;
+ }
+ }
+ if ($any_to_delete) {
+ print "\n" if !$no_prompt;
+ my $choice = $no_prompt ? "y" : "";
+ while ($choice !~ /^[ynp]$/) {
+ print "Delete all chosen files? (y/n/p) ";
+ $choice = <STDIN>;
+ chomp $choice;
+ }
+ if ($choice eq "y") {
+ for my $delete_group (@final_delete_files) {
+ for my $filename (@$delete_group) {
+ warn "Deleting $filename\n" if ($verbose);
+ if (!unlink($filename)) {
+ warn "WARNING: Unable to delete $filename\n";
+ }
+ }
+ }
+ } elsif ($choice eq "p") {
+ print "\n" if $#final_groups == 0;
+ # Go back to group output, starting at last group
+ output_groups($#final_groups);
+ next;
+ }
+ }
+ last;
+}
+
+__END__
+
+=head1 NAME
+
+imagedatadupes - find duplicate images
+
+=head1 SYNOPSIS
+
+B<imagedatadupes> [option ...] file/directory ...
+
+ Options:
+ -I, --isolate
+ -d, --delete
+ -N, --no-prompt
+ -Q, --quick
+ -n, --number
+ -v, --verbose
+ -h, --help
+ --default-choice=CHOICE
+ --exif-ignore=TAG,...
+ --no-default-exif-ignore
+ --print-exif-ignore
+ --skip-mime-check
+
+=head1 DESCRIPTION
+
+B<imagedatadupes> compares image files to check for duplicates and optionally deletes them.
+Duplicates are found even if the files are not identical, as long as the image data is the same,
+and the metadata is similar enough based on certain criteria.
+
+The images are first compared using signatures of the image data calculated with ImageMagick(1).
+Then, images with matching signatures are compared for exact equality of the image data. Lastly,
+the metadata returned by ExifTool(1) is compared (this includes various metadata tags, not just
+EXIF data). If the metadata in one file is a superset of the metadata in another file, the
+former is preferred over the latter when deciding which files to keep.
+
+The specific use-case for this is files where the EXIF data was modified to add a comment (e.g. where
+a photo was taken). If an old copy of the same image still exists, it won't be found as a duplicate
+by programs such as fdupes(1) and jdupes(1) since the EXIF data is different, but the new version
+only contains more data than the old one, so there's no point in keeping the old copy. In order for this
+to work in practice, some metadata tags that ExifTool returns, such as "FileModifyDate", need to be
+ignored (see B<--exif-ignore>, B<--no-default-exif-ignore>, and B<--print-exif-ignore>).
+
+The output format consists of groups of files separated by empty lines, with each file being preceded
+by [+] or [-], depending on whether it was automatically selected for keeping or deletion, respectively.
+If B<--delete> is not used, duplicates are only printed and no actual deletion is performed.
+
+If multiple files are identical (both image data and metadata), files that were included in earlier
+command-line parameters are preferred for keeping. If multiple files included in the same command-line
+parameter are identical and B<--isolate> is not used, the file whose absolute path comes first
+lexicographically is preferred for keeping.
+
+Before being output, the groups are sorted first based on the number of files in each group, then
+based on the lexicographical order of the absolute path of the first file in each group.
+This is mainly so that it is easier to spot outliers, such as a group where the file chosen
+for keeping (i.e. the first file in the group) is not in the expected directory.
+
+See B<CAVEATS> for some issues that can arise.
+
+B<WARNING:> Always make backups of your files before using this program to delete duplicates. There is
+a nonzero chance that bugs still exist which could cause the wrong files to be deleted. If you have
+directories containing files that should not be modified in any way, and you only want to check if
+there are duplicates of those files in other directories, you may want to change the permissions
+of those files so they are read-only and a malfunction of this program cannot cause any issues.
+At least that's what a paranoid person such as the author of this program would do.
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<-I>, B<--isolate>
+
+Isolate the command-line parameters from each other. In other words, for each group of
+duplicates, all files belonging to the same command-line parameter as the file selected
+for keeping are also kept.
+
+=item B<-d>, B<--delete>
+
+Prompt user for deletion of duplicates. Note that the actual deletion is not performed until all
+duplicate groups have been processed and the user has been prompted once more to confirm that all
+the chosen files should really be deleted.
+The options are as follows:
+
+=over 4
+
+=item B<y>
+
+Delete the files that were automatically selected for deletion in the
+current duplicate group.
+
+=item B<Y>
+
+Delete the files that were automatically selected for deletion in the
+current duplicate group and in all subsequent duplicate groups.
+
+=item B<n>
+
+Do not delete the files that were automatically selected for deletion
+in the current duplicate group.
+
+=item B<N>
+
+Do not delete the files that were automatically selected for deletion
+in the current duplicate group and in all subsequent duplicate groups.
+
+=item B<p>
+
+Go back to the previous duplicate group. This is particularly useful
+together with B<--default-choice> so that choices can be made very
+quickly, but mistakes can still be corrected by going back to previous
+duplicate groups. Previously chosen options are not saved, so when
+B<p> is used to go back multiple groups, all choices have to be made
+again.
+
+=back
+
+Note that B<Y> and B<N> also cause the final prompt asking if all chosen
+files should be deleted to be skipped.
+
+=item B<-N>, B<--no-prompt>
+
+When used in conjunction with B<--delete>, automatically delete duplicates without prompting the user.
+This option should be used with care as it can lead to data loss. Make sure to always have backups
+of your data in case something goes wrong.
+
+=item B<-Q>, B<--quick>
+
+Only determine duplicates based on the data signature and metadata (skip the check for exact
+equality of the image data).
+
+=item B<-n>, B<--number>
+
+Print the number of the current group and the total number of groups for each duplicate group
+that is output. This is useful in interactive mode so there is an indication of how many
+groups are still left.
+
+=item B<-v>, B<--verbose>
+
+Print informational messages to stdout.
+This currently includes some file reading errors, as well as information on the metadata
+comparison (e.g. which tags were present in one file but not in the other one).
+Additionally, all filenames are printed once more right before they are actually deleted.
+
+=item B<-h>, B<--help>
+
+Print usage information, then exit.
+
+=item B<--default-choice>=CHOICE
+
+Set a default choice to use in interactive mode. This must be one of the characters that can be
+entered at a prompt during interactive mode. When this is set, it is possible to simply press
+enter at each prompt in order to use the default choice. Note that this is only used during
+interactive mode, it does not have any effect when B<--no-prompt> is in effect.
+
+=item B<--exif-ignore>=TAG,...
+
+Add additional metadata tags to the list of tags that should be ignored when comparing two files.
+This option can be given multiple times. Alternatively, multiple tags can be given at once
+as a comma-separated list. The B<--verbose> option might help to figure out why two images
+aren't being considered as duplicates. Additionally, ExifTool can be used to find the list
+of all tag names an image contains using the command "exiftool -s image.jpg". It is generally
+a good idea to start with the default ignore list and only add more tags later if needed.
+Adding too many tags to the ignore list can easily lead to data loss when files are considered
+to be duplicates even though the values for some ignored metadata tags were different.
+
+=item B<--no-default-exif-ignore>
+
+Do not use the default list of ignored metadata tags.
+
+WARNING: As this program has not reached version 1.0 yet, the default list of ignored metadata tags
+may still change without warning. Always use B<--exif-ignore> together with B<--no-default-exif-ignore>
+if you want to be certain that nothing unexpected happens.
+
+=item B<--print-exif-ignore>
+
+Print all metadata tags that will be ignored, including the default ones if they haven't been turned off
+using B<--no-default-exif-ignore>.
+
+=item B<--skip-mime-check>
+
+Do not check the MIME type of files. Instead, simply process anything that is supported by ImageMagick
+and ExifTool. By default, only MIME types starting with "image" are processed in order to avoid issues
+with ImageMagick trying to open large video files or similar.
+
+=back
+
+=head1 CAVEATS
+
+The ordering of the files within a duplicate group in the output can be a bit weird.
+In particular, if a file is chosen for keeping because it contains more metadata, it will be displayed
+at the beginning of the file list, regardless of the regular sorting. This can seem strange when
+there are multiple duplicates each in multiple command-line parameters since it can happen that the
+duplicates from one command-line parameter are not written right after each another if one of them was
+moved to the beginning of the output due to its metadata.
+
+Before printing the output, a final check is performed to make sure that none of the files in
+a duplicate group have the same path (this should only happen if the same directory/file was given
+multiple times). This can cause only one file to be displayed for a duplicate group.
+In a similar vein, when B<--isolate> is used, it is possible for a duplicate group to be printed
+even though none of the files are considered for deletion.
+
+If there are any errors while reading a file, the file is ignored during further processing.
+Errors that are only due to a filetype not being supported are normally not printed (except
+in rare circumstances) in order to avoid spamming the output with error messages for all
+non-image files. These are only printed when B<--verbose> is used.
+
+Originally, any warnings given by ExifTool also caused the file to be ignored. However, this
+caused issues because it meant that identical files which both included the same warning
+were not considered as duplicates. Thus, warnings are now treated almost the same as
+other metadata. The only difference is that warnings cannot be used to decide that one file
+contains a superset of another file's metadata. This is a safety feature as it is theoretically
+possible for two files to be identical, except that one is damaged in some way, leading to a
+warning. If warnings were treated exactly the same as other metadata, the file without the
+warning would be deleted as it contains less metadata, but this would probably not be the
+correct choice. In that case, a warning is printed and the files are not considered as
+duplicates so they can be examined manually later. If warnings should be disregarded
+entirely when comparing files, the "Warning" tag can be ignored using B<--exif-ignore>.
+
+If there is an unknown error during the exact comparison of the image data, this might be due to
+memory restrictions in the ImageMagick configuration. Try opening the ImageMagick configuration
+file "policy.xml" and changing the maximum memory. The configuration file is probably located
+at /etc/ImageMagick-6/policy.xml or /etc/ImageMagick-7/policy.xml, depending on the version of
+ImageMagick. There should be a line similar to
+'<policy domain="resource" name="memory" value="256MiB"/>' that can be changed to increase the
+maximum memory. There is also a similar line with "disk" instead of "memory" that can be used
+to control how much temporary disk space ImageMagick is allowed to use when there is not enough
+memory. The "map" and "area" options may also be of interest. See the ImageMagick documentation
+for a description of the other options in the policy.xml file. Note that, depending on how much
+memory is available, it might be a good idea to set the disk limit to 0 in order to fail more
+quickly and avoid a lot of disk writes when large files are opened.
+
+ImageMagick seems to sometimes open files without any error even though they don't contain any
+image data. For instance, it managed to open some JPEG files that had been corrupted and only
+contained zeroes, calculating the same signature for all of them. This should not be an issue
+normally since a MIME type check is now performed before trying to read any files with ImageMagick.
+Another interesting case found in the wild dealt with .exi files, which appeared to contain just
+the EXIF data from JPEG files. These passed the MIME type check, and ImageMagick calculated the
+same signature for all of them, but returned an error later on during the exact image data
+comparison. Such cases should generally not be a problem since the image data or metadata
+comparison should still fail, causing the files to be treated as different, but it's probably
+still a good idea to keep these edge cases in mind.
+
+The image data used for the comparisons is whatever ImageMagick decompresses the image
+files to, which might include various inaccuracies. However, that probably isn't an issue
+in any realistic scenario.
+
+For simplicity, all comparisons are performed even when B<--isolate> is used and it technically
+isn't necessary to compare files from the same command-line parameter.
+
+This program completely ignores all symbolic links in order to avoid all the issues that
+can be caused by them.
+
+=head1 EXAMPLES
+
+=over 8
+
+=item B<imagedatadupes -dnI --default-choice=y directory1 directory2 ...>
+
+Look for duplicates in the given directories, making sure that duplicate images belonging to the
+same command-line paramater are either all kept or all deleted. The user is prompted to confirm
+the deletion of each duplicate group, with the default choice being set to B<y> so that it is
+easy to always press enter (if a mistake is made, B<p> can be entered at the prompt to go back
+to the previous duplicate group). In order to give an indication of progress during the
+prompting for deletion, the output groups are numbered.
+
+=item B<imagedatadupes -dnI --default-choice=y --exif-ignore=ThumbnailOffset directory1 directory2 ...>
+
+Same as above, but additionally ignore the "ThumbnailOffset" tag when comparing image metadata.
+
+=back
+
+=head1 EXIT STATUS
+
+Always 0, unless the arguments given were invalid.
+
+=head1 SEE ALSO
+
+ExifTool(1), fdupes(1), findimagedupes(1p), ImageMagick(1), jdupes(1)
+
+=head1 LICENSE
+
+Copyright (c) 2026 lumidify <nobody@lumidify.org>
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
