lumia

Archive checksum manager
git clone git://lumidify.org/lumia.git (fast, but not encrypted)
git clone https://lumidify.org/git/lumia.git (encrypted, but very slow)
Log | Files | Refs | README | LICENSE

lumia.pl (44930B)


      1 #!/usr/bin/env perl
      2 
      3 # TODO: some way to avoid writing .lumidify* in dirs but still index them?
      4 # TODO: store modified date and checksum files with changed date
      5 # TODO: add option to just check dir structure or maybe check if everything exists
      6 # TODO: add option to compare cksums of two dirs
      7 # TODO: exit status!
      8 
      9 use strict;
     10 use warnings;
     11 use File::Spec::Functions qw(catfile abs2rel);
     12 use File::Basename qw(basename dirname);
     13 use String::ShellQuote;
     14 use Pod::Usage;
     15 use Getopt::Long;
     16 
     17 # the file used to store checksums for files
     18 my $CKSUM_FILE = ".lumidify_archive_cksums";
     19 # the file used to store directory names
     20 my $DIR_FILE = ".lumidify_archive_dirs";
     21 # the file read to ignore files or directories
     22 my $IGNORE_FILE = ".lumidify_archive_ignore";
     23 # the file containing checksums of $CKSUM_FILE and $DIR_FILE
     24 my $DOUBLE_CKSUM_FILE = ".lumidify_archive_cksums.cksum";
     25 
     26 # uncomment this instead of the lines below to use
     27 # sha256 instead of cksum as the hash algorithm
     28 # Note: this isn't really tested properly
     29 #my $CKSUM_CMD = 'sha256 -q';
     30 #my $CKSUM_NUMFIELDS = 1;
     31 my $CKSUM_CMD = 'cksum';
     32 my $CKSUM_NUMFIELDS = 2;
     33 
     34 my %SPECIAL_FILES = (
     35 	$CKSUM_FILE => 1,
     36 	$DIR_FILE => 1,
     37 	$IGNORE_FILE => 1,
     38 	$DOUBLE_CKSUM_FILE => 1
     39 );
     40 
     41 # escape a filename for writing into the checksum files
     42 sub escape_filename {
     43 	my $file = shift;
     44 	$file =~ s/\\/\\\\/g;
     45 	$file =~ s/"/\\"/g;
     46 	return $file;
     47 }
     48 
     49 # make a generic file iterator
     50 # $file_func determines whether a file should be returned by the iterator
     51 # $dir_func is called for each directory and returns all files that
     52 # should be added to the queue
     53 sub make_file_iter {
     54 	my ($file_func, $dir_func, @queue) = @_;
     55 	return sub {
     56 		while (@queue) {
     57 			my $file = pop @queue;
     58 			if (-d $file) {
     59 				my $new_files = $dir_func->($file);
     60 				next if !defined $new_files;
     61 				push @queue, @$new_files;
     62 			}
     63 			return $file if $file_func->($file);
     64 		}
     65 		return;
     66 	};
     67 }
     68 
     69 # make a basic filename iterator, which simply returns all files
     70 # for which $file_func returns a true value
     71 sub make_file_iter_basic {
     72 	my ($file_func, @files) = @_;
     73 	make_file_iter $file_func, sub {
     74 		my $dh;
     75 		if (!opendir $dh, $_[0]) {
     76 			warn "WARNING: Unable to open directory \"$_[0]\"!";
     77 			return [];
     78 		}
     79 		my @new_files = map "$_[0]/$_", grep {$_ ne "." && $_ ne ".."} readdir $dh;
     80 		closedir $dh;
     81 		return \@new_files;
     82 	}, @files;
     83 }
     84 
     85 # make an interator that only returns the directories which are present
     86 # in the $DIR_FILE files, in addition to the files and directories that
     87 # were originally passed as arguments
     88 # note: this returns nonexistent directories if those are still
     89 # specified in the lumia files
     90 sub make_lumia_iter {
     91 	my ($quiet, @dirs) = @_;
     92 	make_file_iter sub {1}, sub {
     93 		my $path = "$_[0]/$DIR_FILE";
     94 		return [] if !-f $path;
     95 		my $dirs = read_file($path, {});
     96 		return if !defined $dirs;
     97 		my @new_dirs;
     98 		foreach my $dir (keys %$dirs) {
     99 			my $dir_path = "$_[0]/$dir";
    100 			if (!-d $dir_path) {
    101 				warn "ERROR: Directory \"$dir_path\" mentioned in " .
    102 					"\"$path\" does not exist or is not directory.\n" if !$quiet;
    103 			}
    104 			# still push it even when it doesn't exist so rmold can work properly
    105 			push @new_dirs, $dir_path;
    106 		}
    107 		return \@new_dirs;
    108 	}, @dirs;
    109 }
    110 
    111 # remove all special lumia files from the given directory
    112 sub clean_files {
    113 	my ($dir, $args) = @_;
    114 	my $iter = make_file_iter_basic sub {exists $SPECIAL_FILES{basename $_[0]};}, $dir;
    115 	while (my $file = $iter->()) {
    116 		if (!unlink $file) {
    117 			warn "WARNING: Unable to remove file \"$file\"!\n";
    118 		} else {
    119 			print "Deleted \"$file\"\n" if !$args->{"q"};
    120 		}
    121 	}
    122 }
    123 
    124 # read a file, processing each line with $handle_cksum_func if set
    125 # and writing the results into $cksums
    126 # $handle_cksum_func must return two values, the checksum of the
    127 # argument and the rest of the string (that is then parsed for
    128 # the filename); if it returns undef, this function also returns undef
    129 sub read_file {
    130 	my ($file, $cksums, $handle_cksum_func) = @_;
    131 	my $fh;
    132 	if (!open $fh, "<", $file) {
    133 		warn "ERROR: Unable to open file \"$file\": $!\n";
    134 		return;
    135 	}
    136 	my $in_fn = 0;
    137 	my $cur_cksum;
    138 	my $cur_str;
    139 	my $cur_fn = "";
    140 	foreach (<$fh>) {
    141 		next if (!$in_fn && /^$/);
    142 		if ($handle_cksum_func && !$in_fn) {
    143 			($cur_cksum, $cur_str) = $handle_cksum_func->($_);
    144 			return undef if !defined $cur_cksum;
    145 		} else {
    146 			$cur_str = $_;
    147 		}
    148 		my $bs = 0;
    149 		foreach my $ch (split(//, $cur_str)) {
    150 			if ($ch eq "\\") {
    151 				$bs++;
    152 				$cur_fn .= "\\" if !($bs %= 2) && $in_fn;
    153 			} elsif ($bs % 2) {
    154 				$cur_fn .= $ch  if $in_fn;
    155 				$bs = 0;
    156 			} elsif ($ch eq "\"") {
    157 				if ($in_fn) {
    158 					$in_fn = 0;
    159 					$cksums->{$cur_fn} = $cur_cksum;
    160 					$cur_fn = "";
    161 					last;
    162 				}
    163 				$in_fn = 1;
    164 			} elsif ($in_fn) {
    165 				$cur_fn .= $ch;
    166 			}
    167 		}
    168 	}
    169 	close $fh;
    170 	if ($in_fn) {
    171 		warn "ERROR: Unterminated filename in file \"$file\"\n";
    172 		return undef;
    173 	}
    174 	return $cksums;
    175 }
    176 
    177 # read a single checksum file, writing the checksums into the hash $cksums and returning it
    178 sub read_cksum_file {
    179 	my ($file, $cksums) = @_;
    180 	return read_file $file, $cksums, sub {
    181 		my $line = shift;
    182 		my @fields = split(/ /, $line, $CKSUM_NUMFIELDS+1);
    183 		if (@fields != $CKSUM_NUMFIELDS+1) {
    184 			warn "WARNING: Malformed line \"$line\" in file \"$file\"\n";
    185 			return;
    186 		}
    187 		my $cur_cksum = join(" ", @fields[0..$CKSUM_NUMFIELDS-1]);
    188 		my $cur_str = $fields[$CKSUM_NUMFIELDS];
    189 		return ($cur_cksum, $cur_str);
    190 	};
    191 }
    192 
    193 # read the checksums and directory names in $dir
    194 sub read_cksums {
    195 	my $dir = shift;
    196 	my $cksums = read_cksum_file("$dir/$CKSUM_FILE", {});
    197 	return undef if !defined $cksums;
    198 	$cksums = read_file("$dir/$DIR_FILE", $cksums);
    199 	return undef if !defined $cksums;
    200 	return $cksums;
    201 }
    202 
    203 # get the checksum output for $path
    204 # returns undef if $CKSUM_CMD returns an error
    205 sub get_cksum {
    206 	my $path = shift;
    207 	my $path_esc = shell_quote $path;
    208 	my $cksum_output = `$CKSUM_CMD -- $path_esc 2>&1`;
    209 	if ($?) {
    210 		warn "ERROR getting cksum for file \"$path\":\n$cksum_output";
    211 		return undef;
    212 	}
    213 	chomp $cksum_output;
    214 	my @fields = split(/ /, $cksum_output, $CKSUM_NUMFIELDS+1);
    215 	return join(" ", @fields[0..$CKSUM_NUMFIELDS-1]);
    216 }
    217 
    218 # check the checksums in $dir/$cksum_file
    219 # if $quiet is set, only print failed files
    220 sub check_cksums {
    221 	my ($dir, $cksum_file, $quiet) = @_;
    222 	my $cksums = read_cksum_file("$dir/$cksum_file", {});
    223 	return 0 if !defined $cksums;
    224 	my $failed = 1;
    225 	foreach my $file (keys %$cksums) {
    226 		my $path = "$dir/$file";
    227 		my $output = get_cksum $path;
    228 		next if !defined $output;
    229 		if ($output eq $cksums->{$file}) {
    230 			print "OK $path\n" if !$quiet;
    231 		} else {
    232 			print "FAILED $path\n";
    233 			$failed = 0;
    234 		}
    235 	}
    236 	return $failed;
    237 }
    238 
    239 # check the checksums of all files and directories in @dirs
    240 sub check_files {
    241 	my $args = shift;
    242 	my @dirs;
    243 	foreach my $file (@_) {
    244 		if (-d $file) {
    245 			push @dirs, $file;
    246 			next;
    247 		}
    248 		my $dir = dirname $file;
    249 		my $base = basename $file;
    250 		if (exists $SPECIAL_FILES{$base}) {
    251 			warn "ERROR: File is reserved for lumia.pl: $file\n";
    252 			next;
    253 		}
    254 		my $cksums = read_cksum_file("$dir/$CKSUM_FILE");
    255 		next if !defined $cksums;
    256 		if (!exists $cksums->{$base}) {
    257 			warn "ERROR: File doesn't exist in checksums: $file\n";
    258 			next;
    259 		}
    260 		my $output = get_cksum "$file";
    261 		next if !defined $output;
    262 		if ($output eq $cksums->{$base}) {
    263 			print "OK $file\n" if !$args->{"q"};
    264 		} else {
    265 			print "FAILED $file\n";
    266 		}
    267 	}
    268 	my $iter = make_lumia_iter 0, @dirs;
    269 	while (my $file = $iter->()) {
    270 		check_cksums $file, $DOUBLE_CKSUM_FILE, $args->{"q"};
    271 		check_cksums $file, $CKSUM_FILE, $args->{"q"};
    272 	}
    273 }
    274 
    275 # write the checksums of the special lumia files given as arguments
    276 # to $DOUBLE_CKSUM_FILE in $dir
    277 sub write_special_cksums {
    278 	my ($dir, @files) = @_;
    279 	my $cksum_file = "$dir/$DOUBLE_CKSUM_FILE";
    280 	my $cksums = {};
    281 	if (-f $cksum_file) {
    282 		$cksums = read_cksum_file $cksum_file, {};
    283 	}
    284 	return if !defined $cksums;
    285 	foreach my $file (@files) {
    286 		my $cksum_output = get_cksum("$dir/$file");
    287 		next if (!defined $cksum_output);
    288 		$cksums->{$file} = $cksum_output;
    289 	}
    290 	write_file($cksum_file, $cksums, 1);
    291 }
    292 
    293 # search for new files that aren't present in the checksum files
    294 # - if $file_func is set, it is called for each new file
    295 # - if $before_dir_func is set, it is called before processing the
    296 #   files in each directory that has new files OR if a directory
    297 #   is entirely new (well, it only checks if $DOUBLE_CKSUM_FILE exists)
    298 # - if $after_dir_func is set, it is called after processing the
    299 #   files in each directory that has new files
    300 sub check_new_files {
    301 	my ($top_dir, $file_func, $before_dir_func, $after_dir_func) = @_;
    302 	my $iter = make_file_iter sub {1}, sub {
    303 		my $dir = shift;
    304 		my $dh;
    305 		if (!opendir $dh, $dir) {
    306 			warn "ERROR: Unable to open directory \"$dir\"!";
    307 			return undef;
    308 		}
    309 		my $read_file_noerror = sub {
    310 			if (-f $_[0]) {
    311 				return $_[1]->($_[0], {}) // {};
    312 			}
    313 			return {};
    314 		};
    315 		my $ignore = $read_file_noerror->("$dir/$IGNORE_FILE", \&read_file);
    316 		my $lumia_dirs = $read_file_noerror->("$dir/$DIR_FILE", \&read_file);
    317 		my $lumia_files = $read_file_noerror->("$dir/$CKSUM_FILE", \&read_cksum_file);
    318 		my @dirs;
    319 		my $found = 0;
    320 		while (my $file = readdir $dh) {
    321 			next if $file eq "." || $file eq "..";
    322 			next if exists $ignore->{$file} || exists $SPECIAL_FILES{$file};
    323 			if (!exists $lumia_dirs->{$file} && !exists $lumia_files->{$file}) {
    324 				if (!$found && defined $before_dir_func) {
    325 					last if !$before_dir_func->($dir);
    326 				}
    327 				if (defined $file_func) {
    328 					$file_func->($dir, $file);
    329 				} else {
    330 					print "$dir/$file\n";
    331 				}
    332 				$found = 1;
    333 			}
    334 			push @dirs, "$dir/$file" if -d "$dir/$file";
    335 		}
    336 		closedir $dh;
    337 		# also call $before_dir_func if the directory has not been initialized yet
    338 		if (!$found && !-f "$dir/$DOUBLE_CKSUM_FILE" && defined $before_dir_func) {
    339 			$before_dir_func->($dir);
    340 		}
    341 		if ($found && defined $after_dir_func) {
    342 			$after_dir_func->($dir);
    343 		}
    344 		return \@dirs;
    345 	}, $top_dir;
    346 	# Is this a horrible hack? I dunno, but it sure is sweet...
    347 	while ($iter->()) {}
    348 }
    349 
    350 # add all new files in $top_dir to the checksum files
    351 sub check_add_new_files {
    352 	my ($top_dir, $args) = @_;
    353 	my $changed_dirs = 0;
    354 	my $changed_files = 0;
    355 	check_new_files $top_dir, sub {
    356 		my ($dir, $file) = @_;
    357 		my $fullpath = "$dir/$file";
    358 		if (-d $fullpath) {
    359 			my $dir_file = "$dir/$DIR_FILE";
    360 			my $fh;
    361 			if (!open $fh, ">>", $dir_file) {
    362 				warn "ERROR: Unable to append to file \"$dir_file\"!";
    363 				return;
    364 			}
    365 			print $fh '"' . escape_filename($file) . '"' . "\n";
    366 			close $fh;
    367 			$changed_dirs = 1;
    368 		} else {
    369 			my $cksum_output = get_cksum $fullpath;
    370 			return if !defined $cksum_output;
    371 			my $cksum_file = "$dir/$CKSUM_FILE";
    372 			my $fh;
    373 			if (!open $fh, ">>", $cksum_file) {
    374 				warn "ERROR: Unable to append to file \"$cksum_file\"!";
    375 				return;
    376 			}
    377 			print $fh $cksum_output . ' "' . escape_filename($file) . '"' . "\n";
    378 			close $fh;
    379 			$changed_files = 1;
    380 		}
    381 		print "Added \"$fullpath\"\n" if !$args->{"q"};
    382 	}, sub {
    383 		if (-f "$_[0]/$DOUBLE_CKSUM_FILE") {
    384 			if (!check_cksums $_[0], $DOUBLE_CKSUM_FILE, 1) {
    385 				warn "Checksum files corrupt in \"$_[0]\", not adding new checksums!\n";
    386 				return 0;
    387 			}
    388 
    389 		} else {
    390 			write_cksums($_[0], {}, 1, 1);
    391 		}
    392 		return 1;
    393 	}, sub {
    394 		if ($changed_dirs) {
    395 			write_special_cksums $_[0], $DIR_FILE;
    396 			$changed_dirs = 0;
    397 		}
    398 		if ($changed_files) {
    399 			write_special_cksums $_[0], $CKSUM_FILE;
    400 			$changed_files = 0;
    401 		}
    402 	};
    403 }
    404 
    405 # write the "checksums" in $contents to $path
    406 # if $is_cksum_file is set, the value each of the keys in $contents points
    407 # to is written before the key
    408 sub write_file {
    409 	my ($path, $contents, $is_cksum_file) = @_;
    410 	my $fh;
    411 	if (!open $fh, ">", $path) {
    412 		warn "ERROR: Unable to open \"$path\" for writing!";
    413 		return;
    414 	}
    415 	foreach my $filename (keys %$contents) {
    416 		if ($is_cksum_file) {
    417 			print $fh "$contents->{$filename} ";
    418 		}
    419 		print $fh '"' . escape_filename($filename) . '"' . "\n";
    420 	}
    421 	close $fh;
    422 }
    423 
    424 # write the checksums in $contents to the file at $path
    425 sub write_cksum_file {
    426 	my ($path, $contents) = @_;
    427 	write_file $path, $contents, 1;
    428 }
    429 
    430 # write the checksums in $contents to $dir
    431 # any keys that point to undef are taken to be directories and vice versa
    432 # $files_modified and $dirs_modified control which of the special lumia
    433 # files actually get written
    434 # note: this doesn't use write_file, etc. in order to (possibly) be a bit more efficient
    435 sub write_cksums {
    436 	my ($dir, $contents, $files_modified, $dirs_modified) = @_;
    437 	# No, this isn't efficient...
    438 	my @special_files;
    439 	my $dirs_fh;
    440 	my $files_fh;
    441 	if ($files_modified) {
    442 		my $path = "$dir/$CKSUM_FILE";
    443 		if (!open $files_fh, ">", $path) {
    444 			warn "ERROR: Unable to open \"$path\" for writing!";
    445 			return;
    446 		}
    447 		push @special_files, $CKSUM_FILE;
    448 	}
    449 	if ($dirs_modified) {
    450 		my $path = "$dir/$DIR_FILE";
    451 		if (!open $dirs_fh, ">", $path) {
    452 			warn "ERROR: Unable to open \"$path\" for writing!";
    453 			return;
    454 		}
    455 		push @special_files, $DIR_FILE;
    456 	}
    457 	foreach my $key (keys %$contents) {
    458 		if ($files_modified && defined $contents->{$key}) {
    459 			print $files_fh $contents->{$key} . ' "' . escape_filename($key) . '"' . "\n";
    460 		} elsif ($dirs_modified && !defined $contents->{$key}) {
    461 			print $dirs_fh '"' . escape_filename($key) . '"' . "\n";
    462 		}
    463 	}
    464 	close $files_fh if defined $files_fh;
    465 	close $dirs_fh if defined $dirs_fh;
    466 	if (@special_files) {
    467 		write_special_cksums $dir, @special_files;
    468 	}
    469 }
    470 
    471 # show all files that are present in the checksum files but don't exist on the filesystem anymore
    472 sub check_old_files {
    473 	my $top_dir = shift;
    474 	my $iter = make_lumia_iter 1, $top_dir;
    475 	while (my $dir = $iter->()) {
    476 		if (-e $dir) {
    477 			my $cksums = read_cksum_file("$dir/$CKSUM_FILE", {}) // {};
    478 			foreach my $file (keys %$cksums) {
    479 				if (!-e "$dir/$file") {
    480 					warn "Nonexistent file: \"$dir/$file\"!\n";
    481 				}
    482 			}
    483 		} else {
    484 			warn "Nonexistent directory: \"$dir\"!\n";
    485 		}
    486 	}
    487 }
    488 
    489 # clean up the lumia checksum files, removing any files that aren't present
    490 # on the filesystem anymore
    491 sub remove_old_files {
    492 	my ($top_dir, $args) = @_;
    493 	my $iter = make_lumia_iter 1, $top_dir;
    494 	while (my $dir = $iter->()) {
    495 		if (!-e $dir) {
    496 			my $parent = dirname $dir;
    497 			my $child = basename $dir;
    498 			my $lumia_dirs = read_file("$parent/$DIR_FILE", {}) // {};
    499 			if (exists $lumia_dirs->{$child}) {
    500 				delete $lumia_dirs->{$child};
    501 				write_file "$parent/$DIR_FILE", $lumia_dirs;
    502 				print "Removed \"$dir\" from \"$parent/$DIR_FILE\"\n" if !$args->{"q"};
    503 				write_special_cksums $parent, $DIR_FILE;
    504 			}
    505 		} else {
    506 			my $cksums = read_cksum_file("$dir/$CKSUM_FILE", {}) // {};
    507 			my $found = 0;
    508 			foreach my $file (keys %$cksums) {
    509 				if (!-e "$dir/$file") {
    510 					delete $cksums->{$file};
    511 					print "Removed \"$dir/$file\" from \"$dir/$CKSUM_FILE\"\n" if !$args->{"q"};
    512 					$found = 1;
    513 				}
    514 			}
    515 			if ($found) {
    516 				write_cksum_file "$dir/$CKSUM_FILE", $cksums;
    517 				write_special_cksums $dir, $CKSUM_FILE;
    518 			}
    519 		}
    520 	}
    521 }
    522 
    523 # sort the given paths into hash based on the dirname
    524 # returns: a hash with the keys being the dirnames of the given paths and
    525 # each one pointing to an array containing the basenames of all paths
    526 # that had this dirname
    527 sub sort_by_dir {
    528 	my %sorted_files;
    529 	foreach my $file (@_) {
    530 		if (!-e $file) {
    531 			warn "ERROR: Source file \"$file\" doesn't exist.\n";
    532 			next;
    533 		}
    534 		my $dir = dirname($file);
    535 		if (!exists($sorted_files{$dir})) {
    536 			$sorted_files{$dir} = [];
    537 		}
    538 		push(@{$sorted_files{$dir}}, basename($file));
    539 	}
    540 	return \%sorted_files;
    541 }
    542 
    543 # check if $dst exists and prompt the user whether it should be overwritten
    544 # returns 0 if it can be overwritten or doesn't exist, 1 if it shouldn't be overwritten
    545 sub prompt_overwrite {
    546 	my $dst = shift;
    547 	if (-e $dst) {
    548 		print STDERR "WARNING: \"$dst\" exists already. Do you want to replace it? (y/n) ";
    549 		my $choice = "";
    550 		while ($choice ne "y" && $choice ne "n") {
    551 			$choice = <STDIN>;
    552 			chomp $choice;
    553 		}
    554 		if ($choice eq "n") {
    555 			warn "Not overwriting \"$dst\"\n";
    556 			return 1;
    557 		} else {
    558 			return 0;
    559 		}
    560 	}
    561 	return 0;
    562 }
    563 
    564 # copies the $src files to $dst and updates the checksums in $dst
    565 # $src: list of source paths
    566 # $dst: destination directory or file (in latter case only one src is allowed)
    567 sub copy_files {
    568 	my ($src, $dst, $args) = @_;
    569 	my $dst_dir = $dst;
    570 	if (!-d $dst) {
    571 		$dst_dir = dirname $dst;
    572 	}
    573 	my $diff_name = 0;
    574 	# check if the file/dir is getting a different name or
    575 	# just being copied into a different directory
    576 	if (!-d $dst && !-d $src->[0]) {
    577 		$diff_name = 1;
    578 	}
    579 	if (!-e $dst && -d $src->[0]) {
    580 		$diff_name = 1;
    581 	}
    582 	my $dst_cksums = read_cksums $dst_dir;
    583 	return if !defined $dst_cksums;
    584 	my $src_sorted = sort_by_dir(@$src);
    585 	my $files_touched = 0;
    586 	my $dirs_touched = 0;
    587 	foreach my $src_dir (keys %$src_sorted) {
    588 		my $src_cksums = read_cksums $src_dir;
    589 		next if !defined $src_cksums;
    590 		foreach my $src_file (@{$src_sorted->{$src_dir}}) {
    591 			my $src_path = "$src_dir/$src_file";
    592 
    593 			my $dst_path = $diff_name ? $dst : "$dst_dir/$src_file";
    594 			if (-d $dst_path && -d $src_path) {
    595 				warn "ERROR: Cannot copy directory to already existing directory\n";
    596 				next;
    597 			}
    598 			if (exists $SPECIAL_FILES{$src_file} || exists $SPECIAL_FILES{basename $dst_path}) {
    599 				warn "ERROR: Not copying special file\n";
    600 				next;
    601 			}
    602 			next if !$args->{"f"} && prompt_overwrite($dst_path);
    603 			my $options = $args->{"v"} ? "-av" : "-a";
    604 			next if system("cp", $options, "--", $src_path, $dst);
    605 
    606 			if (-d $src_path) {
    607 				$dirs_touched = 1;
    608 			} else {
    609 				$files_touched = 1;
    610 			}
    611 
    612 			if (exists $src_cksums->{$src_file}) {
    613 				if ($diff_name) {
    614 					$dst_cksums->{basename $dst} = $src_cksums->{$src_file};
    615 				} else {
    616 					$dst_cksums->{$src_file} = $src_cksums->{$src_file};
    617 				}
    618 			} else {
    619 				warn "WARNING: \"$src_path\" not in cksum or directory list\n";
    620 			}
    621 		}
    622 	}
    623 	write_cksums $dst_dir, $dst_cksums, $files_touched, $dirs_touched;
    624 }
    625 
    626 # move a file (or directory) from $src to $dst, prompting for confirmation if $dst already exists;
    627 # automatically appends the basename of $src to $dst if $dst is a directory
    628 sub move_file {
    629 	my ($src, $dst, $args) = @_;
    630 	if (exists $SPECIAL_FILES{basename $src} || exists $SPECIAL_FILES{basename $dst}) {
    631 		warn "ERROR: Not moving special file\n";
    632 		return 1;
    633 	}
    634 	if (-d $dst) {
    635 		$dst .= "/" . basename($src);
    636 	}
    637 	return 1 if !$args->{"f"} && prompt_overwrite($dst);
    638 	my $ret;
    639 	if ($args->{"v"}) {
    640 		$ret = system("mv", "-v", "--", $src, $dst);
    641 	} else {
    642 		$ret = system("mv", "--", $src, $dst);
    643 	}
    644 	return 1 if $ret;
    645 	if (-e $src) {
    646 		warn "ERROR: file could not be removed from source but will still be " .
    647 			"removed from checksum database\n";
    648 	}
    649 	return 0;
    650 }
    651 
    652 # move all files/directories in $src_files from $src_dir to $dst_dir ($src_files
    653 # only contains the basenames of the files), removing them from the checksum files
    654 # in $src_dir and adding them to $dst_cksums
    655 sub move_from_same_dir {
    656 	my ($src_dir, $src_files, $dst_cksums, $dst_dir, $args) = @_;
    657 	my $src_cksums = read_cksums $src_dir;
    658 	return if !defined $src_cksums;
    659 	my $files_touched = 0;
    660 	my $dirs_touched = 0;
    661 	foreach my $src_file (@$src_files) {
    662 		my $fullpath = "$src_dir/$src_file";
    663 		my $tmp_dirs_touched = 0;
    664 		my $tmp_files_touched = 0;
    665 		if (-d $fullpath) {
    666 			$tmp_dirs_touched = 1;
    667 		} else {
    668 			$tmp_files_touched = 1;
    669 		}
    670 
    671 		next if move_file($fullpath, $dst_dir, $args);
    672 
    673 		# need to be able to check if the path is a directory
    674 		# before actually moving it
    675 		$dirs_touched ||= $tmp_dirs_touched;
    676 		$files_touched ||= $tmp_files_touched;
    677 		if (exists $src_cksums->{$src_file}) {
    678 			$dst_cksums->{$src_file} = $src_cksums->{$src_file};
    679 			delete $src_cksums->{$src_file};
    680 		} else {
    681 			warn "WARNING: \"$src_dir/$src_file\" not in cksum or directory list.\n";
    682 		}
    683 	}
    684 	write_cksums $src_dir, $src_cksums, $files_touched, $dirs_touched;
    685 	return ($files_touched, $dirs_touched);
    686 }
    687 
    688 # rename a single file or directory from $src to $dst
    689 sub move_rename {
    690 	my ($src, $dst, $args) = @_;
    691 	my $src_dir = dirname $src;
    692 	my $dst_dir = dirname $dst;
    693 	my $src_file = basename $src;
    694 	my $dst_file = basename $dst;
    695 
    696 	my $src_cksums = read_cksums $src_dir;
    697 	return if !defined $src_cksums;
    698 	my $dst_cksums = {};
    699 	# if a file is simply being renamed in the same dir, the cksums
    700 	# should only be loaded and written once
    701 	if ($src_dir eq $dst_dir) {
    702 		%$dst_cksums = %$src_cksums;
    703 		delete $dst_cksums->{$src_file};
    704 	} else {
    705 		$dst_cksums = read_cksums $dst_dir;
    706 		return if !defined $dst_cksums;
    707 	}
    708 
    709 	my $files_touched = 0;
    710 	my $dirs_touched = 0;
    711 	if (-d $src) {
    712 		$dirs_touched = 1;
    713 	} else {
    714 		$files_touched = 1;
    715 	}
    716 
    717 	return if move_file($src, $dst, $args);
    718 
    719 	if (exists($src_cksums->{$src_file})) {
    720 		$dst_cksums->{$dst_file} = $src_cksums->{$src_file};
    721 		delete $src_cksums->{$src_file};
    722 	} else {
    723 		warn "WARNING: \"$src\" not in cksum or directory list.\n";
    724 	}
    725 	write_cksums $dst_dir, $dst_cksums, $files_touched, $dirs_touched;
    726 	if ($src_dir ne $dst_dir) {
    727 		write_cksums $src_dir, $src_cksums, $files_touched, $dirs_touched;
    728 	}
    729 }
    730 
    731 # move all files and directories in $src to $dst
    732 # - if $dst does not exist, $src is only allowed to contain one path, which is
    733 # renamed to $dst
    734 # - if $dst is a file, $src is only allowed to contain a single path (which
    735 # must be a file), which is renamed to $dst
    736 # - otherwise, all files and directories in $src are moved to $dst
    737 # $src: list of source paths
    738 # $dst: destination directory or file (in latter case only one src is allowed)
    739 sub move_files {
    740 	my ($src, $dst, $args) = @_;
    741 	if (!-d $dst && $#$src != 0) {
    742 		die "move: only one source argument allowed when destination is a file\n";
    743 	}
    744 	if (!-d $dst && !-d $src->[0]) {
    745 		move_rename $src->[0], $dst, $args;
    746 		return;
    747 	}
    748 	if (!-e $dst && -d $src->[0]) {
    749 		move_rename $src->[0], $dst, $args;
    750 		return;
    751 	}
    752 	if (-e $dst && !-d $dst && -d $src->[0]) {
    753 		die "move: can't move directory to file\n";
    754 	}
    755 	# Separate files by current dir so the cksum and dir files only need to be opened once
    756 	my $src_files = sort_by_dir(@$src);
    757 	my $dst_cksums = read_cksums $dst;
    758 	return if !defined $dst_cksums;
    759 	my $files_touched = 0;
    760 	my $dirs_touched = 0;
    761 	foreach my $src_dir (keys %$src_files) {
    762 		my ($tmp_files_touched, $tmp_dirs_touched) = move_from_same_dir $src_dir, $src_files->{$src_dir}, $dst_cksums, $dst, $args;
    763 		$files_touched ||= $tmp_files_touched;
    764 		$dirs_touched ||= $tmp_dirs_touched;
    765 	}
    766 	write_cksums $dst, $dst_cksums, $files_touched, $dirs_touched;
    767 }
    768 
    769 # remove a file or directory from the filesystem
    770 sub remove_file_dir {
    771 	my ($path, $args) = @_;
    772 	my $options = $args->{"f"} ? "-rf" : "-r";
    773 	if (system("rm", $options, "--", $path)) {
    774 		return 1;
    775 	}
    776 	if (-e $path) {
    777 		warn "ERROR: Unable to remove \"$path\" from filesystem but " .
    778 			"will still be removed from checksum database\n";
    779 	}
    780 	return 0;
    781 }
    782 
    783 # remove all files in one directory, updating the checksum files in the process
    784 # note: the files are only allowed to be basenames, i.e., they must be the
    785 # actual filenames present in the checksum files
    786 sub remove_from_same_dir {
    787 	my ($args, $dir, @files) = @_;
    788 	my $cksums = read_cksums $dir;
    789 	return if !defined $cksums;
    790 	my $dirs_touched = 0;
    791 	my $files_touched = 0;
    792 	foreach my $file (@files) {
    793 		if (exists $SPECIAL_FILES{$file}) {
    794 			warn "ERROR: not removing special file $file\n";
    795 			next;
    796 		}
    797 		my $fullpath = "$dir/$file";
    798 		if (!-e $fullpath) {
    799 			warn "\"$fullpath\": No such file or directory.\n";
    800 		}
    801 		next if remove_file_dir($fullpath, $args);
    802 		if (exists $cksums->{$file}) {
    803 			if (defined $cksums->{$file}) {
    804 				$files_touched = 1;
    805 			} else {
    806 				$dirs_touched = 1;
    807 			}
    808 			delete $cksums->{$file};
    809 		} else {
    810 			warn "WARNING: \"$file\" not in cksum or directory list.\n";
    811 		}
    812 	}
    813 	write_cksums $dir, $cksums, $files_touched, $dirs_touched;
    814 }
    815 
    816 # remove all given files and directories, updating the appropriate checksum
    817 # files in the process
    818 sub remove_files {
    819 	my $args = shift;
    820 	my $sorted_files = sort_by_dir(@_);
    821 	foreach my $dir (keys %$sorted_files) {
    822 		remove_from_same_dir($args, $dir, @{$sorted_files->{$dir}});
    823 	}
    824 }
    825 
    826 # create the given directories, initializing them with empty checksum files
    827 # note: does not work like "mkdir -p", i.e., the new directories have to
    828 # be located inside already existing directories
    829 sub make_dirs {
    830 	my @created_dirs;
    831 	foreach (@_) {
    832 		if (system("mkdir", "--", $_)) {
    833 			warn "ERROR creating directory $_\n";
    834 			next;
    835 		}
    836 		push(@created_dirs, $_);
    837 	}
    838 	# Separate files by current dir so the cksum and dir files only need to be opened once
    839 	my %dirs;
    840 	foreach my $dir (@created_dirs) {
    841 		write_cksums $dir, {}, 1, 1;
    842 		my $parent = dirname($dir);
    843 		if (!exists($dirs{$parent})) {
    844 			$dirs{$parent} = [];
    845 		}
    846 		push(@{$dirs{$parent}}, basename($dir));
    847 	}
    848 	foreach my $parent (keys %dirs) {
    849 		my $parent_dirs = read_file "$parent/$DIR_FILE", {};
    850 		next if !defined $parent_dirs;
    851 		foreach my $dir (@{$dirs{$parent}}) {
    852 			$parent_dirs->{$dir} = "";
    853 		}
    854 		write_file "$parent/$DIR_FILE", $parent_dirs;
    855 		write_special_cksums $parent, $DIR_FILE;
    856 	}
    857 }
    858 
    859 # extract all special lumia files from $src_dir to $dst_dir, recreating the
    860 # entire directory structure in the process
    861 sub extract {
    862 	my ($src_dir, $dst_dir, $args) = @_;
    863 	my $iter = make_lumia_iter 0, $src_dir;
    864 	my $options = $args->{"v"} ? "-av" : "-a";
    865 	while (my $dir = $iter->()) {
    866 		my $final_dir = abs2rel $dir, $src_dir;
    867 		my $fulldir = catfile $dst_dir, $final_dir;
    868 		system("mkdir", "-p", "--", $fulldir);
    869 		foreach my $file (keys %SPECIAL_FILES) {
    870 			my $filepath = catfile $dir, $file;
    871 			if (-e $filepath) {
    872 				system("cp", $options, "--", $filepath, catfile($fulldir, $file));
    873 			}
    874 		}
    875 	}
    876 }
    877 
    878 # update the checksums of the given files
    879 # ignores any directories given as arguments
    880 sub update {
    881 	my @files;
    882 	foreach (@_) {
    883 		if (-d $_) {
    884 			warn "Ignoring directory \"$_\"\n";
    885 		} else {
    886 			push @files, $_;
    887 		}
    888 	}
    889 	my $sorted_files = sort_by_dir @files;
    890 	foreach my $dir (keys %$sorted_files) {
    891 		my $cksums = read_cksum_file "$dir/$CKSUM_FILE", {};
    892 		next if !defined $cksums;
    893 		my $changed = 0;
    894 		foreach my $file (@{$sorted_files->{$dir}}) {
    895 			my $cksum_output = get_cksum "$dir/$file";
    896 			next if !defined $cksum_output;
    897 			$cksums->{$file} = $cksum_output;
    898 			$changed = 1;
    899 		}
    900 		if ($changed) {
    901 			write_cksum_file "$dir/$CKSUM_FILE", $cksums;
    902 			write_special_cksums $dir, $CKSUM_FILE;
    903 		}
    904 	}
    905 }
    906 
    907 sub update_special {
    908 	my $dir = shift;
    909 	write_special_cksums $dir, $CKSUM_FILE, $DIR_FILE;
    910 }
    911 
    912 my %args;
    913 Getopt::Long::Configure("bundling");
    914 GetOptions(\%args, "f|force", "q|quiet", "v|verbose", "h|help");
    915 
    916 pod2usage(-exitval => 0, -verbose => 2) if $args{"h"};
    917 pod2usage(-exitval => 1, -verbose => 1) if @ARGV < 1;
    918 
    919 my $cmd = shift;
    920 
    921 if ($cmd eq "mv") {
    922 	die "mv requires at least two arguments\n" if @ARGV < 2;
    923 	my @src = @ARGV[0..$#ARGV-1];
    924 	move_files \@src, $ARGV[-1], \%args;
    925 } elsif ($cmd eq "rm") {
    926 	die "rm requires at least one argument\n" if @ARGV < 1;
    927 	remove_files \%args, @ARGV;
    928 } elsif ($cmd eq "addnew") {
    929 	my $dir = @ARGV ? $ARGV[0] : ".";
    930 	check_add_new_files $dir, \%args;
    931 } elsif ($cmd eq "checknew") {
    932 	my $dir = @ARGV ? $ARGV[0] : ".";
    933 	check_new_files $dir;
    934 } elsif ($cmd eq "checkold") {
    935 	my $dir = @ARGV ? $ARGV[0] : ".";
    936 	check_old_files $dir;
    937 } elsif ($cmd eq "rmold") {
    938 	my $dir = @ARGV ? $ARGV[0] : ".";
    939 	remove_old_files $dir, \%args;
    940 } elsif ($cmd eq "check") {
    941 	if (@ARGV < 1) {
    942 		check_files \%args, ".";
    943 	} else {
    944 		check_files \%args, @ARGV;
    945 	}
    946 } elsif ($cmd eq "clean") {
    947 	my $dir = @ARGV ? $ARGV[0] : ".";
    948 	clean_files $dir, \%args;
    949 } elsif ($cmd eq "extract") {
    950 	my $src_dir = ".";
    951 	my $dst_dir;
    952 	if (@ARGV == 2) {
    953 		$src_dir = $ARGV[0];
    954 		$dst_dir = $ARGV[1];
    955 	} elsif (@ARGV == 1) {
    956 		$dst_dir = $ARGV[0];	
    957 	} else {
    958 		die "Invalid number of arguments\n";
    959 	}
    960 	if (!-d $src_dir) {
    961 		die "ERROR: Directory \"$src_dir\" does not exist.\n";
    962 	}
    963 	if (!-d $dst_dir) {
    964 		die "ERROR: Directory \"$dst_dir\" does not exist.\n";
    965 	}
    966 	extract $src_dir, $dst_dir;
    967 } elsif ($cmd eq "cp") {
    968 	die "cp requires at least two arguments\n" if @ARGV < 2;
    969 	my @src = @ARGV[0..$#ARGV-1];
    970 	copy_files \@src, $ARGV[-1], \%args;
    971 } elsif ($cmd eq "mkdir") {
    972 	die "mkdir requires at least one argument\n" if @ARGV < 1;
    973 	make_dirs @ARGV;
    974 } elsif ($cmd eq "update") {
    975 	die "update requires at least one argument\n" if @ARGV < 1;
    976 	update @ARGV;
    977 } elsif ($cmd eq "updatespecial") {
    978 	die "Invalid number of arguments\n" if @ARGV > 1;
    979 	my $dir = @ARGV ? $ARGV[0] : ".";
    980 	update_special $dir;
    981 } else {
    982 	pod2usage(-exitval => 1, -verbose => 1);
    983 }
    984 
    985 __END__
    986 
    987 =head1 NAME
    988 
    989 lumia.pl - Manage checksums on a filesystem
    990 
    991 =head1 SYNOPSIS
    992 
    993 B<lumia.pl> command [-hqfv] arguments
    994 
    995 =head1 OPTIONS
    996 
    997 =over 8
    998 
    999 =item B<-h>, B<--help>
   1000 
   1001 Show the full documentation.
   1002 
   1003 =item B<-q>, B<--quiet>
   1004 
   1005 Only output errors.
   1006 
   1007 =item B<-f>, B<--force>
   1008 
   1009 Overwrite files without prompting for confirmation.
   1010 
   1011 =item B<-v>, B<--verbose>
   1012 
   1013 Print each file that is processed by the command.
   1014 
   1015 =back
   1016 
   1017 See the full documentation for details on which commands support which options
   1018 and what they do.
   1019 
   1020 It does not matter if the options are written before or after the command.
   1021 
   1022 If C<--> is written anywhere on the command line, option parsing is stopped,
   1023 so that files starting with a hyphen can still be specified.
   1024 
   1025 Note that C<-q> and C<-v> aren't exactly opposites - C<-q> applies to commands
   1026 like B<check>, where it suppresses printing of the individual files, while
   1027 C<-v> applies to commands like B<cp>, where it is just passed on to the system
   1028 command called in the background.
   1029 
   1030 Note further that this is very inconsistent, like the rest of the program, but
   1031 the author has made too many bad decisions to rectify that problem at the moment.
   1032 
   1033 =head1 COMMANDS
   1034 
   1035 Note that some commands support multiple files/directories as arguments and others,
   1036 for which it would make just as much sense, don't. That's just the way it is.
   1037 
   1038 =over 8
   1039 
   1040 =item B<addnew> [-q] [directory]
   1041 
   1042 Walks through B<directory>, adding all new files to the checksum database.
   1043 B<directory> defaults to the current directory.
   1044 
   1045 C<-q> suppresses the printing of each file or directory as it is added.
   1046 
   1047 =item B<checknew> [directory]
   1048 
   1049 Walks through B<directory>, printing all files that aren't part of the checksum
   1050 database. B<directory> defaults to the current directory.
   1051 
   1052 =item B<checkold> [directory]
   1053 
   1054 Prints all files in the checksum database that do not exist on the filesystem anymore.
   1055 B<directory> defaults to the current directory.
   1056 
   1057 =item B<rmold> [-q] [directory]
   1058 
   1059 Removes all files found by B<checkold> from the database. B<directory> defaults to
   1060 the current directory.
   1061 
   1062 C<-q> suppresses the printing of each file as it is removed.
   1063 
   1064 =item B<check> [-q] file/directory ...
   1065 
   1066 Verifies the checksums of all files given, recursing through any directories. If no
   1067 files or directories are given, the current directory is used.
   1068 
   1069 Note that the checksum database in the corresponding directory will be read again for
   1070 every file given on the command line, even if 1000 files in the same directory are given.
   1071 This problem does not occur when recursing through directories, so it is best to only
   1072 give files directly when checking a few. This problem wouldn't be too difficult to
   1073 fix, but, frankly, I'm too lazy, especially since I only added the feature to check
   1074 files individually as a convenience when I want to quickly check a single file in a
   1075 large directory.
   1076 
   1077 To explain why it is this way: The directory recursion is done using an iterator, which
   1078 has the directories pushed onto its queue in the beginning. The iterator only returns
   1079 directories, which are then checked all in one go, but this means that files given on
   1080 the command line need to be handled specially.
   1081 
   1082 C<-q> suppresses the printing of all good checksums but still allows a message to
   1083 be printed when a checksum failed.
   1084 
   1085 =item B<clean> [-q] [directory]
   1086 
   1087 Removes all lumia special files used to store the checksum database from B<directory>
   1088 recursively. B<directory> defaults to the current directory.
   1089 
   1090 Note that this recurses through the entire directory tree, not just the part that is
   1091 actually linked together by the checksum database.
   1092 
   1093 Warning: This just blindly removes all files with one of the special lumia names,
   1094 even if they weren't actually created by lumia.pl.
   1095 
   1096 C<-q> suppresses the printing of each file as it is deleted.
   1097 
   1098 =item B<extract> [-v] [source] destination
   1099 
   1100 Recreates the entire directory structure from B<source> in B<destination>, but only
   1101 copies the special files used to store the checksum database. B<source> defaults to
   1102 the current directory.
   1103 
   1104 C<-v> prints each file as it is copied.
   1105 
   1106 Note that this overwrites files in the destination directory without confirmation.
   1107 
   1108 =item B<mkdir> directory ...
   1109 
   1110 Creates the given directories, initializing them with empty checksum database files.
   1111 
   1112 =item B<update> file ...
   1113 
   1114 Recalculates the checksums for the given files and replaces them in the database.
   1115 
   1116 Note: Directories given as arguments are ignored.
   1117 
   1118 This is mainly meant to quickly "touch" a file after it was modified (e.g. a
   1119 notes file that is occasionally updated).
   1120 
   1121 =item B<updatespecial> [directory]
   1122 
   1123 Recalculates the checksums for the special files C<.lumidify_archive_dirs> and
   1124 C<.lumidify_archive_cksums> and writes them to C<.lumidify_archive_cksums.cksum>.
   1125 B<directory> defaults to the current directory.
   1126 
   1127 This is only meant to be used if, for some reason, the checksum files had to
   1128 be edited manually and thus don't match the checksums in C<.lumidify_archive_cksums.cksum>
   1129 anymore.
   1130 
   1131 =item B<rm> [-f] file ...
   1132 
   1133 Removes the given files and directories recursively from the filesystem and
   1134 checksum database. The following caveats apply:
   1135 
   1136 If any actual errors occur while deleting the file/directory (i.e. the system
   1137 command C<rm> returns a non-zero exit value), the checksum or directory B<is
   1138 left in the database>. If the system C<rm> does not return a non-zero exit value,
   1139 but the file/directory still exists afterwards (e.g. there was a permission
   1140 error and the user answered "n" when prompted), a warning message is printed,
   1141 but the files B<are removed from the database> (if the database can be
   1142 written to).
   1143 
   1144 It is an error if there are no checksum database files in the directory
   1145 of a file named on the command line.
   1146 
   1147 C<-f> is passed through to the system C<rm> command.
   1148 
   1149 =item B<cp> [-vf] source target
   1150 
   1151 =item B<cp> [-vf] source ... directory
   1152 
   1153 Copies the given source files, updating the checksum database in the process.
   1154 
   1155 If the last argument is a file, there must be only one source argument, also a file,
   1156 which is then copied to the target.
   1157 
   1158 If the last argument is a directory, all source arguments are copied into it.
   1159 
   1160 It is an error if a source or destination directory does not contain any
   1161 checksum database files.
   1162 
   1163 B<cp> will issue a warning and skip to the next argument if it is asked to
   1164 merge a directory with an already existing directory. For instance, attempting
   1165 to run C<cp dir1 dir2>, where C<dir2> already contains a directory named
   1166 C<dir1>, will result in an error. This may change in the future, when the
   1167 program is modified to recursively copy the files manually, instead of simply
   1168 calling the system C<cp> on each of the arguments. If this was supported in
   1169 the current version, none of the checksums inside that directory would be
   1170 updated, so it wouldn't be very useful.
   1171 
   1172 C<-v> is passed through to the system C<cp> command.
   1173 
   1174 C<-f> silently overwrites files without prompting the user, much like the
   1175 C<-f> option in the system C<cp> command. This is handled manually by the
   1176 program, though, in order to actually determine what the user chose. See
   1177 also the caveat mentioned above.
   1178 
   1179 =item B<mv> [-f] source target
   1180 
   1181 =item B<mv> [-f] source ... directory
   1182 
   1183 Moves the given source files, updating the checksum database in the process.
   1184 
   1185 If the last argument is a file or does not exist, there must be only one source
   1186 argument, which is renamed to the target name.
   1187 
   1188 If the last argument is an existing directory, all source arguments are moved
   1189 into it.
   1190 
   1191 It is an error if a source or destination directory does not contain any
   1192 checksum database files.
   1193 
   1194 B<mv> behaves the same as B<rm> with regards to checking if the source file
   1195 is still present after the operation and other error handling.
   1196 
   1197 C<-f> is handled in the same manner as with B<cp>.
   1198 
   1199 =back
   1200 
   1201 =head1 DESCRIPTION
   1202 
   1203 lumia.pl is meant for managing checksums of files in order to prevent bitrot.
   1204 It does this by storing several special files in each directory to keep track
   1205 of the checksums:
   1206 
   1207 =over 8
   1208 
   1209 =item B<.lumidify_archive_cksums>
   1210 
   1211 Contains the checksums of all files in the directory.
   1212 
   1213 =item B<.lumidify_archive_dirs>
   1214 
   1215 Contains a list of all directories in the directory.
   1216 
   1217 =item B<.lumidify_archive_cksums.cksum>
   1218 
   1219 Contains the checksums of B<.lumidify_archive_cksums> and B<.lumidify_archive_dirs>
   1220 in order to provide two-layer protection against bitrot.
   1221 
   1222 =item B<.lumidify_archive_ignore>
   1223 
   1224 Contains a list of files and directories that should be ignored by lumia.pl.
   1225 Note that this is only read and never written to, unless the command B<clean>
   1226 is used. It is, however, still copied over by the B<extract> command.
   1227 
   1228 =back
   1229 
   1230 When the documentation for the commands talks about the "checksum database",
   1231 it simply means these files.
   1232 
   1233 All file/directory names are enclosed in quotes, with any backslashes or quotes
   1234 inside the name escaped with another backslash. The names are allowed to have
   1235 newlines in them.
   1236 
   1237 The list files only contain a list of filenames, with a newline between the
   1238 closing quote of one name and the opening quote of the next one.
   1239 
   1240 The checksum files additionally contain the output of the checksum program
   1241 used and a space before the starting quote of the filename.
   1242 
   1243 =head1 MOTIVATION
   1244 
   1245 There are already several programs that can be used to check for bitrot,
   1246 as listed in L</"SEE ALSO">. However, all programs I tried either were
   1247 much too complicated for my taste or just did everything behind my back.
   1248 I wanted a simple tool that did exactly what I told it to and also allowed
   1249 me to keep the old checksums when reorganizing files, in order to avoid
   1250 regenerating the checksums from corrupt files. Since I couldn't find those
   1251 features in any program I tried, I wrote my own.
   1252 
   1253 =head1 DESIGN DECISIONS
   1254 
   1255 It may strike some readers as a peculiar idea to save the checksum files in
   1256 I<every single directory>, but this choice was made after much deliberation.
   1257 The other option I could think of was to have one big database, but that
   1258 would have made all commands much more difficult to implement and additionally
   1259 necessitated opening the entire database for every operation. With individual
   1260 files in each directory, operations like B<cp> become quite trivial (ignoring
   1261 all the edge cases) since only the toplevel checksums need to be copied to
   1262 the new destination, and any subdirectories already contain the checksums.
   1263 
   1264 This method is not without its drawbacks, however. The most glaring problem
   1265 I have found is that there is no way to store the checksums of read-only
   1266 directories or any special directories that cannot be littered with the
   1267 checksum files because that would clash with other software. Despite these
   1268 drawbacks, however, I decided to stick with it because it works for almost
   1269 all cases and doesn't have any of the serious drawbacks that other options
   1270 would have had.
   1271 
   1272 The names of the special files were chosen to be ".lumidify_archive*" not
   1273 out of vanity, but mainly because I couldn't think of any regular files
   1274 with those names, making them a good choice to avoid clashes.
   1275 
   1276 The name of the program, C<lumia.pl> (for "lumidify archive"), was similarly
   1277 chosen because it did not clash with any programs installed on my system and
   1278 thus allowed for easy tab-completion.
   1279 
   1280 =head1 HASH ALGORITHMS
   1281 
   1282 By default, the simple cksum algorithm is used to get the checksums. This
   1283 is not very secure, but the main purpose of the program is to prevent
   1284 bitrot, for which cksum should be sufficient, especially since it is much
   1285 faster than other algorithms.
   1286 
   1287 There is currently no convenient way to change the algorithm other than
   1288 changing the $CKSUM_CMD and $CKSUM_NUMFIELDS variables at the top of
   1289 lumia.pl. $CKSUM_CMD must be the command that returns the checksum
   1290 when it is given a file, and $CKSUM_NUMFIELDS specifies the number of
   1291 space-separated fields the checksum consists of. This has to be specified
   1292 in order to determine where the checksum ends and the filename begins in
   1293 the output. This would be redundant if all implementations of cksum
   1294 supported '-q' for outputting only the checksum, but that only seems to
   1295 be supported by some implementations.
   1296 
   1297 =head1 USAGE SCENARIOS
   1298 
   1299 =over 8
   1300 
   1301 =item B<Security auditing>
   1302 
   1303 This program is B<NOT> designed to provide any security auditing, as should
   1304 be clear from the fact that the checksums are stored right in the same
   1305 directory as the files. See mtree(8) for that.
   1306 
   1307 If you want to, however, you could set $CKSUM_CMD to a secure hash (not cksum)
   1308 and B<extract> the checksums to a separate directory, which you keep in a
   1309 safe place. You could then use the regular C<cp> command to simply replace
   1310 all the checksums with the ones from your backup, in case an attacker modified
   1311 the checksum database in the directory with the actual files you're trying to
   1312 protect. I don't know if there would be any point in doing that, though.
   1313 
   1314 =item B<Managing archives>
   1315 
   1316 This is the purpose I wrote the program for.
   1317 
   1318 You can simply initialize your archive directory with the B<addnew> command.
   1319 Whenever you add new files, just run B<addnew> again. If you want to reorganize
   1320 the archive, you can use the limited commands available.
   1321 
   1322 I usually just use rsync(1) to copy the entire archive directory over to other
   1323 backup drives and then use the B<check> command again on the new drive.
   1324 
   1325 I also have checksums for the main data directory on my computer (except for
   1326 things like git repositories, which I don't want littered with the database
   1327 files). Here, I use the B<update> command for files that I edit more often
   1328 and occasionally run B<check> on the entire directory.
   1329 
   1330 Since the database files are written in each directory, you can run the
   1331 B<addnew> command in any subdirectory when you've added new files there.
   1332 
   1333 =back
   1334 
   1335 =head1 PERFORMANCE
   1336 
   1337 Due to the extensive use of iterators and the author's bad life choices,
   1338 some functions, such as B<addnew> and B<check>, run more slowly than they
   1339 would if they were programmed more efficiently, especially on many small
   1340 files and folders. Too bad.
   1341 
   1342 =head1 PORTABILITY
   1343 
   1344 This program was written on OpenBSD. It will probably work on most other
   1345 reasonably POSIX-Compliant systems, although I cannot guarantee anything.
   1346 $CKSUM_CMD may need to be modified at the top of lumia.pl. The file
   1347 operation commands are called directly with system(), so those need to
   1348 be available.
   1349 
   1350 It will most certainly not work on Windows, but that shouldn't be a
   1351 problem for anyone important.
   1352 
   1353 =head1 BUGS
   1354 
   1355 All system commands (unless I forgot some) are called with "--" before
   1356 listing the actual files, so files beginning with hyphens should be
   1357 supported. I have tested the commands with filenames starting with spaces
   1358 and hyphens and also containing newlines, but there may very well be issues
   1359 still. Please notify me if you find any filenames that do not work. Handling
   1360 filenames properly is difficult.
   1361 
   1362 There are probably many other edge cases, especially in the B<mv>, B<cp>,
   1363 and B<rm> commands. Please notify me if you find an issue.
   1364 
   1365 Operations on files containing newlines may cause Perl to print a warning
   1366 "Unsuccessful stat on filename containing newline" even though nothing is
   1367 wrong since (as described in B<mv> and B<rm>) existence of the file is
   1368 checked afterwards. I didn't feel like disabling warnings, and no normal
   1369 person should be working with files containing newlines anyways, so that's
   1370 the way it is.
   1371 
   1372 =head1 EXIT STATUS
   1373 
   1374 Always 0, unless the arguments given were invalid. We don't do errors around here.
   1375 
   1376 On a more serious note - I should probably change that at some point.
   1377 For the time being, if you want to run B<check> in a script, you can test
   1378 the output printed when the C<-q> option is used, since this won't output
   1379 anything if there are no errors. Do note, though, that actual errors (file not
   1380 found, etc.) are printed to STDERR, while incorrect checksums are printed
   1381 to STDOUT.
   1382 
   1383 =head1 SEE ALSO
   1384 
   1385 par2(1), mtree(8), aide(1), bitrot(no man page)
   1386 
   1387 =head1 LICENSE
   1388 
   1389 Copyright (c) 2019, 2020, 2021 lumidify <nobody[at]lumidify.org>
   1390 
   1391 Permission to use, copy, modify, and/or distribute this software for any
   1392 purpose with or without fee is hereby granted, provided that the above
   1393 copyright notice and this permission notice appear in all copies.
   1394 
   1395 THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   1396 WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
   1397 MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
   1398 ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
   1399 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
   1400 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
   1401 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
   1402 
   1403 =cut