Thursday, December 3, 2009

Remove duplicate files on a Unix system

#! /bin/sh -e

if [ "$#" = "0" ]; then WHERE=.; else WHERE=$1; fi
if [ "`uname`" = "Darwin" ]; then MD5='md5 -r'; else MD5=md5sum; fi

prev=""

find $WHERE -type f -exec $MD5 '{}' ';' | sort |
(
    IFS='\n'
    while read a
    do
        b=`echo "$a" | cut -c 1-32`
        c=`echo "$a" | cut -c 34-`
        if [ "$b" = "$prev" ];
        then
            rm -v "$c"
        fi
        prev="$b"
    done
)

No comments: