Alexander/threadwatcher
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

added deduping feature

Browse Source
This commit is contained in:
Alexander Bocken 2021-08-15 14:31:48 +02:00
parent 0ba9ae5b38
commit 06befea402
Signed by: Alexander
GPG Key ID: 1D237BE83F9B05E8
1 changed files with 33 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
threadwatcher
View File

@ -25,6 +25,9 @@ list lists all currently watched URLs and where they are downloading to
edit open threads file in \$EDITOR/vim to manually edit. edit open threads file in \$EDITOR/vim to manually edit.
clean deletes threads file. This will not delete already downloaded material. clean deletes threads file. This will not delete already downloaded material.
prune manually prune list of threads. Deletes all finished threads from list. prune manually prune list of threads. Deletes all finished threads from list.
dedupe [DIR]
remove all duplicate files from current download directories. (Comparing sha512 sums)
If no DIR is given, will check all currently downloading directories.
help display this help and exit." help display this help and exit."
#included personal prompt script here as function for portability. #included personal prompt script here as function for portability.
@ -188,6 +191,34 @@ prune(){
cleanup cleanup
} }
dedupe(){
if [ -n "$1" ]; then
dirs="$1"
echo "Deduping $1 manually..."
else
if [ "$(wc -l < "$URLFILE")" -gt 0 ]; then
echo "Deduping download directories..."
else
echo "No download directories to dedupe."
cleanup && exit
fi
#tac used to prioritze newly added threads.
dirs="$( tac "$URLFILE" | cut -f2 | awk '!seen[$0]++' )"
fi
echo "$dirs" | while read -r dir; do
echo "Generating checksums for $dir ..."
duplicates="$( sha512sum "$dir"/* | sort | awk 'seen[$1]++' )"
if [ -n "$duplicates" ]; then
echo "Deleting duplicates..."
echo "$duplicates" | cut -d' ' -f1,2 --complement |
while read -r file; do
rm -v "$file"
done
echo "$(echo "$duplicates" | wc -l) duplicate files have been deleted in $dir"
fi
done
}
add() { add() {
dl_location="$(makepathabsolute "$2")" dl_location="$(makepathabsolute "$2")"
if grep -qP "^$1\t" "$URLFILE"; then if grep -qP "^$1\t" "$URLFILE"; then
@ -197,8 +228,7 @@ add() {
if [ "$dl_location" != "$dl_location_already" ]; then if [ "$dl_location" != "$dl_location_already" ]; then
prompt "Do you want to change download directory to $2?" && prompt "Do you want to change download directory to $2?" &&
new_location="$dl_location" || new_location="$dl_location"
exit 0
[ -z "$new_location" ] && exit [ -z "$new_location" ] && exit
# Wait for last scan to finish in case of quick successive additions. # Wait for last scan to finish in case of quick successive additions.
# Otherwise there is a potential loss of threads # Otherwise there is a potential loss of threads
@ -274,6 +304,7 @@ case "$1" in
pgrep threadwatcher | grep -v "^$$$" | xargs -r kill pgrep threadwatcher | grep -v "^$$$" | xargs -r kill
${EDITOR:-vim} "$URLFILE";; ${EDITOR:-vim} "$URLFILE";;
"prune") prune;; "prune") prune;;
"dedupe") dedupe "$2";;
"help") echo "$help";; "help") echo "$help";;
*)echo "Incorrect usage. Correct usage:" *)echo "Incorrect usage. Correct usage:"
echo "$help" && exit 1;; echo "$help" && exit 1;;