#!/bin/bash #A script that interacts with 4chans API to checks for media to download out of threads. #It uses the file name used by the uploader. #consider using it in a cronjob intermittently with something like #*/10 * * * * /home//.local/bin/threadwatcher scan THREADWATCHER_DIR=${XDG_DATA_HOME:-$HOME/.local/share}/4chan_watcher URLFILE="$THREADWATCHER_DIR/threads" TMP_URLFILE=/tmp/4chan_thread_watcher_tmp$$ [ -d "$THREADWATCHER_DIR" ] || mkdir -p "$THREADWATCHER_DIR" [ -f "$URLFILE" ] || touch "$URLFILE" ping -q -c 1 1.1.1.1 > /dev/null || ping -q -c 1 1.0.0.1 > /dev/null || ping -q -c 1 example.org || { echo "No internet connection detected."; exit ;} scan(){ if [ $(wc -l "$URLFILE"| cut -d ' ' -f1) -gt 0 ]; then echo "scanning threads..." else echo "no threads to watch over currently" exit fi while read -r line; do running_dls=0 url="$(echo "$line" | cut -f1)" echo "scanning $url" dl_location="$(echo "$line" | cut -f2)" mkdir -p "$dl_location" echo "downloading to $dl_location" json_url="$(echo "$url" | sed -E 's/boards\.(4chan|4channel)/a.4cdn/; s/$/.json/')" curl -s -L "$json_url" | jq . > /tmp/content$$ if [ -z "$(> "$TMP_URLFILE" mkdir -p "$dl_location" fi files_json="$(jq '.posts[] | if has("filename") then {filename: "\(.no)_\(.filename)\(.ext)", location: "\(.tim)\(.ext)", md5: .md5} else null end ' < /tmp/content$$ | grep -vE '^null$')" #pastes together a multiline var using process substitution with layout: filename location md5 files="$(paste <(paste <(echo "$files_json" | jq '.filename' | tr -d '"') <(echo "$files_json" | jq '.location' | tr -d '"')) <(echo "$files_json" | jq '.md5' | tr -d '"'))" echo "$files" | while read -r file_line; do filename="$(echo "$file_line" | cut -f1)" master_location="$(echo "$file_line" | cut -f2 | tr -d '"')" filelocation="$dl_location/$filename" correct_md5="$(echo "$file_line" | cut -f3)" [ -f "$filelocation" ] && slave_md5="$(openssl dgst -md5 -binary "$filelocation" | openssl enc -base64)" board="$(echo "$url" | cut -d '/' -f4)" file_url="https://i.4cdn.org/$board/$master_location" if [ -f "$filelocation" ] && [ "$correct_md5" = "$slave_md5" ]; then true else if [ "$correct_md5" != "$slave_md5" ] && [ -f "$filelocation" ]; then rm "$filelocation" echo "removed $filename because of incorrect checksum, redownloading." fi [ -f "$filelocation" ] || touch "$filelocation" #to keep atime order correct? #limit concurrent dls if [ $running_dls -gt 25 ]; then wait running_dls=0 fi wget -q -O "$filelocation" "$file_url" & echo downloading "$filelocation" ((running_dls=running_dls+1)) fi done wait done<"$URLFILE" mv "$TMP_URLFILE" "$URLFILE" } case "$1" in "add") if echo "$3" | grep -qE '^/'; then dl_location="$3" else dl_location="$HOME/$3" fi if grep -qP "^$2\t" "$URLFILE"; then dl_location_already="$(grep -P "^$2\t" "$URLFILE" | cut -f2)" notify-send "threadwatcher" "Thread already being watched. currently downloads to $dl_location_already" choice="$(dmenuinput "If you want to change location of thread to download, please insert new directory now:")" if echo "$choice" | grep -qE '^/'; then new_location="$3" else new_location="$HOME/$3" fi [ -z "$new_location" ] && exit sed -i "s|$dl_location_already|$new_location|" "$URLFILE" mv $dl_location_already/* "$new_location" notify-send "threadwatcher" "already downloaded files moved to $new_location. New files will also be downloaded there" else printf "%s\t%s\n" "$2" "$dl_location" | tee -ai "$URLFILE" echo "added $2 to threadwatcher list. Downloading to $dl_location" fi scan;; "scan") scan;; "list") printf "Thread:\t\t\t\t\t\tDownload location:\n" sed "s|$HOME|~|" "$URLFILE";; "clean") echo "Watchlist used up to now:" cat "$URLFILE" echo "Deleting..." rm "$URLFILE" touch "$URLFILE";; "edit") ${EDITOR:-vim} "$URLFILE";; *)printf "Incorrect usage.\n\tthreadwatcher [add URL DL_LOCATION] [list] [edit] [clean]\n"; exit 1;; esac