Alexander/threadwatcher
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

cleanup, more realiable firewall and connection handling

Browse Source
This commit is contained in:
Alexander Bocken 2021-06-16 10:48:08 +02:00
parent ab9ff3b29e
commit 7623b91a31
Signed by: Alexander
GPG Key ID: 1D237BE83F9B05E8
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
threadwatcher
View File

@ -45,14 +45,15 @@ scan(){
done done
#Create lock file to stop override of URLFILE while scanning #Create lock file to stop override of URLFILE while scanning
touch /tmp/threadwatcher.lock touch /tmp/threadwatcher.lock
ping -q -c 1 1.1.1.1 > /dev/null || ping -q -c 1 1.0.0.1 > /dev/null || ping -q -c 1 4channel.org || { echo "No internet connection detected."; exit ;} ping -q -c 1 4channel.org || { echo "Cannot connect to 4chan."; exit 1;}
if [ $(wc -l < "$URLFILE") -gt 0 ]; then if [ "$(wc -l < "$URLFILE")" -gt 0 ]; then
echo "scanning threads..." echo "scanning threads..."
else else
echo "no threads to watch over currently" echo "no threads to watch over currently"
exit exit
fi fi
#tac used to prioritze newly added threads.
tac "$URLFILE" | while read -r line; do tac "$URLFILE" | while read -r line; do
running_dls=0 running_dls=0
url="$(echo "$line" | cut -f1)" url="$(echo "$line" | cut -f1)"
@ -63,14 +64,14 @@ scan(){
thread_title="$(jq '.posts[0].sub' < /tmp/content$$ | tr -d '"')" thread_title="$(jq '.posts[0].sub' < /tmp/content$$ | tr -d '"')"
echo "$url ($thread_title) $(echo "$dl_location" | sed "s|$HOME|~|")" echo "$url ($thread_title) $(echo "$dl_location" | sed "s|$HOME|~|")"
if [ -z "$(</tmp/content$$)" ]; then if [ -z "$(</tmp/content$$)" ]; then
#check for internet again, just in case #check for internet again, in case something has changed during the download process
if ping -q -c 1 1.1.1.1 > /dev/null || ping -q -c 1 1.0.0.1 > /dev/null || ping -q -c 1 example.org if ping -q -c 1 4channel.org >/dev/null
then then
echo "Thread $url not found ($dl_location) deleting from cached list of threads to watch" echo "Thread $url not found ($dl_location) deleting from cached list of threads to watch"
notify-send "threadwatcher" "Thread downloading to $(echo "$dl_location" | sed "s|$HOME|~|") is complete now." notify-send "threadwatcher" "Thread downloading to $(echo "$dl_location" | sed "s|$HOME|~|") is complete now."
else else
echo "No internet connection detected." echo "Cannot connect to 4chan."
exit exit 1
fi fi
continue continue
else else
@ -81,10 +82,11 @@ scan(){
rm /tmp/content$$ rm /tmp/content$$
#pastes together a multiline var using process substitution with #pastes together a multiline var using process substitution with
#layout: filename location md5 #layout: filename location md5
#only reason for bash here with process substitution #only real reason for bash here with process substitution
#(gets messy with lots of temp files otherwise)
files="$(paste <(paste <(echo "$files_json" | jq '.filename' | tr -d '"') <(echo "$files_json" | jq '.location' | tr -d '"')) <(echo "$files_json" | jq '.md5' | tr -d '"'))" files="$(paste <(paste <(echo "$files_json" | jq '.filename' | tr -d '"') <(echo "$files_json" | jq '.location' | tr -d '"')) <(echo "$files_json" | jq '.md5' | tr -d '"'))"
echo "$files" | while read -r file_line; do echo "$files" | while read -r file_line; do
#better cleanup like in booksplitter? #TODO: better cleanup like in booksplitter?
filename="$(echo "$file_line" | cut -f1 | tr ' ' '_')" filename="$(echo "$file_line" | cut -f1 | tr ' ' '_')"
master_location="$(echo "$file_line" | cut -f2 | tr -d '"')" master_location="$(echo "$file_line" | cut -f2 | tr -d '"')"
filelocation="$dl_location/$filename" filelocation="$dl_location/$filename"
@ -99,8 +101,6 @@ scan(){
rm "$filelocation" rm "$filelocation"
echo "removed $filename because of incorrect checksum, redownloading." echo "removed $filename because of incorrect checksum, redownloading."
fi fi
[ -f "$filelocation" ] ||
touch "$filelocation" #to keep atime order correct?
#limit concurrent dls #limit concurrent dls
if [ $running_dls -gt 25 ]; then if [ $running_dls -gt 25 ]; then
wait wait