Skip to content

Commit

Permalink
code cleanup, status output improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@shadoxxhd
shadoxxhd committed Sep 1, 2024
1 parent 0c10d43 commit e887c50
Show file tree
Hide file tree
Showing 2 changed files with 64 additions and 63 deletions.
Binary file modified dist/downloader.exe
View file
Binary file not shown.
127 changes: 64 additions & 63 deletions downloader.pyw
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import subprocess
from winpty import PTY, PtyProcess # PTY, WinptyError # real time communication
from winpty import PTY, PtyProcess, WinptyError # PTY, WinptyError # real time communication
import tkinter as tk
from tkinter import ttk
from tkinter import messagebox
Expand Down Expand Up @@ -84,7 +84,7 @@ class Options:
theme: str = "default"
steampath: str = "steamcmd"
defaultpath: str = "mods"
batchsize: int = 50
batchsize: int = 500
# login
login: str = ""
passw: str = ""
Expand Down Expand Up @@ -373,7 +373,7 @@ def getWids(text):
if options.getDetails:
size, name = details[wid]
totalsize += size
download.append((appid,wid,name,size)) # appid, wid, name, size
download.append((int(appid),int(wid),name,size)) # appid, wid, name, size
appids.add(int(appid))
#print("download list populated")
except Exception as ex:
Expand All @@ -385,13 +385,13 @@ def getWids(text):
name = unquote(re.findall(r'workshopItemTitle">([^<]*)<', x.text)[0])
if options.getDetails:
log(f"{name}: {bytesAsSize(size)}")
download.append((appid,wid,name,size))
download.append((int(appid),int(wid),name,size))
appids.add(int(appid))
totalsize += size
else:
log('"'+line+'" doesn\'t look like a valid workshop item...\n')
if(options.getDetails):
log(f"total download size: {bytesAsSize(totalsize)}")
if(options.getDetails and len(download)>1):
log(f"total download size: {bytesAsSize(totalsize)}")
return (download, totalsize, appids)

def download():
Expand Down Expand Up @@ -450,11 +450,11 @@ def download():
success = 0
totalBytes = 0

# prepare regex
stripANSI = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~]*)') # removes ansi escape sequences
findDownloading = re.compile(r'Downloading item (\d+) ')
findSuccess = re.compile(r'Success\. Downloaded item (\d+) to "[^"]+" \((\d+) bytes\)')
findFailure = re.compile(r'')
# prepare regex (apparently re caches regex automatically -> unnecessary)
#stripANSI = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~]*)') # removes ansi escape sequences
#findDownloading = re.compile(r'Downloading item (\d+) ')
#findSuccess = re.compile(r'Success\. Downloaded item (\d+) to "[^"]+" \((\d+) bytes\)')
#findFailure = re.compile(r'')

for i in range(math.ceil(l/lim)):
#for appid in download:
Expand All @@ -470,7 +470,7 @@ def download():
else:
args.append('+login anonymous')
for appid, wid, name, size in batch:
args.append(f'+workshop_download_item {appid} {int(wid)}')
args.append(f'+workshop_download_item {appid} {wid}')
args.append("+quit")

if debug:
Expand All @@ -485,55 +485,43 @@ def download():
#t = threading.Thread(target=killThread,args=[process])
#t.start()

previous = fails + success
j=0
downloading = 0
while(process.isalive() or (j:=j+1)>3):
while(process.isalive()):
try:
#line = process.readline()
line = process.read(length=10000,blocking=False) # may fix last/single item not being registered
if debug:
print(f"[{repr(line)}]")
#line = stripANSI.sub("",line.strip())
if line=="":
log("", False) # prevents window from hanging
continue
#if line[:8] == "Download":
matches = 0
result = re.search(r'Downloading item (\d+) ', line)
if result:
wid = result[1] #re.match("(?>[^\\d]*)(\\d+)", line)[1]
log(f"Downloading {data[wid][1]} ... ", False)
if (result := re.search(r'Downloading item (\d+) ', line)):
wid = int(result[1]) #re.match("(?>[^\\d]*)(\\d+)", line)[1]
log(f"Downloading {data[wid][1]} ..", False)
status[wid] = 1
matches += 1
downloading += 1
result = re.search(r'Success\. Downloaded item (\d+) to "([^"]+)" \((\d+) bytes\)', line)
if result: #line[:8] == "Success.":
#match = re.match('(?>[^\\d]*)(\\d+)[^\\"]+\\"[^\\"]+\\" \\((\\d+) bytes\\)', line)
wid = result[1] #match[1]
if (result := re.search(r'Success\. Downloaded item (\d+) to "([^"]+)" \((\d+) bytes\)', line)):
wid = int(result[1]) #match[1]
path = result[2]
bts = int(result[3]) #match[2]
#wid = re.match("(?>[^\\d]*)(\\d+)", line)[1]
#bts = re.match("\\((\\d+) bytes\\)", line)[1]
status[wid] = 2 # success
totalBytes += bts
#log(f"{data[wid][1]} successfully downloaded ({bytesAsSize(bts)})") # data[wid][2]
log(f" done ({bytesAsSize(bts)}) [{totalBytes/totalsize*100:2.1f}]")
log(f" done ({bytesAsSize(bts)}) [{totalBytes/totalsize*100:2.1f}%]")
success += 1
matches += 1
downloading -= 1
result = re.search(r'ERROR! Download item (\d+) failed \(([^\)]+)\)', line)
if result: #line[:5] == "ERROR":
wid = result[1] #re.match("(?>[^\\d]*)(\\d+)", line)[1]
if (result := re.search(r'ERROR! Download item (\d+) failed \(([^\)]+)\)', line)):
wid = int(result[1]) #re.match("(?>[^\\d]*)(\\d+)", line)[1]
reason = result[2] #re.search("\\(([^\\)]+)\\)", line)[1]
status[wid] = 3 # error
errors[wid] = reason
#log(f"Error downloading {data[wid][1]} ({reason}).")
log(f"ERROR ({reason})")
if(options.steamdb and anonymous and options.anon_ids):
if(int(appid) not in options.anon_ids):
if(appid not in options.anon_ids):
log(f"The corresponding game isn't known to allow anonymous downloads. You might need to use an account that owns that game.")
fails += 1
matches += 1
downloading -= 1
#if matches < 1:
if downloading > 0:
Expand All @@ -546,40 +534,53 @@ def download():
pass
del process

# END BATCH
# END BATCH DOWNLOAD

# move mods
pc = {} # path cache
moves = {}

# initialize path cache
for appid in appids:
pc[appid]=pc.get(appid,options.cfg.get(str(appid),'path',
fallback = options.defaultpath and os.path.join(options.defaultpath,str(appid))))
moves[appid] = 0

for appid, wid, name, size in download:
if wid not in status:
errors[wid]="no record"
elif status[wid] == 1: # "still downloading" -> makes no sense
pass # TODO
elif status[wid] != 2: # download not successful
continue
if os.path.exists(modpath(options.steampath,appid,wid)):
# download was probably successful
status[wid]=2
path = pc.get(appid,None)
if path:
#log("Moving "+str(wid)+" to "+path+" ...",0)
if(os.path.exists(os.path.join(path,str(wid)))):
# already exists -> delete old version
shutil.rmtree(os.path.join(path,str(wid)))
shutil.move(modpath(options.steampath,appid,wid),os.path.join(path,str(wid)))
#log(" DONE")
moves[appid] += 1
pc[appid]=path
else:
status[wid]=3 # no files found -> failed
errors[wid]="unknown"
for appid in appids:
if moves[appid]>0:
log(f"Moved {moves[appid]} items to {pc[appid]}")
# TODO: clean internal download directory, in case of partial downloads (configurable)

# move mods
pc = {} # path cache
for appid, wid, name, size in batch:
if wid not in status:
errors[wid]="no record"
elif status[wid] == 1: # "still downloading" -> makes no sense
pass # TODO
elif status[wid] != 2: # download not successful
continue
if os.path.exists(modpath(options.steampath,appid,wid)):
# download was probably successful
status[wid]=2
if appid in pc or options.cfg.get(str(appid),'path',fallback=None) or options.defaultpath:
path = pc.get(appid,options.cfg.get(str(appid),'path',
fallback = options.defaultpath and os.path.join(options.defaultpath,str(appid))))
log("Moving "+str(wid)+" to "+path+" ...",0)
if(os.path.exists(os.path.join(path,str(wid)))):
# already exists -> delete old version
shutil.rmtree(os.path.join(path,str(wid)))
shutil.move(modpath(options.steampath,appid,wid),os.path.join(path,str(wid)))
log(" DONE")
pc[appid]=path
else:
status[wid]=3 # no files found -> failed
errors[wid]="unknown"
# reset input, then add back errored items to try again
URLinput.delete("1.0", tk.END)
for wid in errors:
URLinput.insert(tk.END, wid+"\n")
URLinput.insert(tk.END, str(wid)+"\n")

# print stats
log(f"downloaded {success} out of {success+fails} requested items ({totalBytes/totalsize*100:2.1f}% filesize) ")
log(f"\ndownloaded {success} out of {success+fails} requested items ({bytesAsSize(totalBytes)}) ") #{totalBytes/totalsize*100:2.1f}% filesize
if len(errors) > 0:
log(f"failed items have been added back to the input field")

Expand Down

0 comments on commit e887c50

Please sign in to comment.