fix 44bbaa3 no aitch if -1 resources

app/make-predictions.py

@@ -34,7 +34,7 @@
 sys.path.append(os.path.join(__dir__, "songexplorer", "bin", "songexplorer", "src"))
 from lib import check_config
 
-use_aitch, _, _ = check_config(os.path.join(__dir__, "configuration.py"))
+use_aitch, _, _, _, _ = check_config(os.path.join(__dir__, "configuration.py"))
 
 M.init(None, os.path.join(__dir__, "configuration.py"), use_aitch)
 V.init(None)

app/post-process.py

@@ -0,0 +1,71 @@
+import sys
+import ast
+import os
+import re
+import csv
+import socket
+from datetime import datetime
+
+delete_wavs = False
+delete_logs = False
+
+print(str(datetime.now())+": start time")
+
+__dir__ = os.path.dirname(__file__)
+
+with open(os.path.join(__dir__, "songexplorer", "bin", "songexplorer", "VERSION.txt"), 'r') as fid:
+    print('SongExplorer version = '+fid.read().strip().replace('\n',', '))
+print("hostname = "+socket.gethostname())
+
+_, wavfile = sys.argv
+print("wavefile = ", wavfile)
+
+sys.path.append(os.path.join(__dir__, "songexplorer", "bin", "songexplorer", "src", "gui"))
+import model as M
+
+M.init(None, os.path.join(__dir__, "configuration.py"), False)
+
+wavfile_noext = M.trim_ext(wavfile)
+
+with open(wavfile_noext+"-classify.log",'r') as fid:
+    for line in fid:
+        if "labels: " in line:
+            m=re.search('labels: (.+)',line)
+            labels = ast.literal_eval(m.group(1))
+        if "audio_tic_rate = " in line:
+            m=re.search('audio_tic_rate = (.+)',line)
+            audio_tic_rate = ast.literal_eval(m.group(1))
+print("labels = ", labels)
+print("audio_tic_rate  = ", audio_tic_rate)
+
+if delete_wavs:
+    for label in labels:
+        fullpath = wavfile_noext+'-'+label+'.wav'
+        os.remove(fullpath)
+        print("deleting ", fullpath)
+
+durations = {}
+counts = {}
+with open(wavfile_noext+'-predicted-1.0pr.csv') as fid:
+    csvreader = csv.reader(fid)
+    for row in csvreader:
+        if row[4] not in durations:
+            durations[row[4]] = 0
+            counts[row[4]] = 0
+        durations[row[4]] += (int(row[2]) - int(row[1]))
+        counts[row[4]] += 1
+
+with open(wavfile_noext+"-post-process.csv",'w') as fid:
+    fid.write("wavfile,label,duration ("+M.context_time_units+"),num events\n")
+    for label in durations.keys():
+        fid.write(os.path.basename(wavfile)+','+
+                  label+','+
+                  str(durations[label] / audio_tic_rate / M.context_time_scale)+','+
+                  str(counts[label])+'\n')
+
+if delete_logs:
+    os.remove(wavfile_noext+'-classify.log')
+    os.remove(wavfile_noext+'-ethogram.log')
+    os.remove(wavfile_noext+'-post-process.log')
+
+print(str(datetime.now())+": finish time")

src/lib.py

@@ -59,28 +59,33 @@ def isreal(l, x):
     isinteger(locals(), "classify_parallelize")
 
     all_minusone = True
-    global_vars = globals().copy()
+    local_vars = locals().copy()
     for resource_kind in ["ncpu_cores", "ngpu_cards", "ngigabytes_memory"]:
-        for job_resource_name in filter(lambda x: resource_kind in x, global_vars.keys()):
-            isinteger(job_resource_name)
-            job_resource_value = global_vars[job_resource_name]
+        for job_resource_name in filter(lambda x: resource_kind in x, local_vars.keys()):
+            isinteger(locals(), job_resource_name)
+            job_resource_value = local_vars[job_resource_name]
             all_minusone &= job_resource_value == -1
+    if all_minusone:
+        print("INFO: all job resources are -1 so only one job will be run at a time")
+
+    return not all_minusone, locals()["server_username"], locals()["server_ipaddr"], local_vars, locals()["source_path"]
+
+def check_config2(config_vars, resource_vars, server_ipaddr):
+    for resource_kind in ["ncpu_cores", "ngpu_cards", "ngigabytes_memory"]:
+        for job_resource_name in filter(lambda x: resource_kind in x, config_vars.keys()):
+            job_resource_value = config_vars[job_resource_name]
             local_resource_name = "local_"+resource_kind
-            local_resource_value = global_vars[local_resource_name]
+            local_resource_value = resource_vars[local_resource_name]
             if job_resource_value > local_resource_value:
                   print("WARNING: "+job_resource_name+" exceeds "+
                         str(local_resource_value)+" "+local_resource_name)
             if server_ipaddr:
                 server_resource_name = "server_"+resource_kind
-                server_resource_value = global_vars[server_resource_name]
+                server_resource_value = resource_vars[server_resource_name]
                 if job_resource_value > server_resource_value:
                       print("WARNING: "+job_resource_name+" exceeds "+
                             str(server_resource_value)+" "+server_resource_name)
 
-    if all_minusone:
-        print("INFO: all job resources are -1 so only one job will be run at a time")
-    return not all_minusone, locals()["server_username"], locals()["server_ipaddr"]
-
 
 def get_srcrepobindirs():
     srcdir = os.path.dirname(os.path.realpath(__file__))

src/songexplorer

@@ -25,14 +25,19 @@ os.environ['PATH'] = os.pathsep.join([*bindirs, *os.environ['PATH'].split(os.pat
 _, configuration_file, port = sys.argv
 
 sys.path.append(os.path.join(os.path.dirname(__file__), "src"))
-from lib import check_config
+from lib import check_config, check_config2
 
-use_aitch, server_username, server_ipaddr = check_config(configuration_file)
+use_aitch, server_username, server_ipaddr, config_vars, source_path = check_config(configuration_file)
 
 local_ncpu_cores = os.cpu_count()
 local_ngpu_cards = len(tf.config.list_physical_devices("GPU"))
 local_ngigabytes_memory = int(psutil.virtual_memory().total/1024/1024/1024)
 
+resource_vars = {
+    "local_ncpu_cores":local_ncpu_cores,
+    "local_ngpu_cards":local_ngpu_cards,
+    "local_ngigabytes_memory":local_ngigabytes_memory}
+
 print("detected "+str(local_ncpu_cores)+" local_ncpu_cores, "+
                   str(local_ngpu_cards)+" local_ngpu_cards, "+
                   str(local_ngigabytes_memory)+" local_ngigabytes_memory")
@@ -50,10 +55,18 @@ if server_ipaddr:
             stdout=PIPE)
     server_ngigabytes_memory = int(p.stdout.decode('ascii').rstrip())
 
+    resource_vars.update({
+        "server_ncpu_cores":server_ncpu_cores,
+        "server_ngpu_cards":server_ngpu_cards,
+        "server_ngigabytes_memory":server_ngigabytes_memory})
+
     print("detected "+str(server_ncpu_cores)+" server_ncpu_cores, "+
                       str(server_ngpu_cards)+" server_ngpu_cards, "+
                       str(server_ngigabytes_memory)+" server_ngigabytes_memory")
 
+check_config2(config_vars, resource_vars, server_ipaddr)
+
+
 repodir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 with open(os.path.join(repodir,"VERSION.txt"), 'r') as fid:
     version = fid.read().replace('\n', ' ')