Merge pull request #60 from tparker-usgs/plot.data_migration

Plot.data migration

pom.xml

@@ -4,7 +4,7 @@
 
 	<groupId>gov.usgs.volcanoes</groupId>
 	<artifactId>winston</artifactId>
-	<version>1.3.0</version>
+	<version>1.3.1</version>
 	<packaging>jar</packaging>
 
 	<name>Winston</name>
@@ -519,15 +519,10 @@
 			<artifactId>log4j</artifactId>
 			<version>1.2.13</version>
 		</dependency>
-		<dependency>
-			<groupId>gov.usgs.volcanoes</groupId>
-			<artifactId>usgs</artifactId>
-			<version>1.3.16</version>
-		</dependency>
 		<dependency>
 			<groupId>gov.usgs.volcanoes</groupId>
 			<artifactId>volcano-core</artifactId>
-			<version>[1.3.18,)</version>
+			<version>1.4.1</version>
 		</dependency>
 		<dependency>
 			<groupId>commons-collections</groupId>
@@ -581,5 +576,10 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+        	<groupId>gov.usgs.volcanoes</groupId>
+        	<artifactId>wwsclient</artifactId>
+        	<version>1.3.3</version>
+        </dependency>
 	</dependencies>
 </project>

src/main/java/gov/usgs/volcanoes/winston/db/Data.java

@@ -6,23 +6,27 @@
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.text.DateFormat;
+import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 import java.util.TimeZone;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import gov.usgs.earthworm.message.TraceBuf;
-import gov.usgs.math.DownsamplingType;
-import gov.usgs.plot.data.HelicorderData;
-import gov.usgs.plot.data.RSAMData;
-import gov.usgs.plot.data.Wave;
 import gov.usgs.volcanoes.core.Zip;
+import gov.usgs.volcanoes.core.data.HelicorderData;
+import gov.usgs.volcanoes.core.data.RSAMData;
 import gov.usgs.volcanoes.core.data.Scnl;
+import gov.usgs.volcanoes.core.data.Wave;
+import gov.usgs.volcanoes.core.legacy.ew.message.TraceBuf;
+import gov.usgs.volcanoes.core.math.DownsamplingType;
+import gov.usgs.volcanoes.core.time.CurrentTime;
 import gov.usgs.volcanoes.core.time.J2kSec;
 import gov.usgs.volcanoes.core.time.Time;
+import gov.usgs.volcanoes.core.time.TimeSpan;
 import gov.usgs.volcanoes.core.util.UtilException;
 
 /**
@@ -87,7 +91,7 @@ public double[] getTimeSpan(final int sid) {
    */
   public double[] getTimeSpan(final String code) {
     if (!winston.checkConnect())
-      return null;
+      return new double[] {0,0};
     try {
       ResultSet rs = winston.getStatement().executeQuery("SELECT st, et FROM `"
           + winston.databasePrefix + "_ROOT`.channels WHERE code='" + code + "'");
@@ -107,7 +111,7 @@ public double[] getTimeSpan(final String code) {
     } catch (final Exception e) {
       LOGGER.error("Could not get time span for channel: {}. ({})", code, e.getLocalizedMessage());
     }
-    return null;
+    return new double[] {0,0};
   }
 
   /**
@@ -129,6 +133,22 @@ private List<String> daysBetween(final double t1, final double t2) {
     return result;
   }
 
+  /**
+   * Get a list of days in a time span
+   *
+   * @param timeSpan timeSpan
+   * @return List of strings representing each day in time span
+   */
+  private List<String> daysBetween(TimeSpan timeSpan) {
+    final ArrayList<String> result = new ArrayList<String>();
+    long time = timeSpan.startTime;
+    while (time < timeSpan.endTime + Time.DAY_IN_MS) {
+      result.add(Time.format(WinstonDatabase.WINSTON_TABLE_DATE_FORMAT, new Date(time)));
+      time += Time.DAY_IN_MS;
+    }
+    return result;
+  }
+
   /**
    * Finds data gaps in a given channel between two times. Returns null
    * on a Winston error. Returns a single item list with the given time span
@@ -157,18 +177,17 @@ public List<double[]> findGaps(final String code, double t1, final double t2) {
 
     try {
       final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy_MM_dd");
+      dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
 
       final List<String> days = daysBetween(t1, t2);
-
-      dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
       final List<double[]> bufs = new ArrayList<double[]>(2 * ONE_DAY);
+
       for (final String day : days) {
         final double tst = J2kSec.parse(WinstonDatabase.WINSTON_TABLE_DATE_FORMAT, day);
         final double tet = tst + ONE_DAY;
         final String table = code + "$$" + day;
         if (!winston.tableExists(code, table))
           continue;
-
         if (tet < t1)
           continue;
         if (tst > t2)
@@ -188,7 +207,7 @@ public List<double[]> findGaps(final String code, double t1, final double t2) {
         rs.close();
       }
 
-      if (bufs == null || bufs.size() == 0) {
+      if (bufs.size() == 0) {
         // there were no tracebufs in the time range, the whole span is a gap.
         gaps.add(new double[] {t1, t2});
         return gaps;
@@ -223,12 +242,89 @@ public List<double[]> findGaps(final String code, double t1, final double t2) {
       }
 
       return gaps;
-    } catch (final Exception e) {
-      e.printStackTrace();
+    } catch (ParseException e) {
+      LOGGER.error("Cannot parse j2kSec from date");
+    } catch (SQLException e) {
+      LOGGER.error("Cannot get times");
     }
     return null;
   }
 
+
+  public List<TimeSpan> findGaps(String code, TimeSpan timeSpan) {
+    final List<TimeSpan> gaps = new ArrayList<TimeSpan>();
+    timeSpan = new TimeSpan(applyLookback(timeSpan.startTime), timeSpan.endTime);
+
+    if (timeSpan.startTime >= timeSpan.endTime) {
+      LOGGER.debug("null timeSpan {}", timeSpan);
+      return gaps;
+    }
+
+    if (!winston.checkConnect()) {
+      LOGGER.debug("Cannot connect to winston");
+      return gaps;
+    }
+
+    if (!winston.useDatabase(code)) {
+      // database didn't exist so the whole thing must be a gap
+      gaps.add(timeSpan);
+      return gaps;
+    }
+
+    final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy_MM_dd");
+    dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
+    final List<String> days = daysBetween(timeSpan);
+
+    double startJ2k = J2kSec.fromEpoch(timeSpan.startTime);
+    double endJ2k = J2kSec.fromEpoch(timeSpan.endTime);
+    double last = startJ2k;
+    for (final String day : days) {
+      List<double[]> bufs;
+      try {
+        bufs = getBufTimes(code, day);
+      } catch (SQLException e) {
+        LOGGER.error("Unable to read day table {}:{}", code, day);
+        bufs = new ArrayList<double[]>();
+      }
+
+      for (double[] buf : bufs) {
+        if (startJ2k >= buf[1] || endJ2k <= buf[0]) {
+          continue;
+        }
+
+        if (buf[0] > last) {
+          gaps.add(new TimeSpan(J2kSec.asEpoch(last), J2kSec.asEpoch(buf[0])));
+        }
+        last = buf[1];
+      }
+    }
+
+    if (last < endJ2k) {
+      gaps.add(new TimeSpan(J2kSec.asEpoch(last), timeSpan.endTime));
+    }
+
+    return gaps;
+  }
+
+  private List<double[]> getBufTimes(String code, String table) throws SQLException {
+    final List<double[]> bufs = new ArrayList<double[]>(2 * ONE_DAY);
+    if (!winston.tableExists(code, table)) {
+      return bufs;
+    }
+
+    String sql = String.format("SELECT st, et FROM `%s` ORDER BY st ASC", table);
+    final ResultSet rs = winston.getStatement().executeQuery(sql);
+    while (rs.next()) {
+      final double start = rs.getDouble(1);
+      final double end = rs.getDouble(2);
+      bufs.add(new double[] {start, end});
+    }
+    rs.close();
+
+    return bufs;
+
+  }
+
   /**
    * Return wave data for timespan t1..t2 for channel w/ id sid; cap result at
    * maxrows size. Currently returns larger than asked for.
@@ -397,7 +493,7 @@ public List<TraceBuf> getTraceBufs(final String code, final double t1, final dou
         traceBufs.add(new TraceBuf(buf));
 
       return traceBufs;
-    } catch (final Exception e) {
+    } catch (final IOException e) {
       LOGGER.error("Could not get TraceBufs for {}, {}->{}", code, t1, t2);
     }
     return null;
@@ -557,7 +653,7 @@ public RSAMData getRSAMData(final Scnl scnl, double t1, final double t2,
             try {
               rs = winston.getStatement().executeQuery("SELECT COUNT(*) FROM (SELECT 1 "
                   + sql.substring(sql.indexOf("FROM")) + ") as T");
-            } catch (final Exception e) {
+            } catch (final SQLException e) {
               // table not found
               continue;
             }
@@ -625,7 +721,7 @@ else if (ds.equals(DownsamplingType.MEAN)) {
       sb.append("((j2ksec-").append(startTime).append(") DIV ").append(dsInt).append(") intNum ");
       sb.append(sql_from_where_clause);
       sb.append(" GROUP BY intNum");
-      
+
       return sb.toString();
     } else
       throw new UtilException("Unknown downsampling type: " + ds);
@@ -657,4 +753,9 @@ private double applyLookback(double time) {
     return Math.max(time, lookback);
   }
 
+  private long applyLookback(long time) {
+    long lookback = CurrentTime.getInstance().now() - (winston.maxDays * Time.DAY_IN_MS);
+    return Math.max(time, lookback);
+  }
+
 }

src/main/java/gov/usgs/volcanoes/winston/db/Deriver.java

@@ -6,9 +6,9 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import gov.usgs.earthworm.message.TraceBuf;
 import gov.usgs.volcanoes.core.CodeTimer;
 import gov.usgs.volcanoes.core.configfile.ConfigFile;
+import gov.usgs.volcanoes.core.legacy.ew.message.TraceBuf;
 import gov.usgs.volcanoes.core.time.Time;
 import gov.usgs.volcanoes.core.util.StringUtils;
 import gov.usgs.volcanoes.core.util.UtilException;

src/main/java/gov/usgs/volcanoes/winston/db/Export.java

@@ -1,13 +1,18 @@
 package gov.usgs.volcanoes.winston.db;
 
+import java.io.IOException;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.TimeZone;
 
-import gov.usgs.plot.data.Wave;
-import gov.usgs.plot.data.file.SeismicDataFile;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import gov.usgs.volcanoes.core.configfile.ConfigFile;
+import gov.usgs.volcanoes.core.data.Wave;
+import gov.usgs.volcanoes.core.data.file.SeismicDataFile;
 import gov.usgs.volcanoes.core.time.J2kSec;
+import gov.usgs.volcanoes.core.util.UtilException;
 
 /**
  *
@@ -16,6 +21,7 @@
  * @author Dan Cervelli
  */
 public class Export {
+  private static final Logger LOGGER = LoggerFactory.getLogger(Export.class);
   private final WinstonDatabase winston;
   private final Data data;
 
@@ -36,32 +42,36 @@ public static void readConfigFile() {
   }
 
   public void export(final String code, final String pre, final double t1, final double t2) {
-    try {
-      final double maxSize = 3600;
-      final double nst = t1 - (t1 % 5);
-      // double net = t2 + (5 - t2 % 5);
-      double ct = nst;
-      int cnt = 0;
-      while (ct < t2) {
-        Wave sw;
+    final double maxSize = 3600;
+    final double nst = t1 - (t1 % 5);
+    // double net = t2 + (5 - t2 % 5);
+    double ct = nst;
+    int cnt = 0;
+    while (ct < t2) {
+      Wave sw = null;
+      try {
         if (t2 - ct > maxSize) {
           sw = data.getWave(code, ct, ct + maxSize, 0);
           ct += maxSize;
         } else {
           sw = data.getWave(code, ct, ct + maxSize, 0);
           ct = t2;
         }
-        if (sw != null) {
-          final SeismicDataFile file =
-              SeismicDataFile.getFile(pre + "_" + code + "_" + cnt + ".txt");
-          file.putWave(code, sw);
+      } catch (UtilException ex) {
+        LOGGER.error("Export error: {}", ex);
+      }
+      if (sw != null) {
+        final SeismicDataFile file =
+            SeismicDataFile.getFile(pre + "_" + code + "_" + cnt + ".txt");
+        file.putWave(code, sw);
+        try {
           file.write();
+        } catch (IOException ex) {
+          LOGGER.error("Unable to write file: {}", ex);
         }
-        System.out.println((ct - t1) + "s");
-        cnt++;
       }
-    } catch (final Exception e) {
-      e.printStackTrace();
+      System.out.println((ct - t1) + "s");
+      cnt++;
     }
   }
 

src/main/java/gov/usgs/volcanoes/winston/db/Input.java

@@ -19,8 +19,8 @@
 import org.slf4j.LoggerFactory;
 
 import cern.colt.Arrays;
-import gov.usgs.earthworm.message.TraceBuf;
 import gov.usgs.volcanoes.core.Zip;
+import gov.usgs.volcanoes.core.legacy.ew.message.TraceBuf;
 import gov.usgs.volcanoes.core.time.CurrentTime;
 import gov.usgs.volcanoes.core.time.J2kSec;
 

src/main/java/gov/usgs/volcanoes/winston/db/InputEW.java

@@ -26,8 +26,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import gov.usgs.earthworm.message.TraceBuf;
 import gov.usgs.volcanoes.core.Zip;
+import gov.usgs.volcanoes.core.legacy.ew.message.TraceBuf;
 import gov.usgs.volcanoes.core.time.CurrentTime;
 import gov.usgs.volcanoes.core.time.J2kSec;
 import gov.usgs.volcanoes.core.time.Time;

src/main/java/gov/usgs/volcanoes/winston/db/WinstonDatabase.java

@@ -29,7 +29,7 @@ public class WinstonDatabase {
 
   public static final String WINSTON_TABLE_DATE_FORMAT = "yyyy_MM_dd";
   public static final String CURRENT_SCHEMA_VERSION = "1.1.1";
-  public static final long MAX_DAYS_UNLIMITED = Long.MAX_VALUE / Time.DAY_IN_S;
+  public static final long MAX_DAYS_UNLIMITED = Long.MAX_VALUE / Time.DAY_IN_MS;
 
   private static final String DEFAULT_DATABASE_PREFIX = "W";
   private static final String DEFAULT_CONFIG_FILENAME = "Winston.config";