Merge branch 'master' into snyk-fix-528f81a34cccbcd72285c6f88de813d6

.github/workflows/build.yml

@@ -72,11 +72,17 @@ jobs:
       run: |
         cd ./target/
         [[ -f ~/.m2/repository/apt-cli-1.7.jar ]] || wget --no-clobber --quiet -O ~/.m2/repository/apt-cli-1.7.jar https://github.com/dernasherbrezon/apt-cli/releases/download/apt-cli-1.7/apt-cli.jar
-        java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename r2cloud --gpg-keyname F2DCBFDCA5A70917  save --patterns ./*.deb
-        java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename stretch --gpg-keyname F2DCBFDCA5A70917  save --patterns ./*.deb
-        java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename buster --gpg-keyname F2DCBFDCA5A70917  save --patterns ./*.deb
         java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename bullseye --gpg-keyname F2DCBFDCA5A70917  save --patterns ./*.deb
         java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename bookworm --gpg-keyname F2DCBFDCA5A70917  save --patterns ./*.deb
         java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename bionic --gpg-keyname F2DCBFDCA5A70917  save --patterns ./*.deb
         java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename focal --gpg-keyname F2DCBFDCA5A70917   save --patterns ./*.deb
         java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename jammy --gpg-keyname F2DCBFDCA5A70917   save --patterns ./*.deb
+        echo "building for old OS"
+        cd ..
+        mvn clean
+        mvn -Dmaven.test.skip=true -Djdk.version="r2cloud-jdk (>=17.0.13-1)" package
+        cd ./target/
+        java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename stretch --gpg-keyname F2DCBFDCA5A70917  save --patterns ./*.deb
+        java -jar ~/.m2/repository/apt-cli-1.7.jar --url s3://r2cloud --component main --codename buster --gpg-keyname F2DCBFDCA5A70917  save --patterns ./*.deb
+        
+        

README.md

@@ -46,9 +46,9 @@ Please check recommended [bill of materials](https://github.com/dernasherbrezon/
     - Login via SSH and create ```r2cloud.txt``` file in /boot directory. This file should contain any random string. This string is a login token. This token will be used during initial setup.
     - Execute the following commands:
 ```
-sudo apt-get install dirmngr lsb-release
-sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A5A70917
-sudo bash -c "echo 'deb http://apt.leosatdata.com $(lsb_release --codename --short) main' > /etc/apt/sources.list.d/r2cloud.list"
+sudo apt-get install curl lsb-release
+curl -fsSL https://leosatdata.com/r2cloud.gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/r2cloud.gpg
+sudo bash -c "echo 'deb [signed-by=/usr/share/keyrings/r2cloud.gpg] http://apt.leosatdata.com $(lsb_release --codename --short) main' > /etc/apt/sources.list.d/r2cloud.list"
 sudo apt-get update
 sudo apt-get install r2cloud
 ```

pom.xml

@@ -28,6 +28,7 @@
 
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<jdk.version>openjdk-17-jre-headless</jdk.version>
 	</properties>
 
 	<distributionManagement>
@@ -88,7 +89,7 @@
 		<dependency>
 			<groupId>ru.r2cloud</groupId>
 			<artifactId>jradio</artifactId>
-			<version>1.101</version>
+			<version>1.103</version>
 		</dependency>
 		<dependency>
 			<groupId>org.nanohttpd</groupId>
@@ -277,7 +278,7 @@
 			<plugin>
 				<groupId>com.aerse.maven</groupId>
 				<artifactId>deb-maven-plugin</artifactId>
-				<version>1.16</version>
+				<version>1.19</version>
 				<executions>
 					<execution>
 						<id>package</id>
@@ -292,19 +293,7 @@
 					<unixGroupId>pi</unixGroupId>
 					<arch>all</arch>
 					<section>embedded</section>
-					<osDependencies>
-						<r2cloud-ui>&gt;=20240303125106</r2cloud-ui>
-						<r2cloud-jdk>&gt;=17.0.5-1</r2cloud-jdk>
-						<nginx-light>&gt;=1.10.3</nginx-light>
-						<rtl-sdr>&gt;=0.6.4</rtl-sdr>
-						<!-- required for stdbuf to force stderr flushing -->
-						<coreutils>&gt;=8.26</coreutils>
-						<!-- required by wxtoimg -->
-						<libxft2>&gt;=2.3.2</libxft2>
-						<sox>&gt;=14.4.1</sox>
-						<!-- required by deps-maven-plugin -->
-						<wget/>
-					</osDependencies>
+					<depends>r2cloud-ui (>=20241102151628), ${jdk.version}, nginx-light(>=1.10.3), rtl-sdr(>=0.6.4), coreutils(>=8.26), libxft2(>=2.3.2), sox(>=14.4.1), wget</depends>
 					<javaServiceWrapper>false</javaServiceWrapper>
 					<fileSets>
 						<fileSet>
@@ -315,10 +304,6 @@
 							<source>${basedir}/src/main/deb/lib</source>
 							<target>/lib</target>
 						</fileSet>
-						<fileSet>
-							<source>${basedir}/src/main/deb/html</source>
-							<target>html</target>
-						</fileSet>
 						<fileSet>
 							<source>${basedir}/src/main/resources/.wxtoimglic</source>
 							<target>/home/pi/.wxtoimglic</target>

src/main/deb/lib/systemd/system/r2cloud.service

@@ -4,7 +4,7 @@ After=time-sync.target
 
 [Service]
 WorkingDirectory=/home/pi/r2cloud/
-ExecStart=/home/pi/r2cloud-jdk/bin/java -cp /home/pi/r2cloud/etc:/home/pi/r2cloud/lib/*:/usr/share/java/r2cloud/* -Djava.library.path=/usr/lib/jni -Djradio.metrics.enabled=true -Duser.timezone=UTC -Djava.util.logging.config.file=/home/pi/r2cloud/etc/logging-prod.properties ru.r2cloud.R2Cloud etc/config-prod.properties
+ExecStart=/usr/bin/java -cp /home/pi/r2cloud/etc:/home/pi/r2cloud/lib/*:/usr/share/java/r2cloud/* -Djava.library.path=/usr/lib/jni -Djradio.metrics.enabled=true -Duser.timezone=UTC -Djava.util.logging.config.file=/home/pi/r2cloud/etc/logging-prod.properties ru.r2cloud.R2Cloud etc/config-prod.properties
 SyslogIdentifier=r2cloud
 SuccessExitStatus=143
 Restart=always

src/main/deb/postinst

@@ -1,6 +1,34 @@
 chmod 755 ${config.installDir}/lib/download.sh
 ${config.installDir}/lib/download.sh ${config.installDir}/lib/ /usr/share/java/r2cloud/
 
+if id "${config.user}" &>/dev/null; then
+    echo "User '${config.user}' already exists."
+else
+    useradd -m ${config.user}
+    echo "User '${config.user}' created."
+fi
+
+if ! getent group ${config.group} > /dev/null; then
+    groupadd ${config.group}
+    echo "Group '${config.group}' created."
+fi
+
+if [ $(getent group dialout) ]; then
+    usermod -a -G dialout ${config.user}
+fi
+
+if [ $(getent group plugdev) ]; then
+    usermod -a -G plugdev ${config.user}
+fi
+
+if [ $(getent group systemd-journal) ]; then
+    usermod -a -G systemd-journal ${config.user}
+fi
+
+if [ -d /var/lib/nginx/proxy ]; then
+	chown -R www-data:www-data /var/lib/nginx/proxy
+fi
+
 chown -R ${config.user}:${config.group} ${config.installDir} /usr/share/java/r2cloud/*.jar
 chmod 640 ${config.installDir}/lib/*.jar /usr/share/java/r2cloud/*.jar
 
@@ -44,10 +72,6 @@ if [ -f /etc/sudoers.d/nginx ]; then
 	chmod 440 /etc/sudoers.d/nginx
 fi
 
-if [ $(getent group systemd-journal) ]; then
-    usermod -a -G systemd-journal ${config.user}
-fi
-
 if [ ! -d /var/log/journal ]; then
     echo "enable persistent journald logs"
     mkdir -p /var/log/journal

src/main/deb/prerm

@@ -1,2 +1,2 @@
 echo "removing libs..."
-rm ${config.installDir}/lib/*.jar
+rm -f ${config.installDir}/lib/*.jar

src/main/java/ru/r2cloud/R2Cloud.java

@@ -95,6 +95,7 @@
 import ru.r2cloud.web.api.observation.ObservationSigMfData;
 import ru.r2cloud.web.api.observation.ObservationSigMfMeta;
 import ru.r2cloud.web.api.observation.ObservationSpectrogram;
+import ru.r2cloud.web.api.schedule.SatelliteLoad;
 import ru.r2cloud.web.api.schedule.ScheduleComplete;
 import ru.r2cloud.web.api.schedule.ScheduleFull;
 import ru.r2cloud.web.api.schedule.ScheduleList;
@@ -284,7 +285,8 @@ public R2Cloud(Configuration props, Clock clock) {
 		index(new DeviceConfigList(deviceManager));
 		index(new DeviceConfigDelete(props, deviceManager));
 		index(new Restart());
-		index(new DeviceSchedule(deviceManager, satelliteDao));
+		index(new DeviceSchedule(deviceManager, satelliteDao, clock));
+		index(new SatelliteLoad(satelliteDao));
 		webServer = new WebServer(props, controllers, auth, signed);
 	}
 

src/main/java/ru/r2cloud/cloud/InfluxDBClient.java

@@ -94,6 +94,11 @@ public void send(Observation obs, Satellite satellite) {
 			frames = 0L;
 		}
 		metric.append(",numberOfDecodedPackets=").append(frames);
+		Long totalSize = obs.getTotalSize();
+		if (totalSize == null) {
+			totalSize = 0L;
+		}
+		metric.append(",totalSize=").append(totalSize);
 		metric.append(",duration=").append(obs.getEndTimeMillis() - obs.getStartTimeMillis());
 		metric.append(" ").append(obs.getStartTimeMillis()).append("000000"); // nanoseconds
 

src/main/java/ru/r2cloud/cloud/LeoSatDataClient.java

@@ -19,6 +19,7 @@
 import java.util.Collections;
 import java.util.Date;
 import java.util.List;
+import java.util.Optional;
 import java.util.TimeZone;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Function;
@@ -105,7 +106,18 @@ public List<Satellite> loadNewLaunches(long lastModified) throws NotModifiedExce
 				}
 				return Collections.emptyList();
 			}
-			List<Satellite> result = readNewLaunches(response.body());
+			Optional<String> lastModifiedOnServer = response.headers().firstValue("Last-Modified");
+			lastModified = 0;
+			if (lastModifiedOnServer.isPresent()) {
+				SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
+				sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+				try {
+					lastModified = sdf.parse(lastModifiedOnServer.get()).getTime();
+				} catch (java.text.ParseException e) {
+					LOG.error("invalid date provided: {}", lastModifiedOnServer.get(), e);
+				}
+			}
+			List<Satellite> result = readNewLaunches(response.body(), lastModified);
 			LOG.info("new satellites from leosatdata were loaded: {}", result.size());
 			return result;
 		} catch (IOException e) {
@@ -141,7 +153,18 @@ public List<Satellite> loadSatellites(long lastModified, Boolean all) throws Not
 				}
 				return Collections.emptyList();
 			}
-			List<Satellite> result = readSatellites(response.body());
+			Optional<String> lastModifiedOnServer = response.headers().firstValue("Last-Modified");
+			lastModified = 0;
+			if (lastModifiedOnServer.isPresent()) {
+				SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
+				sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+				try {
+					lastModified = sdf.parse(lastModifiedOnServer.get()).getTime();
+				} catch (java.text.ParseException e) {
+					LOG.error("invalid date provided: {}", lastModifiedOnServer.get(), e);
+				}
+			}
+			List<Satellite> result = readSatellites(response.body(), lastModified);
 			LOG.info("satellites from leosatdata were loaded: {}", result.size());
 			return result;
 		} catch (IOException e) {
@@ -227,7 +250,7 @@ private HttpResponse<String> sendWithRetry(HttpRequest request, HttpResponse.Bod
 		}
 	}
 
-	private static List<Satellite> readSatellites(String body) {
+	private static List<Satellite> readSatellites(String body, long lastModified) {
 		JsonValue parsedJson;
 		try {
 			parsedJson = Json.parse(body);
@@ -258,12 +281,15 @@ private static List<Satellite> readSatellites(String body) {
 				continue;
 			}
 			cur.setSource(SatelliteSource.LEOSATDATA);
+			if (cur.getLastUpdateTime() == 0) {
+				cur.setLastUpdateTime(lastModified);
+			}
 			result.add(cur);
 		}
 		return result;
 	}
 
-	private List<Satellite> readNewLaunches(String body) {
+	private List<Satellite> readNewLaunches(String body, long lastModified) {
 		JsonValue parsedJson;
 		try {
 			parsedJson = Json.parse(body);
@@ -302,6 +328,9 @@ private List<Satellite> readNewLaunches(String body) {
 			newLaunch.setPriority(Priority.HIGH);
 			newLaunch.setTle(readTle(asObject.get("tle")));
 			newLaunch.setSource(SatelliteSource.LEOSATDATA);
+			if (newLaunch.getLastUpdateTime() == 0) {
+				newLaunch.setLastUpdateTime(lastModified);
+			}
 			result.add(newLaunch);
 		}
 		return result;

src/main/java/ru/r2cloud/cloud/SatnogsClient.java

@@ -84,10 +84,6 @@ public List<Satellite> loadSatellites() {
 		List<Satellite> result = loadAllSatellites(groupBySatelliteId);
 		for (Satellite cur : result) {
 			dedupTransmittersByKey(cur);
-			if (!cur.getPriority().equals(Priority.HIGH)) {
-				continue;
-			}
-			cur.setTle(loadTleBySatelliteId(cur.getId()));
 		}
 		LOG.info("satellites from satnogs were loaded: {}", result.size());
 		return result;
@@ -180,6 +176,8 @@ private List<Satellite> readSatellites(String body, Map<String, List<Transmitter
 		List<Satellite> result = new ArrayList<>();
 		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
 		sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+		SimpleDateFormat updatedFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS");
+		updatedFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
 		for (int i = 0; i < parsedArray.size(); i++) {
 			JsonValue jsonValue = parsedArray.get(i);
 			if (!jsonValue.isObject()) {
@@ -212,6 +210,16 @@ private List<Satellite> readSatellites(String body, Map<String, List<Transmitter
 			// thus disabled by default even if "new launch"
 			// https://gitlab.com/librespacefoundation/satnogs/satnogs-db/-/issues/551
 			cur.setEnabled(false);
+			cur.setTle(loadTleBySatelliteId(id));
+			String updatedStr = getStringSafely(satellite, "updated");
+			if (updatedStr != null) {
+				try {
+					updatedStr = updatedStr.substring(0, updatedStr.length() - 4);
+					cur.setLastUpdateTime(updatedFormat.parse(updatedStr).getTime());
+				} catch (java.text.ParseException e) {
+					// ignore
+				}
+			}
 			// schedule observations anyway
 			if (status.equalsIgnoreCase("future")) {
 				cur.setId(id);

src/main/java/ru/r2cloud/model/DecoderResult.java

@@ -9,9 +9,18 @@ public class DecoderResult {
 	private String channelA;
 	private String channelB;
 	private Long numberOfDecodedPackets = 0L;
+	private Long totalSize = 0L;
 
 	private File imagePath;
 	private File dataPath;
+
+	public Long getTotalSize() {
+		return totalSize;
+	}
+
+	public void setTotalSize(Long totalSize) {
+		this.totalSize = totalSize;
+	}
 
 	public File getRawPath() {
 		return rawPath;

src/main/java/ru/r2cloud/model/Observation.java

@@ -30,6 +30,7 @@ public class Observation {
 	private String channelA;
 	private String channelB;
 	private Long numberOfDecodedPackets;
+	private Long totalSize = 0L;
 
 	private String rawURL;
 	private File rawPath;
@@ -203,6 +204,14 @@ public void setChannelB(String channelB) {
 		this.channelB = channelB;
 	}
 
+	public Long getTotalSize() {
+		return totalSize;
+	}
+
+	public void setTotalSize(Long totalSize) {
+		this.totalSize = totalSize;
+	}
+
 	public Long getNumberOfDecodedPackets() {
 		return numberOfDecodedPackets;
 	}
@@ -307,6 +316,10 @@ public static Observation fromJson(JsonObject meta) {
 		if (numberOfDecodedPackets != null) {
 			result.setNumberOfDecodedPackets(numberOfDecodedPackets.asLong());
 		}
+		JsonValue totalSize = meta.get("totalSize");
+		if (totalSize != null) {
+			result.setTotalSize(totalSize.asLong());
+		}
 		result.setRawURL(meta.getString("rawURL", null));
 		result.setaURL(meta.getString("aURL", null));
 		result.setSpectogramURL(meta.getString("spectogramURL", null));
@@ -352,6 +365,9 @@ public JsonObject toJson(SignedURL signed) {
 		if (getNumberOfDecodedPackets() != null) {
 			json.add("numberOfDecodedPackets", getNumberOfDecodedPackets());
 		}
+		if (totalSize != null) {
+			json.add("totalSize", getTotalSize());
+		}
 		addNullable("rawURL", getRawURL(), signed, json);
 		addNullable("aURL", getaURL(), signed, json);
 		addNullable("spectogramURL", getSpectogramURL(), signed, json);