From 67011be08f00190e0e7f6f59bea99579d1debcb3 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Fri, 20 Jan 2017 11:23:57 +0100 Subject: [PATCH 01/19] use individual pixel gains (from integralGain file) the template pulse integral between the "half height" and "half height + 30 slices" I have calculated with python to be 24.37 but it cannot stay like this! --- .../extraction/SinglePulseExtraction.java | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/main/java/fact/extraction/SinglePulseExtraction.java b/src/main/java/fact/extraction/SinglePulseExtraction.java index 6db293bb4b..71ff8e8179 100644 --- a/src/main/java/fact/extraction/SinglePulseExtraction.java +++ b/src/main/java/fact/extraction/SinglePulseExtraction.java @@ -8,6 +8,11 @@ import java.util.Arrays; import fact.features.singlePulse.timeSeriesExtraction.SinglePulseExtractor; import fact.features.singlePulse.timeSeriesExtraction.ElementWise; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import stream.io.SourceURL; +import stream.io.CsvStream; + /** * Extracts a list of arrival slice positions of photons for each pixel @@ -34,6 +39,9 @@ * and some modifications from Jens Buss */ public class SinglePulseExtraction implements Processor { + + static Logger log = LoggerFactory.getLogger(SinglePulseExtraction.class); + @Parameter(required=true, description="") private String dataKey = null; @@ -83,6 +91,15 @@ public class SinglePulseExtraction implements Processor { private int npix = Constants.NUMBEROFPIXEL; private int roi = 300; + @Parameter( + required = false, + description = "The url to the inputfiles for the gain calibration constants", + defaultValue="classpath:/default/gain_sorted_20131127.csv" + ) + protected SourceURL url = null; + + protected double[] integralGains = null; + @Override public Data process(Data input) { @@ -114,7 +131,7 @@ public Data process(Data input) { ); double[] pixelTimeSeries = ElementWise.multiply( - pixelTimeSeriesInMv, 1.0/config.factSinglePeAmplitudeInMv); + pixelTimeSeriesInMv, 24.37/integralGains[pix]); SinglePulseExtractor.Result result = spe.extractFromTimeSeries( pixelTimeSeries); @@ -148,6 +165,40 @@ private void addStartSliceOffset(int[][] arr) { } } + public double[] loadIntegralGainFile(SourceURL inputUrl, Logger log) { + double[] integralGains = new double[npix]; + Data integralGainData = null; + try { + CsvStream stream = new CsvStream(inputUrl, " "); + stream.setHeader(false); + stream.init(); + integralGainData = stream.readNext(); + + for (int i = 0 ; i < npix ; i++){ + String key = "column:" + (i); + integralGains[i] = (Double) integralGainData.get(key); + } + return integralGains; + + } catch (Exception e) { + log.error("Failed to load integral Gain data: {}", e.getMessage()); + e.printStackTrace(); + return null; + } + } + + public void setUrl(SourceURL url) { + try { + integralGains = loadIntegralGainFile(url,log); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + this.url = url; + } + + public SourceURL getUrl() { + return url; + } public void setDataKey(String dataKey) { From 6359d714c97b7a4420bc685ad502cc907a6933a9 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Fri, 20 Jan 2017 13:47:04 +0100 Subject: [PATCH 02/19] implement some tests --- examples/viewer.xml | 1 + .../extraction/SinglePulseExtraction.java | 9 ++- .../timeSeriesExtraction/TemplatePulse.java | 78 +++++++++++++++++-- .../java/fact/utils/TemplatePulseTest.java | 58 ++++++++++++++ 4 files changed, 139 insertions(+), 7 deletions(-) create mode 100644 src/test/java/fact/utils/TemplatePulseTest.java diff --git a/examples/viewer.xml b/examples/viewer.xml index 71491dc624..4a87079a3f 100644 --- a/examples/viewer.xml +++ b/examples/viewer.xml @@ -25,6 +25,7 @@ height/2.) { + return i - 1; + } + } + return samples.length; + } + + public static double integrate(double[] samples, int start, int length) { + double sum = 0; + for (int i=0; i < length; i++) { + sum += samples[start+i]; + } + return sum; + } + + public static double mean(double[] foo) { + if (foo.length == 0) { return Double.NaN; } + + double m = 0.; + for (double a: foo){ + m += a; + } + return m / foo.length; + } + } \ No newline at end of file diff --git a/src/test/java/fact/utils/TemplatePulseTest.java b/src/test/java/fact/utils/TemplatePulseTest.java new file mode 100644 index 0000000000..1fc8cea435 --- /dev/null +++ b/src/test/java/fact/utils/TemplatePulseTest.java @@ -0,0 +1,58 @@ +package fact.utils; + +import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; +import junit.framework.Assert; +import org.junit.Test; + +public class TemplatePulseTest { + + @Test + public void testIfResultNear24() { + + double result = TemplatePulse.factSinglePePulseIntegral(); + System.out.println(result); + Assert.assertTrue( + Math.abs(result - 24.37) < 0.01 + ); + } + + @Test + public void testIntegrateEmptySamples() { + Throwable e = null; + try { + double[] samples = new double[]{}; + TemplatePulse.integrate(samples, 0, 10); + }catch(Throwable ex) { + e = ex; + } + Assert.assertTrue(e instanceof IndexOutOfBoundsException); + } + + @Test + public void testIntegrate() { + double[] samples = new double[]{1,2,3}; + Assert.assertEquals( + 6., + TemplatePulse.integrate(samples, 0, 3) + ); + } + + @Test + public void testMean() { + double[] foo = new double[]{1,2,3}; + Assert.assertEquals( + 2., + TemplatePulse.mean(foo) + ); + } + + @Test + public void testfindHalfHeightPosition() { + double[] foo = new double[]{0.1, 0.3, 0.7, 1, 1, 0.5, 0.3}; + Assert.assertEquals( + 1, + TemplatePulse.findHalfHeightPosition(foo, 1.) + ); + } + +} \ No newline at end of file From fdeca23c538b8f23347f382bd98174a092560f46 Mon Sep 17 00:00:00 2001 From: sebastian Date: Sat, 21 Jan 2017 21:55:49 +0100 Subject: [PATCH 03/19] create a new calibration service to provide the integral single pulse gains. This is done, because there are two processors (BasicExtraction and SinglePulseExtraction) needing this gain calibration. --- examples/apply_model.xml | 2 +- examples/fellwalker_example.xml | 1 + examples/forErna/std_analysis_erna.xml | 1 + examples/forErna/std_analysis_mc_erna.xml | 1 + examples/mc_viewer.xml | 1 + examples/measure_performance.xml | 1 + examples/stdAnalysis/data/analysis.xml | 3 +- examples/stdAnalysis/mc/analysis_mc.xml | 2 +- .../stdAnalysis/mc/analysis_mc_dispRF.xml | 1 + .../studies/closedShutterGainCalibration.xml | 1 + examples/studies/extractionTest.xml | 1 + examples/studies/fits_writer.xml | 1 + examples/studies/jumpStudy.xml | 1 + examples/studies/muon_fitting.xml | 1 + examples/studies/muon_identification.xml | 1 + examples/studies/pedestalNsbStudy.xml | 3 +- .../singlePeMinimalExample.xml | 1 + .../std_analysis_on_reconstructed_data.xml | 1 + examples/studies/test_saturated_pulses.xml | 5 +- .../studies/timeCalibrationComparison.xml | 1 + examples/viewer.xml | 3 +- .../SinglePulseGainCalibService.java | 69 +++++++++++++ .../java/fact/extraction/BasicExtraction.java | 51 ++-------- .../extraction/SinglePulseExtraction.java | 97 +++++++------------ .../resources/default/data/extraction.xml | 4 +- .../resources/default/mc/extraction_mc.xml | 4 +- .../fact/features/BasicExtractionTest.java | 8 +- .../java/fact/parameter/ParameterTest.java | 7 +- .../parameter/PhotonChargeParameterTest.java | 8 +- 29 files changed, 160 insertions(+), 121 deletions(-) create mode 100644 src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java diff --git a/examples/apply_model.xml b/examples/apply_model.xml index d5d7325d95..71eaa865d6 100644 --- a/examples/apply_model.xml +++ b/examples/apply_model.xml @@ -12,7 +12,7 @@ - + diff --git a/examples/fellwalker_example.xml b/examples/fellwalker_example.xml index a99eaeae5e..a569b2ab1a 100644 --- a/examples/fellwalker_example.xml +++ b/examples/fellwalker_example.xml @@ -10,6 +10,7 @@ + diff --git a/examples/forErna/std_analysis_erna.xml b/examples/forErna/std_analysis_erna.xml index 8c591342ba..5bef51d55a 100644 --- a/examples/forErna/std_analysis_erna.xml +++ b/examples/forErna/std_analysis_erna.xml @@ -11,6 +11,7 @@ + diff --git a/examples/forErna/std_analysis_mc_erna.xml b/examples/forErna/std_analysis_mc_erna.xml index a5983bf9f8..4f9d5585d5 100644 --- a/examples/forErna/std_analysis_mc_erna.xml +++ b/examples/forErna/std_analysis_mc_erna.xml @@ -8,6 +8,7 @@ + diff --git a/examples/mc_viewer.xml b/examples/mc_viewer.xml index 505b1e25f6..bc2a701c6a 100644 --- a/examples/mc_viewer.xml +++ b/examples/mc_viewer.xml @@ -8,6 +8,7 @@ + diff --git a/examples/measure_performance.xml b/examples/measure_performance.xml index daae93157f..3316de5f55 100644 --- a/examples/measure_performance.xml +++ b/examples/measure_performance.xml @@ -7,6 +7,7 @@ + diff --git a/examples/stdAnalysis/data/analysis.xml b/examples/stdAnalysis/data/analysis.xml index 421e3c1cd5..aa70016536 100644 --- a/examples/stdAnalysis/data/analysis.xml +++ b/examples/stdAnalysis/data/analysis.xml @@ -9,8 +9,9 @@ - + + diff --git a/examples/stdAnalysis/mc/analysis_mc.xml b/examples/stdAnalysis/mc/analysis_mc.xml index 990c7dbda8..518642b85c 100644 --- a/examples/stdAnalysis/mc/analysis_mc.xml +++ b/examples/stdAnalysis/mc/analysis_mc.xml @@ -8,7 +8,7 @@ - + diff --git a/examples/stdAnalysis/mc/analysis_mc_dispRF.xml b/examples/stdAnalysis/mc/analysis_mc_dispRF.xml index a997a79cee..4289d213aa 100644 --- a/examples/stdAnalysis/mc/analysis_mc_dispRF.xml +++ b/examples/stdAnalysis/mc/analysis_mc_dispRF.xml @@ -11,6 +11,7 @@ + diff --git a/examples/studies/closedShutterGainCalibration.xml b/examples/studies/closedShutterGainCalibration.xml index b83bd42b17..6371d362c9 100644 --- a/examples/studies/closedShutterGainCalibration.xml +++ b/examples/studies/closedShutterGainCalibration.xml @@ -8,6 +8,7 @@ + diff --git a/examples/studies/extractionTest.xml b/examples/studies/extractionTest.xml index 8bc0ecf7a5..9bb0702fb7 100644 --- a/examples/studies/extractionTest.xml +++ b/examples/studies/extractionTest.xml @@ -6,6 +6,7 @@ + diff --git a/examples/studies/fits_writer.xml b/examples/studies/fits_writer.xml index ceadb9ad04..69d52e3d9b 100644 --- a/examples/studies/fits_writer.xml +++ b/examples/studies/fits_writer.xml @@ -9,6 +9,7 @@ + diff --git a/examples/studies/jumpStudy.xml b/examples/studies/jumpStudy.xml index de1a27149d..419c10205a 100644 --- a/examples/studies/jumpStudy.xml +++ b/examples/studies/jumpStudy.xml @@ -6,6 +6,7 @@ + diff --git a/examples/studies/muon_fitting.xml b/examples/studies/muon_fitting.xml index 8213fed911..b5818806c9 100644 --- a/examples/studies/muon_fitting.xml +++ b/examples/studies/muon_fitting.xml @@ -7,6 +7,7 @@ + diff --git a/examples/studies/muon_identification.xml b/examples/studies/muon_identification.xml index cbb7bc23b7..47a52209fd 100644 --- a/examples/studies/muon_identification.xml +++ b/examples/studies/muon_identification.xml @@ -10,6 +10,7 @@ + diff --git a/examples/studies/pedestalNsbStudy.xml b/examples/studies/pedestalNsbStudy.xml index ddff530cb4..5f5f7d36d1 100644 --- a/examples/studies/pedestalNsbStudy.xml +++ b/examples/studies/pedestalNsbStudy.xml @@ -6,6 +6,7 @@ + @@ -18,7 +19,7 @@ - diff --git a/examples/studies/singlePeExtractor/singlePeMinimalExample.xml b/examples/studies/singlePeExtractor/singlePeMinimalExample.xml index e8cd4bcf66..d073301cc0 100644 --- a/examples/studies/singlePeExtractor/singlePeMinimalExample.xml +++ b/examples/studies/singlePeExtractor/singlePeMinimalExample.xml @@ -11,6 +11,7 @@ + diff --git a/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml b/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml index a842d1492f..d8877a228c 100644 --- a/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml +++ b/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml @@ -11,6 +11,7 @@ + diff --git a/examples/studies/test_saturated_pulses.xml b/examples/studies/test_saturated_pulses.xml index e802852706..a1dbdb41af 100644 --- a/examples/studies/test_saturated_pulses.xml +++ b/examples/studies/test_saturated_pulses.xml @@ -4,6 +4,7 @@ + @@ -19,7 +20,7 @@ - @@ -27,7 +28,7 @@ - diff --git a/examples/studies/timeCalibrationComparison.xml b/examples/studies/timeCalibrationComparison.xml index a98c3b5712..8d0899f6cb 100644 --- a/examples/studies/timeCalibrationComparison.xml +++ b/examples/studies/timeCalibrationComparison.xml @@ -6,6 +6,7 @@ + diff --git a/examples/viewer.xml b/examples/viewer.xml index 4a87079a3f..fa19a0b8c4 100644 --- a/examples/viewer.xml +++ b/examples/viewer.xml @@ -8,6 +8,7 @@ + @@ -25,7 +26,7 @@ Date: Tue, 24 Jan 2017 09:52:39 +0100 Subject: [PATCH 04/19] Implented a processor, to calibrate Timelines from mV to SinglePulseAmplitude; This should be used before BasicExtraction or any extration happens. --- .../SinglePulseGainCalibration.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/main/java/fact/datacorrection/SinglePulseGainCalibration.java diff --git a/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java b/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java new file mode 100644 index 0000000000..1a2b47f00b --- /dev/null +++ b/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java @@ -0,0 +1,94 @@ +package fact.datacorrection; + +import fact.Constants; +import fact.calibrationservice.SinglePulseGainCalibService; +import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; +import stream.Data; +import stream.ProcessContext; +import stream.StatefulProcessor; +import stream.annotations.Parameter; + +/** + * Calibrates the unit of TimeLines from mV to SinglePulseAmplitude. + * + * This means, after this calibration the *height* or *amplitude* of a single pulse is 1. + * This means further, that the integral typically performed by BasicExtraction, + * will not be 1. It will instead have the same value, the *TemplatePulse* has, when integrated + * the same way. + */ +public class SinglePulseGainCalibration implements StatefulProcessor{ + + @Parameter( + required = true, + description = "Name of the output array" + ) + private String dataKey = "DataCalibrated"; + + @Parameter( + required = true, + description = "Name of input array" + ) + private String outputKey = "DataCalibrated"; + + @Parameter( + required = true, + description = "The calibration service for the integral single pulse gain" + ) + private SinglePulseGainCalibService gainService; + private double[] gainCorrection; + + @Override + public void init(ProcessContext processContext) throws Exception { + double factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); // unit: SinglePulseAmplitude * slices + if(gainService == null) { + gainCorrection = new double[Constants.NUMBEROFPIXEL]; + for(int i=0; i value: ~10 + } + + } + + @Override + public Data process(Data input) { + double[] TimeLines = ((double[]) input.get(dataKey)).clone(); + + final int npix = Constants.NUMBEROFPIXEL; + final int roi = TimeLines.length / npix; + + for (int pix = 0; pix < npix; pix++) { + for (int slice =0; slice < roi; slice++){ + TimeLines[pix*roi + slice] /= gainCorrection[pix]; + // now TimeLines unit: SinglePulseAmplitude, i.e. a single pulse has amplitude 1 in this units. + } + } + + input.put(outputKey, TimeLines); + return input; + } + + + @Override + public void resetState() throws Exception { + // empty on purpose + } + + @Override + public void finish() throws Exception { + // empty on purpose + } + + public void setDataKey(String dataKey) { + this.dataKey = dataKey; + } + + public void setOutputKey(String outputKey) { + this.outputKey = outputKey; + } + + public void setGainService(SinglePulseGainCalibService gainService) { + this.gainService = gainService; + } +} From d44ad0ebf7d21e305da27f9595ff96605e204138 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 24 Jan 2017 11:19:35 +0100 Subject: [PATCH 05/19] use SinglePulseGainCalibration before BasicExtraction BasicExtraction became a StatefulProcessor now: - to calculate the template integral ParameterTest uses SinglePulseGaincalibration before extraction starts. --- .../java/fact/extraction/BasicExtraction.java | 34 ++++++---- .../extraction/TimeGradientExtraction.java | 2 +- .../fact/features/BasicExtractionTest.java | 1 - .../java/fact/parameter/ParameterTest.java | 68 ++++++++++--------- .../parameter/PhotonChargeParameterTest.java | 11 +-- 5 files changed, 62 insertions(+), 54 deletions(-) diff --git a/src/main/java/fact/extraction/BasicExtraction.java b/src/main/java/fact/extraction/BasicExtraction.java index 8b03d5beca..52736ce3bc 100644 --- a/src/main/java/fact/extraction/BasicExtraction.java +++ b/src/main/java/fact/extraction/BasicExtraction.java @@ -2,12 +2,13 @@ import fact.Constants; import fact.Utils; -import fact.calibrationservice.SinglePulseGainCalibService; +import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; import org.jfree.chart.plot.IntervalMarker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import stream.Data; -import stream.Processor; +import stream.ProcessContext; +import stream.StatefulProcessor; import stream.annotations.Parameter; /** @@ -22,7 +23,7 @@ * @author Fabian Temme * */ -public class BasicExtraction implements Processor { +public class BasicExtraction implements StatefulProcessor{ static Logger log = LoggerFactory.getLogger(BasicExtraction.class); @Parameter(required = true, description="key to the data array") protected String dataKey = null; @@ -30,8 +31,6 @@ public class BasicExtraction implements Processor { protected String outputKeyMaxAmplPos = null; @Parameter(required = true, description="outputKey for the calculated photoncharge") protected String outputKeyPhotonCharge = null; - @Parameter(required = true, description = "The calibration service for the integral single pulse gain") - SinglePulseGainCalibService gainService; @Parameter(required = false, description="start slice of the search window for the max amplitude", defaultValue="35") protected int startSearchWindow = 35; @Parameter(required = false, description="range of the search window for the max amplitude", defaultValue="90") @@ -46,6 +45,7 @@ public class BasicExtraction implements Processor { protected double[] integralGains = null; private int npix = Constants.NUMBEROFPIXEL; + public double factSinglePePulseIntegral; @Override public Data process(Data input) { @@ -53,7 +53,6 @@ public Data process(Data input) { int roi = (Integer) input.get("NROI"); npix = (Integer) input.get("NPIX"); - integralGains = gainService.getIntegralSinglePulseGain(); double[] data = (double[]) input.get(dataKey); @@ -67,11 +66,11 @@ public Data process(Data input) { for (int pix = 0; pix < npix; pix++) { positions[pix] = calculateMaxPosition(pix, startSearchWindow, startSearchWindow+rangeSearchWindow, roi, data); mPositions[pix] = new IntervalMarker(positions[pix],positions[pix] + 1); - + int halfHeightPos = calculatePositionHalfHeight(pix, positions[pix],positions[pix]-rangeHalfHeightWindow, roi, data); Utils.checkWindow(halfHeightPos, integrationWindow, validMinimalSlice, roi); - photonCharge[pix] = calculateIntegral(pix, halfHeightPos, integrationWindow, roi, data) / integralGains[pix]; + photonCharge[pix] = calculateIntegral(pix, halfHeightPos, integrationWindow, roi, data) / factSinglePePulseIntegral; mPhotonCharge[pix] = new IntervalMarker(halfHeightPos,halfHeightPos + integrationWindow); } input.put(outputKeyMaxAmplPos, positions); @@ -141,10 +140,6 @@ public void setDataKey(String dataKey) { this.dataKey = dataKey; } - public void setGainService(SinglePulseGainCalibService gainService) { - this.gainService = gainService; - } - public String getOutputKeyMaxAmplPos() { return outputKeyMaxAmplPos; } @@ -201,4 +196,19 @@ public int getValidMinimalSlice() { public void setValidMinimalSlice(int validMinimalSlice) { this.validMinimalSlice = validMinimalSlice; } + + @Override + public void init(ProcessContext processContext) throws Exception { + factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); // unit: SinglePulseAmplitude * slices + } + + @Override + public void resetState() throws Exception { + + } + + @Override + public void finish() throws Exception { + + } } diff --git a/src/main/java/fact/extraction/TimeGradientExtraction.java b/src/main/java/fact/extraction/TimeGradientExtraction.java index 4dee031b27..f7930170b1 100644 --- a/src/main/java/fact/extraction/TimeGradientExtraction.java +++ b/src/main/java/fact/extraction/TimeGradientExtraction.java @@ -69,7 +69,7 @@ public Data process(Data input) { int halfHeightPos = calculatePositionHalfHeight(px, positions[px],positions[px]-rangeHalfHeightWindow, roi, data); Utils.checkWindow(halfHeightPos, integrationWindow, validMinimalSlice, roi); - photonCharge[px] = calculateIntegral(px, halfHeightPos, integrationWindow, roi, data) / integralGains[px]; + photonCharge[px] = calculateIntegral(px, halfHeightPos, integrationWindow, roi, data) / factSinglePePulseIntegral; mPhotonCharge[px] = new IntervalMarker(halfHeightPos,halfHeightPos + integrationWindow); } diff --git a/src/test/java/fact/features/BasicExtractionTest.java b/src/test/java/fact/features/BasicExtractionTest.java index af148330c1..26157d5e4e 100644 --- a/src/test/java/fact/features/BasicExtractionTest.java +++ b/src/test/java/fact/features/BasicExtractionTest.java @@ -38,7 +38,6 @@ public void setup() throws Exception { extraction.setDataKey("test"); extraction.setOutputKeyMaxAmplPos(positionsKey); extraction.setOutputKeyPhotonCharge(photonChargeKey); - extraction.setGainService(igs); } @Test diff --git a/src/test/java/fact/parameter/ParameterTest.java b/src/test/java/fact/parameter/ParameterTest.java index 48e5b628fd..38e134f47a 100644 --- a/src/test/java/fact/parameter/ParameterTest.java +++ b/src/test/java/fact/parameter/ParameterTest.java @@ -2,6 +2,7 @@ import fact.calibrationservice.ConstantCalibService; import fact.cleaning.TwoLevelTimeMedian; +import fact.datacorrection.SinglePulseGainCalibration; import fact.extraction.BasicExtraction; import fact.extraction.RisingEdgeForPositions; import fact.features.DistributionFromShower; @@ -62,42 +63,47 @@ public void setUp() throws Exception { e.printStackTrace(); } - URL drsUrl = FITSStreamTest.class - .getResource("/testDrsFile.drs.fits.gz"); - DrsCalibration pr = new DrsCalibration(); - pr.setUrl(new SourceURL(drsUrl)); - pr.setOutputKey(key); + URL drsUrl = FITSStreamTest.class + .getResource("/testDrsFile.drs.fits.gz"); + DrsCalibration pr = new DrsCalibration(); + pr.setUrl(new SourceURL(drsUrl)); + pr.setOutputKey(key); pr.init(null); - pr.process(item); + pr.process(item); SinglePulseGainCalibService igs = new SinglePulseGainCalibService(); igs.setIntegralGainFile(new SourceURL(FITSStreamTest.class.getResource("/defaultIntegralGains.csv"))); - BasicExtraction bE = new BasicExtraction(); - bE.setDataKey(key); - bE.setOutputKeyMaxAmplPos(positions); - bE.setOutputKeyPhotonCharge(photonCharge); - bE.setGainService(igs); - bE.process(item); - - RisingEdgeForPositions pR = new RisingEdgeForPositions(); - pR.setDataKey(key); - pR.setAmplitudePositionsKey(positions); - pR.setOutputKey(arrivalTime); - pR.process(item); - - TwoLevelTimeMedian poser = new TwoLevelTimeMedian(); - poser.setCalibService(calibService); - poser.setPhotonChargeKey(photonCharge); - poser.setArrivalTimeKey(arrivalTime); - poser.setOutputKey(shower); - poser.setCorePixelThreshold(1); - poser.setNeighborPixelThreshold(0.1); - poser.setMinNumberOfPixel(1); - poser.setTimeLimit(40); - poser.process(item); - - + SinglePulseGainCalibration spgc = new SinglePulseGainCalibration(); + spgc.setGainService(igs); + spgc.setDataKey(key); + spgc.setOutputKey(key); + spgc.init(null); + spgc.process(item); + + BasicExtraction bE = new BasicExtraction(); + bE.setDataKey(key); + bE.setOutputKeyMaxAmplPos(positions); + bE.setOutputKeyPhotonCharge(photonCharge); + bE.init(null); + bE.process(item); + + RisingEdgeForPositions pR = new RisingEdgeForPositions(); + pR.setDataKey(key); + pR.setAmplitudePositionsKey(positions); + pR.setOutputKey(arrivalTime); + pR.process(item); + + TwoLevelTimeMedian poser = new TwoLevelTimeMedian(); + poser.setCalibService(calibService); + poser.setPhotonChargeKey(photonCharge); + poser.setArrivalTimeKey(arrivalTime); + poser.setOutputKey(shower); + poser.setCorePixelThreshold(1); + poser.setNeighborPixelThreshold(0.1); + poser.setMinNumberOfPixel(1); + poser.setTimeLimit(40); + poser.process(item); DistributionFromShower dist = new DistributionFromShower(); dist.setPixelSetKey(shower); diff --git a/src/test/java/fact/parameter/PhotonChargeParameterTest.java b/src/test/java/fact/parameter/PhotonChargeParameterTest.java index 3b8d173013..9657cd0a91 100644 --- a/src/test/java/fact/parameter/PhotonChargeParameterTest.java +++ b/src/test/java/fact/parameter/PhotonChargeParameterTest.java @@ -1,12 +1,9 @@ package fact.parameter; import fact.extraction.BasicExtraction; -import fact.io.FITSStreamTest; -import fact.calibrationservice.SinglePulseGainCalibService; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import stream.io.SourceURL; import static org.junit.Assert.assertTrue; @@ -27,19 +24,15 @@ public class PhotonChargeParameterTest extends ParameterTest { @Test public void testValidParameter() throws Exception { // //start processor with the correct parameter - assertTrue("Expecteds output already in data item", + assertTrue("Expected output already in data item", !item.containsKey(outputKey)); - SinglePulseGainCalibService igs = new SinglePulseGainCalibService(); - igs.setIntegralGainFile(new SourceURL(FITSStreamTest.class.getResource("/defaultIntegralGains.csv"))); - BasicExtraction extraction = new BasicExtraction(); extraction.setDataKey(key); extraction.setOutputKeyMaxAmplPos(positions); extraction.setOutputKeyPhotonCharge(outputKey); - extraction.setGainService(igs); extraction.process(item); - assertTrue("Expecteds output not in data item but it should be there", + assertTrue("Expected output not in data item but it should be there", item.containsKey(outputKey)); // item.remove(outputKey); } From aacfc67a4d95650d2ab890294f4846eb0b1189cb Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 24 Jan 2017 11:35:42 +0100 Subject: [PATCH 06/19] Get rid of SinglePulseCalibration inside SinglepulseExtraction --- .../extraction/SinglePulseExtraction.java | 56 ++----------------- 1 file changed, 5 insertions(+), 51 deletions(-) diff --git a/src/main/java/fact/extraction/SinglePulseExtraction.java b/src/main/java/fact/extraction/SinglePulseExtraction.java index 1cbd903e3e..27591b1e4a 100644 --- a/src/main/java/fact/extraction/SinglePulseExtraction.java +++ b/src/main/java/fact/extraction/SinglePulseExtraction.java @@ -3,15 +3,11 @@ import fact.Constants; import fact.Utils; import stream.Data; -import stream.StatefulProcessor; -import stream.ProcessContext; +import stream.Processor; import stream.annotations.Parameter; import java.util.Arrays; -import java.net.URL; import fact.features.singlePulse.timeSeriesExtraction.SinglePulseExtractor; import fact.features.singlePulse.timeSeriesExtraction.ElementWise; -import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; -import fact.calibrationservice.SinglePulseGainCalibService; /** * Extracts a list of arrival slice positions of photons for each pixel @@ -37,7 +33,7 @@ * Created by Sebastian Mueller * and some modifications from Jens Buss */ -public class SinglePulseExtraction implements StatefulProcessor { +public class SinglePulseExtraction implements Processor { @Parameter(required=true, description="") private String dataKey = null; @@ -85,37 +81,10 @@ public class SinglePulseExtraction implements StatefulProcessor { ) protected int extractionWindowLengthInSlices = 225; - @Parameter( - required = false, - description = "The calibration service for the integral single pulse gain", - defaultValue = "null") - SinglePulseGainCalibService gainService = null; - protected double[] gainCorrection = null; - protected double factSinglePePulseIntegral; private int npix = Constants.NUMBEROFPIXEL; private int roi = 300; - @Override - public void init(ProcessContext processContext) throws Exception { - - factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); - - if(gainService == null) { - gainCorrection = new double[npix]; - for(int i=0; i Date: Wed, 25 Jan 2017 17:32:20 +0100 Subject: [PATCH 07/19] Revert "Get rid of SinglePulseCalibration inside SinglepulseExtraction" This reverts commit aacfc67a4d95650d2ab890294f4846eb0b1189cb. --- .../extraction/SinglePulseExtraction.java | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/src/main/java/fact/extraction/SinglePulseExtraction.java b/src/main/java/fact/extraction/SinglePulseExtraction.java index 27591b1e4a..1cbd903e3e 100644 --- a/src/main/java/fact/extraction/SinglePulseExtraction.java +++ b/src/main/java/fact/extraction/SinglePulseExtraction.java @@ -3,11 +3,15 @@ import fact.Constants; import fact.Utils; import stream.Data; -import stream.Processor; +import stream.StatefulProcessor; +import stream.ProcessContext; import stream.annotations.Parameter; import java.util.Arrays; +import java.net.URL; import fact.features.singlePulse.timeSeriesExtraction.SinglePulseExtractor; import fact.features.singlePulse.timeSeriesExtraction.ElementWise; +import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; +import fact.calibrationservice.SinglePulseGainCalibService; /** * Extracts a list of arrival slice positions of photons for each pixel @@ -33,7 +37,7 @@ * Created by Sebastian Mueller * and some modifications from Jens Buss */ -public class SinglePulseExtraction implements Processor { +public class SinglePulseExtraction implements StatefulProcessor { @Parameter(required=true, description="") private String dataKey = null; @@ -81,10 +85,37 @@ public class SinglePulseExtraction implements Processor { ) protected int extractionWindowLengthInSlices = 225; + @Parameter( + required = false, + description = "The calibration service for the integral single pulse gain", + defaultValue = "null") + SinglePulseGainCalibService gainService = null; + protected double[] gainCorrection = null; + protected double factSinglePePulseIntegral; private int npix = Constants.NUMBEROFPIXEL; private int roi = 300; + @Override + public void init(ProcessContext processContext) throws Exception { + + factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); + + if(gainService == null) { + gainCorrection = new double[npix]; + for(int i=0; i Date: Wed, 25 Jan 2017 17:32:39 +0100 Subject: [PATCH 08/19] Revert "use SinglePulseGainCalibration before BasicExtraction" This reverts commit d44ad0ebf7d21e305da27f9595ff96605e204138. --- .../java/fact/extraction/BasicExtraction.java | 34 ++++------ .../extraction/TimeGradientExtraction.java | 2 +- .../fact/features/BasicExtractionTest.java | 1 + .../java/fact/parameter/ParameterTest.java | 68 +++++++++---------- .../parameter/PhotonChargeParameterTest.java | 11 ++- 5 files changed, 54 insertions(+), 62 deletions(-) diff --git a/src/main/java/fact/extraction/BasicExtraction.java b/src/main/java/fact/extraction/BasicExtraction.java index 52736ce3bc..8b03d5beca 100644 --- a/src/main/java/fact/extraction/BasicExtraction.java +++ b/src/main/java/fact/extraction/BasicExtraction.java @@ -2,13 +2,12 @@ import fact.Constants; import fact.Utils; -import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; +import fact.calibrationservice.SinglePulseGainCalibService; import org.jfree.chart.plot.IntervalMarker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import stream.Data; -import stream.ProcessContext; -import stream.StatefulProcessor; +import stream.Processor; import stream.annotations.Parameter; /** @@ -23,7 +22,7 @@ * @author Fabian Temme * */ -public class BasicExtraction implements StatefulProcessor{ +public class BasicExtraction implements Processor { static Logger log = LoggerFactory.getLogger(BasicExtraction.class); @Parameter(required = true, description="key to the data array") protected String dataKey = null; @@ -31,6 +30,8 @@ public class BasicExtraction implements StatefulProcessor{ protected String outputKeyMaxAmplPos = null; @Parameter(required = true, description="outputKey for the calculated photoncharge") protected String outputKeyPhotonCharge = null; + @Parameter(required = true, description = "The calibration service for the integral single pulse gain") + SinglePulseGainCalibService gainService; @Parameter(required = false, description="start slice of the search window for the max amplitude", defaultValue="35") protected int startSearchWindow = 35; @Parameter(required = false, description="range of the search window for the max amplitude", defaultValue="90") @@ -45,7 +46,6 @@ public class BasicExtraction implements StatefulProcessor{ protected double[] integralGains = null; private int npix = Constants.NUMBEROFPIXEL; - public double factSinglePePulseIntegral; @Override public Data process(Data input) { @@ -53,6 +53,7 @@ public Data process(Data input) { int roi = (Integer) input.get("NROI"); npix = (Integer) input.get("NPIX"); + integralGains = gainService.getIntegralSinglePulseGain(); double[] data = (double[]) input.get(dataKey); @@ -66,11 +67,11 @@ public Data process(Data input) { for (int pix = 0; pix < npix; pix++) { positions[pix] = calculateMaxPosition(pix, startSearchWindow, startSearchWindow+rangeSearchWindow, roi, data); mPositions[pix] = new IntervalMarker(positions[pix],positions[pix] + 1); - + int halfHeightPos = calculatePositionHalfHeight(pix, positions[pix],positions[pix]-rangeHalfHeightWindow, roi, data); Utils.checkWindow(halfHeightPos, integrationWindow, validMinimalSlice, roi); - photonCharge[pix] = calculateIntegral(pix, halfHeightPos, integrationWindow, roi, data) / factSinglePePulseIntegral; + photonCharge[pix] = calculateIntegral(pix, halfHeightPos, integrationWindow, roi, data) / integralGains[pix]; mPhotonCharge[pix] = new IntervalMarker(halfHeightPos,halfHeightPos + integrationWindow); } input.put(outputKeyMaxAmplPos, positions); @@ -140,6 +141,10 @@ public void setDataKey(String dataKey) { this.dataKey = dataKey; } + public void setGainService(SinglePulseGainCalibService gainService) { + this.gainService = gainService; + } + public String getOutputKeyMaxAmplPos() { return outputKeyMaxAmplPos; } @@ -196,19 +201,4 @@ public int getValidMinimalSlice() { public void setValidMinimalSlice(int validMinimalSlice) { this.validMinimalSlice = validMinimalSlice; } - - @Override - public void init(ProcessContext processContext) throws Exception { - factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); // unit: SinglePulseAmplitude * slices - } - - @Override - public void resetState() throws Exception { - - } - - @Override - public void finish() throws Exception { - - } } diff --git a/src/main/java/fact/extraction/TimeGradientExtraction.java b/src/main/java/fact/extraction/TimeGradientExtraction.java index f7930170b1..4dee031b27 100644 --- a/src/main/java/fact/extraction/TimeGradientExtraction.java +++ b/src/main/java/fact/extraction/TimeGradientExtraction.java @@ -69,7 +69,7 @@ public Data process(Data input) { int halfHeightPos = calculatePositionHalfHeight(px, positions[px],positions[px]-rangeHalfHeightWindow, roi, data); Utils.checkWindow(halfHeightPos, integrationWindow, validMinimalSlice, roi); - photonCharge[px] = calculateIntegral(px, halfHeightPos, integrationWindow, roi, data) / factSinglePePulseIntegral; + photonCharge[px] = calculateIntegral(px, halfHeightPos, integrationWindow, roi, data) / integralGains[px]; mPhotonCharge[px] = new IntervalMarker(halfHeightPos,halfHeightPos + integrationWindow); } diff --git a/src/test/java/fact/features/BasicExtractionTest.java b/src/test/java/fact/features/BasicExtractionTest.java index 26157d5e4e..af148330c1 100644 --- a/src/test/java/fact/features/BasicExtractionTest.java +++ b/src/test/java/fact/features/BasicExtractionTest.java @@ -38,6 +38,7 @@ public void setup() throws Exception { extraction.setDataKey("test"); extraction.setOutputKeyMaxAmplPos(positionsKey); extraction.setOutputKeyPhotonCharge(photonChargeKey); + extraction.setGainService(igs); } @Test diff --git a/src/test/java/fact/parameter/ParameterTest.java b/src/test/java/fact/parameter/ParameterTest.java index 38e134f47a..48e5b628fd 100644 --- a/src/test/java/fact/parameter/ParameterTest.java +++ b/src/test/java/fact/parameter/ParameterTest.java @@ -2,7 +2,6 @@ import fact.calibrationservice.ConstantCalibService; import fact.cleaning.TwoLevelTimeMedian; -import fact.datacorrection.SinglePulseGainCalibration; import fact.extraction.BasicExtraction; import fact.extraction.RisingEdgeForPositions; import fact.features.DistributionFromShower; @@ -63,47 +62,42 @@ public void setUp() throws Exception { e.printStackTrace(); } - URL drsUrl = FITSStreamTest.class - .getResource("/testDrsFile.drs.fits.gz"); - DrsCalibration pr = new DrsCalibration(); - pr.setUrl(new SourceURL(drsUrl)); - pr.setOutputKey(key); + URL drsUrl = FITSStreamTest.class + .getResource("/testDrsFile.drs.fits.gz"); + DrsCalibration pr = new DrsCalibration(); + pr.setUrl(new SourceURL(drsUrl)); + pr.setOutputKey(key); pr.init(null); - pr.process(item); + pr.process(item); SinglePulseGainCalibService igs = new SinglePulseGainCalibService(); igs.setIntegralGainFile(new SourceURL(FITSStreamTest.class.getResource("/defaultIntegralGains.csv"))); - SinglePulseGainCalibration spgc = new SinglePulseGainCalibration(); - spgc.setGainService(igs); - spgc.setDataKey(key); - spgc.setOutputKey(key); - spgc.init(null); - spgc.process(item); - - BasicExtraction bE = new BasicExtraction(); - bE.setDataKey(key); - bE.setOutputKeyMaxAmplPos(positions); - bE.setOutputKeyPhotonCharge(photonCharge); - bE.init(null); - bE.process(item); - - RisingEdgeForPositions pR = new RisingEdgeForPositions(); - pR.setDataKey(key); - pR.setAmplitudePositionsKey(positions); - pR.setOutputKey(arrivalTime); - pR.process(item); - - TwoLevelTimeMedian poser = new TwoLevelTimeMedian(); - poser.setCalibService(calibService); - poser.setPhotonChargeKey(photonCharge); - poser.setArrivalTimeKey(arrivalTime); - poser.setOutputKey(shower); - poser.setCorePixelThreshold(1); - poser.setNeighborPixelThreshold(0.1); - poser.setMinNumberOfPixel(1); - poser.setTimeLimit(40); - poser.process(item); + BasicExtraction bE = new BasicExtraction(); + bE.setDataKey(key); + bE.setOutputKeyMaxAmplPos(positions); + bE.setOutputKeyPhotonCharge(photonCharge); + bE.setGainService(igs); + bE.process(item); + + RisingEdgeForPositions pR = new RisingEdgeForPositions(); + pR.setDataKey(key); + pR.setAmplitudePositionsKey(positions); + pR.setOutputKey(arrivalTime); + pR.process(item); + + TwoLevelTimeMedian poser = new TwoLevelTimeMedian(); + poser.setCalibService(calibService); + poser.setPhotonChargeKey(photonCharge); + poser.setArrivalTimeKey(arrivalTime); + poser.setOutputKey(shower); + poser.setCorePixelThreshold(1); + poser.setNeighborPixelThreshold(0.1); + poser.setMinNumberOfPixel(1); + poser.setTimeLimit(40); + poser.process(item); + + DistributionFromShower dist = new DistributionFromShower(); dist.setPixelSetKey(shower); diff --git a/src/test/java/fact/parameter/PhotonChargeParameterTest.java b/src/test/java/fact/parameter/PhotonChargeParameterTest.java index 9657cd0a91..3b8d173013 100644 --- a/src/test/java/fact/parameter/PhotonChargeParameterTest.java +++ b/src/test/java/fact/parameter/PhotonChargeParameterTest.java @@ -1,9 +1,12 @@ package fact.parameter; import fact.extraction.BasicExtraction; +import fact.io.FITSStreamTest; +import fact.calibrationservice.SinglePulseGainCalibService; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import stream.io.SourceURL; import static org.junit.Assert.assertTrue; @@ -24,15 +27,19 @@ public class PhotonChargeParameterTest extends ParameterTest { @Test public void testValidParameter() throws Exception { // //start processor with the correct parameter - assertTrue("Expected output already in data item", + assertTrue("Expecteds output already in data item", !item.containsKey(outputKey)); + SinglePulseGainCalibService igs = new SinglePulseGainCalibService(); + igs.setIntegralGainFile(new SourceURL(FITSStreamTest.class.getResource("/defaultIntegralGains.csv"))); + BasicExtraction extraction = new BasicExtraction(); extraction.setDataKey(key); extraction.setOutputKeyMaxAmplPos(positions); extraction.setOutputKeyPhotonCharge(outputKey); + extraction.setGainService(igs); extraction.process(item); - assertTrue("Expected output not in data item but it should be there", + assertTrue("Expecteds output not in data item but it should be there", item.containsKey(outputKey)); // item.remove(outputKey); } From ba86cd87703c6bc54def63366a8edfca6a89d3d3 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 17:32:59 +0100 Subject: [PATCH 09/19] Revert "Implented a processor, to calibrate Timelines" This reverts commit b52aa7d9e06b864d7fa277528970236b9969ccdf. --- .../SinglePulseGainCalibration.java | 94 ------------------- 1 file changed, 94 deletions(-) delete mode 100644 src/main/java/fact/datacorrection/SinglePulseGainCalibration.java diff --git a/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java b/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java deleted file mode 100644 index 1a2b47f00b..0000000000 --- a/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java +++ /dev/null @@ -1,94 +0,0 @@ -package fact.datacorrection; - -import fact.Constants; -import fact.calibrationservice.SinglePulseGainCalibService; -import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; -import stream.Data; -import stream.ProcessContext; -import stream.StatefulProcessor; -import stream.annotations.Parameter; - -/** - * Calibrates the unit of TimeLines from mV to SinglePulseAmplitude. - * - * This means, after this calibration the *height* or *amplitude* of a single pulse is 1. - * This means further, that the integral typically performed by BasicExtraction, - * will not be 1. It will instead have the same value, the *TemplatePulse* has, when integrated - * the same way. - */ -public class SinglePulseGainCalibration implements StatefulProcessor{ - - @Parameter( - required = true, - description = "Name of the output array" - ) - private String dataKey = "DataCalibrated"; - - @Parameter( - required = true, - description = "Name of input array" - ) - private String outputKey = "DataCalibrated"; - - @Parameter( - required = true, - description = "The calibration service for the integral single pulse gain" - ) - private SinglePulseGainCalibService gainService; - private double[] gainCorrection; - - @Override - public void init(ProcessContext processContext) throws Exception { - double factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); // unit: SinglePulseAmplitude * slices - if(gainService == null) { - gainCorrection = new double[Constants.NUMBEROFPIXEL]; - for(int i=0; i value: ~10 - } - - } - - @Override - public Data process(Data input) { - double[] TimeLines = ((double[]) input.get(dataKey)).clone(); - - final int npix = Constants.NUMBEROFPIXEL; - final int roi = TimeLines.length / npix; - - for (int pix = 0; pix < npix; pix++) { - for (int slice =0; slice < roi; slice++){ - TimeLines[pix*roi + slice] /= gainCorrection[pix]; - // now TimeLines unit: SinglePulseAmplitude, i.e. a single pulse has amplitude 1 in this units. - } - } - - input.put(outputKey, TimeLines); - return input; - } - - - @Override - public void resetState() throws Exception { - // empty on purpose - } - - @Override - public void finish() throws Exception { - // empty on purpose - } - - public void setDataKey(String dataKey) { - this.dataKey = dataKey; - } - - public void setOutputKey(String outputKey) { - this.outputKey = outputKey; - } - - public void setGainService(SinglePulseGainCalibService gainService) { - this.gainService = gainService; - } -} From 4ba4614cd10f249601671fdf54037a9dd8f74748 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 18:03:36 +0100 Subject: [PATCH 10/19] fix imports for TemplatePulse --- src/main/java/fact/photonstream/SinglePulseExtraction.java | 1 + src/test/java/fact/utils/TemplatePulseTest.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/fact/photonstream/SinglePulseExtraction.java b/src/main/java/fact/photonstream/SinglePulseExtraction.java index cbc94bd694..668be00f49 100644 --- a/src/main/java/fact/photonstream/SinglePulseExtraction.java +++ b/src/main/java/fact/photonstream/SinglePulseExtraction.java @@ -2,6 +2,7 @@ import fact.Constants; import fact.Utils; +import fact.photonstream.timeSeriesExtraction.TemplatePulse; import stream.Data; import stream.StatefulProcessor; import stream.ProcessContext; diff --git a/src/test/java/fact/utils/TemplatePulseTest.java b/src/test/java/fact/utils/TemplatePulseTest.java index 1fc8cea435..ff1b628b20 100644 --- a/src/test/java/fact/utils/TemplatePulseTest.java +++ b/src/test/java/fact/utils/TemplatePulseTest.java @@ -1,6 +1,7 @@ package fact.utils; -import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; + +import fact.photonstream.timeSeriesExtraction.TemplatePulse; import junit.framework.Assert; import org.junit.Test; From 44fa913515bb42d300ea67d391825a2327b04ec9 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 18:06:19 +0100 Subject: [PATCH 11/19] use @Service for service --- src/main/java/fact/extraction/BasicExtraction.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/fact/extraction/BasicExtraction.java b/src/main/java/fact/extraction/BasicExtraction.java index 8b03d5beca..45e79ca0ef 100644 --- a/src/main/java/fact/extraction/BasicExtraction.java +++ b/src/main/java/fact/extraction/BasicExtraction.java @@ -9,6 +9,7 @@ import stream.Data; import stream.Processor; import stream.annotations.Parameter; +import stream.annotations.Service; /** * This processor performs a basic extraction on the data array. It contains three steps: @@ -30,7 +31,7 @@ public class BasicExtraction implements Processor { protected String outputKeyMaxAmplPos = null; @Parameter(required = true, description="outputKey for the calculated photoncharge") protected String outputKeyPhotonCharge = null; - @Parameter(required = true, description = "The calibration service for the integral single pulse gain") + @Service(required = true, description = "The calibration service for the integral single pulse gain") SinglePulseGainCalibService gainService; @Parameter(required = false, description="start slice of the search window for the max amplitude", defaultValue="35") protected int startSearchWindow = 35; From b24928a7d6fd134fd949bfe46bdc079f61a5b04f Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 18:11:01 +0100 Subject: [PATCH 12/19] use @Service for service; remove defeault_value; WS --- .../java/fact/photonstream/SinglePulseExtraction.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/fact/photonstream/SinglePulseExtraction.java b/src/main/java/fact/photonstream/SinglePulseExtraction.java index 668be00f49..08b943dbfc 100644 --- a/src/main/java/fact/photonstream/SinglePulseExtraction.java +++ b/src/main/java/fact/photonstream/SinglePulseExtraction.java @@ -11,6 +11,7 @@ import fact.calibrationservice.SinglePulseGainCalibService; import fact.photonstream.timeSeriesExtraction.SinglePulseExtractor; import fact.photonstream.timeSeriesExtraction.ElementWise; +import stream.annotations.Service; /** * Extracts a list of arrival slice positions of photons for each pixel @@ -84,10 +85,10 @@ public class SinglePulseExtraction implements StatefulProcessor { ) protected int extractionWindowLengthInSlices = 225; - @Parameter( - required = false, + @Service( + required = false, description = "The calibration service for the integral single pulse gain", - defaultValue = "null") + ) SinglePulseGainCalibService gainService = null; protected double[] gainCorrection = null; @@ -99,7 +100,7 @@ public class SinglePulseExtraction implements StatefulProcessor { public void init(ProcessContext processContext) throws Exception { factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); - + if(gainService == null) { gainCorrection = new double[npix]; for(int i=0; i Date: Wed, 25 Jan 2017 18:11:46 +0100 Subject: [PATCH 13/19] typo, sorry --- src/main/java/fact/photonstream/SinglePulseExtraction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fact/photonstream/SinglePulseExtraction.java b/src/main/java/fact/photonstream/SinglePulseExtraction.java index 08b943dbfc..c047baf0c8 100644 --- a/src/main/java/fact/photonstream/SinglePulseExtraction.java +++ b/src/main/java/fact/photonstream/SinglePulseExtraction.java @@ -87,7 +87,7 @@ public class SinglePulseExtraction implements StatefulProcessor { @Service( required = false, - description = "The calibration service for the integral single pulse gain", + description = "The calibration service for the integral single pulse gain" ) SinglePulseGainCalibService gainService = null; From b4bd6c93647658b570a756d7cd6e2244057a4944 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 18:17:47 +0100 Subject: [PATCH 14/19] convert indents to spaces --- .../SinglePulseGainCalibService.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java b/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java index c1ebe57c9c..0ada73f3b8 100644 --- a/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java +++ b/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java @@ -20,12 +20,12 @@ public class SinglePulseGainCalibService implements CalibrationService { public double[] integralSinglePulseGain; @Parameter( - required = false, - description = "The path to the integral single pulse gain file." + required = false, + description = "The path to the integral single pulse gain file." ) SourceURL integralGainFile; - - public void init() { + + public void init() { integralSinglePulseGain = new double[Constants.NUMBEROFPIXEL]; Data integralGainData = null; try { @@ -41,19 +41,19 @@ public void init() { } catch (Exception e) { log.error( - "Failed to load the integral single pulse gain file: {}", - e.getMessage()); + "Failed to load the integral single pulse gain file: {}", + e.getMessage()); e.printStackTrace(); } } - public double[] getIntegralSinglePulseGain() { - if (isInit == false){ - init(); - isInit = true; - } - return integralSinglePulseGain; - } + public double[] getIntegralSinglePulseGain() { + if (isInit == false){ + init(); + isInit = true; + } + return integralSinglePulseGain; + } @Override public void reset() throws Exception { @@ -63,7 +63,7 @@ public void setIntegralGainFile(SourceURL integralGainFile) { this.integralGainFile = integralGainFile; } - public int[] getBadPixel(DateTime eventTimeStamp) {return new int[0];} - - public int[] getNotUsablePixels(DateTime eventTimeStamp) {return new int[0];} + public int[] getBadPixel(DateTime eventTimeStamp) {return new int[0];} + + public int[] getNotUsablePixels(DateTime eventTimeStamp) {return new int[0];} } \ No newline at end of file From 8ab69469ca601167741cbc0a8cb9614ec6d1b3a4 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 18:32:56 +0100 Subject: [PATCH 15/19] implement Service instead of CalibrationService --- .../calibrationservice/SinglePulseGainCalibService.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java b/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java index 0ada73f3b8..11a07164b9 100644 --- a/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java +++ b/src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java @@ -2,17 +2,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.joda.time.DateTime; import stream.annotations.Parameter; import stream.io.SourceURL; import stream.io.CsvStream; import stream.Data; import fact.Constants; +import stream.service.Service; /** * **/ -public class SinglePulseGainCalibService implements CalibrationService { +public class SinglePulseGainCalibService implements Service { Logger log = LoggerFactory.getLogger(SinglePulseGainCalibService.class); @@ -63,7 +63,4 @@ public void setIntegralGainFile(SourceURL integralGainFile) { this.integralGainFile = integralGainFile; } - public int[] getBadPixel(DateTime eventTimeStamp) {return new int[0];} - - public int[] getNotUsablePixels(DateTime eventTimeStamp) {return new int[0];} } \ No newline at end of file From 047f431b4085e6f9ce99e6ab82d5511ffad10c71 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 18:54:40 +0100 Subject: [PATCH 16/19] remove superfluous factor 10 --- src/main/java/fact/photonstream/SinglePulseExtraction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/fact/photonstream/SinglePulseExtraction.java b/src/main/java/fact/photonstream/SinglePulseExtraction.java index c047baf0c8..75ba856f91 100644 --- a/src/main/java/fact/photonstream/SinglePulseExtraction.java +++ b/src/main/java/fact/photonstream/SinglePulseExtraction.java @@ -145,7 +145,7 @@ public Data process(Data input) { double[] pixelTimeSeries = ElementWise.multiply( pixelTimeSeriesInMv, - 1.0/config.factSinglePeAmplitudeInMv * gainCorrection[pix] + 1.0/gainCorrection[pix] ); SinglePulseExtractor.Result result = spe.extractFromTimeSeries( From 6a72d30d75b07685f15f38524dec234fb1e1cf8f Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 19:28:00 +0100 Subject: [PATCH 17/19] use SinglePulseGainCalibService in ConvertSinglePulses2Timeseries as well --- examples/viewer.xml | 1 + .../ConvertSinglePulses2Timeseries.java | 50 ++++++++++++++++--- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/examples/viewer.xml b/examples/viewer.xml index 9772c40455..421b7570b4 100644 --- a/examples/viewer.xml +++ b/examples/viewer.xml @@ -33,6 +33,7 @@ singlePulsesKey="PhotonArrivals" baseLineKey="PhotonArrivalsBaseLine" timeSeriesKey="ReconstructedDataCalibrated" + gainService="gainService" /> diff --git a/src/main/java/fact/photonstream/ConvertSinglePulses2Timeseries.java b/src/main/java/fact/photonstream/ConvertSinglePulses2Timeseries.java index edaf72d839..f3b2ae838e 100644 --- a/src/main/java/fact/photonstream/ConvertSinglePulses2Timeseries.java +++ b/src/main/java/fact/photonstream/ConvertSinglePulses2Timeseries.java @@ -1,19 +1,22 @@ package fact.photonstream; +import fact.Constants; +import fact.calibrationservice.SinglePulseGainCalibService; import fact.photonstream.timeSeriesExtraction.AddFirstArrayToSecondArray; -import fact.photonstream.timeSeriesExtraction.SinglePulseExtractor; import fact.photonstream.timeSeriesExtraction.TemplatePulse; import fact.photonstream.timeSeriesExtraction.ElementWise; import org.apache.commons.lang3.ArrayUtils; import stream.Data; -import stream.Processor; +import stream.ProcessContext; +import stream.StatefulProcessor; import stream.annotations.Parameter; +import stream.annotations.Service; /** * Created by jebuss on 28.10.16. */ -public class ConvertSinglePulses2Timeseries implements Processor { +public class ConvertSinglePulses2Timeseries implements StatefulProcessor { @Parameter(required = true, description = "The arrival slices of the single pulses.") private String singlePulsesKey = null; @@ -26,6 +29,14 @@ public class ConvertSinglePulses2Timeseries implements Processor { @Parameter(required = false, description = "The reconstructed baseline of the original time series.") private String baseLineKey = null; + @Service( + required = false, + description = "The calibration service for the integral single pulse gain" + ) + SinglePulseGainCalibService gainService = null; + protected double[] gainCorrection = null; + private int npix = Constants.NUMBEROFPIXEL; + @Override public Data process(Data input) { @@ -57,12 +68,11 @@ public Data process(Data input) { currentTimeSeries = ElementWise.add(currentTimeSeries, baseLine[pix]); timeSeries = ArrayUtils.addAll(timeSeries, currentTimeSeries); - } - SinglePulseExtractor.Config config = new SinglePulseExtractor.Config(); - timeSeries = ElementWise.multiply( - timeSeries, - config.factSinglePeAmplitudeInMv); + timeSeries = ElementWise.multiply( + timeSeries, + gainCorrection[pix]); + } input.put(timeSeriesKey, timeSeries); @@ -84,4 +94,28 @@ public void setBaseLineKey(String baseLineKey) { public void setRoi(int roi) { this.roi = roi; } + + @Override + public void init(ProcessContext processContext) throws Exception { + + double factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); + + if(gainService == null) { + gainCorrection = new double[npix]; + for(int i=0; i Date: Wed, 25 Jan 2017 20:42:29 +0100 Subject: [PATCH 18/19] another place found, where factSinglePeAmplitudeInMv was used removed it entirely ... so it cannot happen again --- src/main/java/fact/photonstream/SinglePulseExtraction.java | 2 +- .../photonstream/timeSeriesExtraction/SinglePulseExtractor.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/fact/photonstream/SinglePulseExtraction.java b/src/main/java/fact/photonstream/SinglePulseExtraction.java index 75ba856f91..8ada2f5a4c 100644 --- a/src/main/java/fact/photonstream/SinglePulseExtraction.java +++ b/src/main/java/fact/photonstream/SinglePulseExtraction.java @@ -157,7 +157,7 @@ public Data process(Data input) { outputWindowLengthInSlices); timeSeriesAfterExtraction[pix] = ElementWise.multiply( result.timeSeriesAfterExtraction, - config.factSinglePeAmplitudeInMv); + gainCorrection[pix]); baseLine[pix] = result.timeSeriesBaseLine(); } diff --git a/src/main/java/fact/photonstream/timeSeriesExtraction/SinglePulseExtractor.java b/src/main/java/fact/photonstream/timeSeriesExtraction/SinglePulseExtractor.java index 05ef7c4131..1c4c7aeb98 100644 --- a/src/main/java/fact/photonstream/timeSeriesExtraction/SinglePulseExtractor.java +++ b/src/main/java/fact/photonstream/timeSeriesExtraction/SinglePulseExtractor.java @@ -19,7 +19,6 @@ public static class Config { public int pulseToLookForLength; public int plateauLength; public int negativePulseLength; - public double factSinglePeAmplitudeInMv; public int maxIterations; public Config() { @@ -29,7 +28,6 @@ public Config() { pulseToLookForLength = 20; plateauLength = 7; negativePulseLength = 300; - factSinglePeAmplitudeInMv = 10.0; maxIterations = 250; } } From d8f42aa785c71bf7a18f702ea9b9661fac439f76 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 20:42:48 +0100 Subject: [PATCH 19/19] add a test --- src/test/java/fact/UtilsTests.java | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/test/java/fact/UtilsTests.java b/src/test/java/fact/UtilsTests.java index 56fdc107a8..45206beb81 100644 --- a/src/test/java/fact/UtilsTests.java +++ b/src/test/java/fact/UtilsTests.java @@ -14,4 +14,24 @@ public void flattenEmpty2dArray(){ Assert.assertEquals(result.length, 0); } + + @Test + public void flattenSomeArrays(){ + + final int num_arrays = 10; + final int array_length = 2; + double[][] someNumbers = new double[num_arrays][array_length]; + for (int i=0; i