DL4Jets changes

.github/workflows/CI.yml

@@ -0,0 +1,126 @@
+name: CI/CD
+
+on:
+  push:
+    branches:
+      - 'master'
+  pull_request:
+  # Run daily at 0:01 UTC
+  schedule:
+  - cron:  '1 0 * * *'
+
+jobs:
+  test:
+    name: "🐍 ${{ matrix.python-version }} • ${{ matrix.runs-on }}"
+    runs-on: ${{ matrix.runs-on }}
+    strategy:
+      matrix:
+        # just one for now
+        runs-on: [ubuntu-latest]
+        python-version: [3.6, 3.7]
+
+    defaults:
+      run:
+        shell: "bash -l {0}"
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name:  'Setup conda env'
+      uses: conda-incubator/setup-miniconda@v2
+      with: 
+        python-version: ${{ matrix.python-version }}
+        mamba-version: "*"    
+        channels: conda-forge,defaults
+        channel-priority: true
+        auto-activate-base: false
+        environment-file: conda_env.yml 
+        activate-environment: DJC2
+
+    - name: "Check conda env"
+      run: |
+        conda env list
+        conda info
+        conda list
+        which python  
+        python -V
+
+    - name: Set env vars 1
+      run: | 
+        # normally one would `source docker_env.sh` but GHA can't do this
+        # It's not possible to chain these - an exported env var only becomes available in next step
+        echo "DEEPJETCORE=$( pwd -P)" >> $GITHUB_ENV
+        echo "PATH=$( pwd -P)/bin:$PATH" >> $GITHUB_ENV
+        echo "PYTHONPATH=$( pwd -P)/../:$PYTHONPATH" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$( pwd -P)/compiled/" >> $GITHUB_ENV
+        # just hardcode it
+        echo "PYTHONPATH=/home/runner/work/DeepJetCore/DeepJetCore/bin:/home/runner/work/DeepJetCore/DeepJetCore:/home/runner/work/DeepJetCore:$PYTHONPATH" >> $GITHUB_ENV
+
+    - name: Check env vars 1
+      run: |
+        echo "DEEPJETCORE"
+        echo $DEEPJETCORE
+        echo "PATH"
+        echo $PATH
+        echo "PYTHONPATH"
+        echo $PYTHONPATH 
+        echo "LD_LIBRARY_PATH"
+        echo $LD_LIBRARY_PATH
+
+    - name: Compile
+      run: |
+        cd $DEEPJETCORE/compiled
+        make -f Makefile_conda -j4
+
+    - name: "Unit tests"
+      run: |
+        cd $DEEPJETCORE/testing/unit
+        python test.py
+        cd $DEEPJETCORE
+
+    - name: Create subpackage
+      run: |
+        python bin/createSubpackage.py --data here
+
+    - name: Set env vars 2
+      run: |
+        # normally one would `source env.sh` but GHA can't do this
+        # It's not possible to chain these - an exported env var only becomes available in next step
+        echo "DEEPJETCORE_SUBPACKAGE=$DEEPJETCORE/here" >> $GITHUB_ENV    
+        echo "PATH=$HERE/scripts:$PATH" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$HERE/modules/compiled:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+        # just hardcode it
+        echo "PYTHONPATH=/home/runner/work/DeepJetCore/DeepJetCore/here/modules:/home/runner/work/DeepJetCore/DeepJetCore/here/modules/datastructures:/home/runner/work/DeepJetCore/DeepJetCore/here/modules/compiled:$PYTHONPATH" >> $GITHUB_ENV
+
+    - name: Check env vars 2
+      run: |
+        echo "DEEPJETCORE_SUBPACKAGE"
+        echo $DEEPJETCORE_SUBPACKAGE
+        echo "PYTHONPATH"
+        echo $PYTHONPATH
+        echo "PATH"
+        echo $PATH 
+        echo "LD_LIBRARY_PATH"
+        echo $LD_LIBRARY_PATH
+
+
+    - name: Convert files
+      run: |   
+        cd $DEEPJETCORE
+        python bin/convertFromSource.py -i here/example_data/train_files.txt -o here/dctrain -c TrainData_example
+
+    - name: "Run training"
+      run: |
+        # cd $DEEPJETCORE
+        python here/Train/training_example.py here/dctrain/dataCollection.djcdc here/training
+
+    - name: "Run eval"
+      run: |
+        predict.py here/training/KERAS_model.h5 here/training/trainsamples.djcdc here/example_data/test_files.txt here/out_predict
+
+

.gitignore

@@ -16,4 +16,9 @@ __pycache__
 *.csv
 *.root
 *.json
+.DS_Store
 #*.*#
+build/
+*.egg-info/
+bin/make_example_data_overlap
+

DJCLayers.py

@@ -0,0 +1,163 @@
+
+djc_global_layers_list={}
+
+from keras.layers import Layer
+import tensorflow as tf
+
+
+
+class StopGradient(Layer):
+    def __init__(self, **kwargs):
+        super(StopGradient, self).__init__(**kwargs)
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def call(self, inputs):
+        return tf.stop_gradient(inputs)
+
+djc_global_layers_list['StopGradient']=StopGradient   
+
+class SelectFeatures(Layer):
+    def __init__(self, index_left, index_right, **kwargs):
+        super(SelectFeatures, self).__init__(**kwargs)
+        self.index_left=index_left
+        self.index_right=index_right
+
+    def compute_output_shape(self, input_shape):
+        return input_shape[:-1] + (self.index_right-self.index_left,)
+
+    def call(self, inputs):
+
+        return inputs[...,self.index_left:self.index_right]
+
+    def get_config(self):
+        config = {'index_left': self.index_left,'index_right': self.index_right}
+        base_config = super(SelectFeatures, self).get_config()
+        return dict(list(base_config.items()) + list(config.items() ))
+
+
+djc_global_layers_list['SelectFeatures']=SelectFeatures
+
+
+class ScalarMultiply(Layer):
+    def __init__(self, factor, **kwargs):
+        super(ScalarMultiply, self).__init__(**kwargs)
+        self.factor=factor
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def call(self, inputs):
+        return inputs*self.factor
+
+    def get_config(self):
+        config = {'factor': self.factor}
+        base_config = super(ScalarMultiply, self).get_config()
+        return dict(list(base_config.items()) + list(config.items() ))
+
+
+djc_global_layers_list['ScalarMultiply']=ScalarMultiply
+
+class Print(Layer):
+    def __init__(self, message, **kwargs):
+        super(Print, self).__init__(**kwargs)
+        self.message=message
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def call(self, inputs):
+        return tf.Print(inputs,[inputs],self.message,summarize=300)
+
+    def get_config(self):
+        config = {'message': self.message}
+        base_config = super(Print, self).get_config()
+        return dict(list(base_config.items()) + list(config.items() ))
+
+
+djc_global_layers_list['Print']=Print
+
+### the following ones should go to DeepJetCore
+
+class ReplaceByNoise(Layer):
+    def __init__(self, **kwargs):
+        super(ReplaceByNoise, self).__init__(**kwargs)
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def call(self, inputs):
+        return tf.random_normal(shape=tf.shape(inputs),
+                                mean=0.0,
+                                stddev=1.0,
+                                dtype='float32')
+
+
+    def get_config(self):
+        base_config = super(ReplaceByNoise, self).get_config()
+        return dict(list(base_config.items()))
+
+
+djc_global_layers_list['ReplaceByNoise']=ReplaceByNoise
+
+
+
+class FeedForward(Layer):
+    def __init__(self, **kwargs):
+        super(FeedForward, self).__init__(**kwargs)
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def call(self, inputs):
+        return 1.*inputs
+
+
+    def get_config(self):
+        base_config = super(FeedForward, self).get_config()
+        return dict(list(base_config.items()))
+
+
+djc_global_layers_list['FeedForward']=FeedForward
+
+
+class Clip(Layer):
+    def __init__(self, min, max , **kwargs):
+        super(Clip, self).__init__(**kwargs)
+        self.min=min
+        self.max=max
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def call(self, inputs):
+        return tf.clip_by_value(inputs, self.min, self.max)
+
+    def get_config(self):
+        config = {'min': self.min, 'max': self.max}
+        base_config = super(Clip, self).get_config()
+        return dict(list(base_config.items()) + list(config.items() ))
+
+djc_global_layers_list['Clip']=Clip
+
+
+class ReduceSumEntirely(Layer):
+    def __init__(self,  **kwargs):
+        super(ReduceSumEntirely, self).__init__(**kwargs)
+
+
+    def compute_output_shape(self, input_shape):
+        return (input_shape[0],1)
+
+    def call(self, inputs):
+        red_axes=(inputs.shape[1:]).as_list()
+        red_axes = [i+1 for i in range(len(red_axes))]
+        return tf.expand_dims(tf.reduce_sum(inputs,axis=red_axes),axis=1)
+
+    def get_config(self):
+        base_config = super(ReduceSumEntirely, self).get_config()
+        return dict(list(base_config.items()))
+
+
+djc_global_layers_list['ReduceSumEntirely']=ReduceSumEntirely

DJCLosses.py

@@ -0,0 +1,11 @@
+
+djc_global_loss_list = {}
+
+import tensorflow as tf
+
+def dummy_loss(truth, pred):
+    #t = tf.Print(truth,[truth],'truth ')
+    #p = tf.Print(pred,[pred],'pred ')
+    return (tf.reduce_mean(truth)-tf.reduce_mean(pred))**2
+
+djc_global_loss_list['dummy_loss']=dummy_loss