adding support for updater service

Signed-off-by: Shekhar Saxena <[email protected]>
kruize · Dec 12, 2024 · 60306c7 · 60306c7
1 parent fa0531a
commit 60306c7
Show file tree

Hide file tree

Showing 6 changed files with 63 additions and 5 deletions.
diff --git a/...main/java/com/autotune/analyzer/recommendations/updater/RecommendationUpdaterService.java b/...main/java/com/autotune/analyzer/recommendations/updater/RecommendationUpdaterService.java
@@ -52,7 +52,7 @@ public static void initiateUpdaterService() {
                     for (Map.Entry<String, KruizeObject> experiment : experiments.entrySet()) {
                         KruizeObject kruizeObject = updater.generateResourceRecommendationsForExperiment(experiment.getValue().getExperimentName());
                         // TODO:// add default updater in kruizeObject and check if GPU recommendations are present
-                        if (kruizeObject.getDefaultUpdater().isEmpty() || kruizeObject.getDefaultUpdater() == null) {
+                        if (kruizeObject.getDefaultUpdater() == null) {
                              kruizeObject.setDefaultUpdater(AnalyzerConstants.RecommendationUpdaterConstants.SupportedUpdaters.VPA);
                         }
 
@@ -61,27 +61,30 @@ public static void initiateUpdaterService() {
                             vpaUpdater.applyResourceRecommendationsForExperiment(kruizeObject);
                         }
                     }
+                    LOGGER.info("Done");
                 } catch (Exception e) {
                     LOGGER.error(e.getMessage());
                 }
-            }, 0, AnalyzerConstants.RecommendationUpdaterConstants.DEFAULT_SLEEP_INTERVAL, TimeUnit.SECONDS);
+            }, AnalyzerConstants.RecommendationUpdaterConstants.DEFAULT_INITIAL_DELAY,
+                    AnalyzerConstants.RecommendationUpdaterConstants.DEFAULT_SLEEP_INTERVAL,
+                    TimeUnit.SECONDS);
         } catch (Exception e) {
             LOGGER.error(AnalyzerErrorConstants.RecommendationUpdaterErrors.UPDTAER_SERVICE_START_ERROR + e.getMessage());
         }
     }
 
     private static Map<String, KruizeObject> getAutoModeExperiments() {
         try {
+            LOGGER.info(AnalyzerConstants.RecommendationUpdaterConstants.InfoMsgs.CHECKING_AUTO_EXP);
             Map<String, KruizeObject> mainKruizeExperimentMap = new ConcurrentHashMap<>();
-            new ExperimentDBService().loadAllExperiments(mainKruizeExperimentMap);
+            new ExperimentDBService().loadAllLMExperiments(mainKruizeExperimentMap);
             // filter map to only include entries where mode is auto or recreate
             Map<String, KruizeObject> filteredMap = mainKruizeExperimentMap.entrySet().stream()
                     .filter(entry -> {
                         String mode = entry.getValue().getMode();
                         return AnalyzerConstants.AUTO.equalsIgnoreCase(mode) || AnalyzerConstants.RECREATE.equalsIgnoreCase(mode);
                     })
                     .collect(Collectors.toConcurrentMap(Map.Entry::getKey, Map.Entry::getValue));
-
             return filteredMap;
         } catch (Exception e) {
             LOGGER.error(e.getMessage());

diff --git a/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java b/src/main/java/com/autotune/analyzer/utils/AnalyzerConstants.java
@@ -684,7 +684,8 @@ private RecommendationUpdaterConstants() {
 
         }
 
-        public static final int DEFAULT_SLEEP_INTERVAL = 120;
+        public static final int DEFAULT_SLEEP_INTERVAL = 60;
+        public static final int DEFAULT_INITIAL_DELAY = 30;
         public static final class SupportedUpdaters {
             public static final String VPA = "vpa";
 
@@ -721,6 +722,7 @@ public static final class InfoMsgs {
             public static final String CREATEING_VPA = "Creating VPA with name: %s";
             public static final String CREATED_VPA = "Created VPA with name: %s";
             public static final String STARTING_SERVICE = "Starting recommendation updater.";
+            public static final String CHECKING_AUTO_EXP = "Searching for experiments with auto or recreate mode.";
             private InfoMsgs() {
 
             }

diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAO.java b/src/main/java/com/autotune/database/dao/ExperimentDAO.java
@@ -43,6 +43,9 @@ public interface ExperimentDAO {
     // If Kruize object restarts load all experiment which are in inprogress
     public List<KruizeExperimentEntry> loadAllExperiments() throws Exception;
 
+    // If Kruize object restarts load all local monitoring experiments which are in inprogress
+    public List<KruizeLMExperimentEntry> loadAllLMExperiments() throws Exception;
+
     // If Kruize object restarts load all results from the experiments which are in inprogress
     List<KruizeResultsEntry> loadAllResults() throws Exception;
 

diff --git a/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java b/src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java
@@ -732,6 +732,29 @@ public List<KruizeExperimentEntry> loadAllExperiments() throws Exception {
         return entries;
     }
 
+    @Override
+    public List<KruizeLMExperimentEntry> loadAllLMExperiments() throws Exception {
+        //todo load only experimentStatus=inprogress , playback may not require completed experiments
+        List<KruizeLMExperimentEntry> entries = null;
+        String statusValue = "failure";
+        Timer.Sample timerLoadAllExp = Timer.start(MetricsConfig.meterRegistry());
+        try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
+            entries = session.createQuery(SELECT_FROM_LM_EXPERIMENTS, KruizeLMExperimentEntry.class).list();
+            // TODO: remove native sql query and transient
+            //getExperimentTypeInKruizeExperimentEntry(entries);
+            statusValue = "success";
+        } catch (Exception e) {
+            LOGGER.error("Not able to load experiment due to {}", e.getMessage());
+            throw new Exception("Error while loading exsisting experiments from database due to : " + e.getMessage());
+        } finally {
+            if (null != timerLoadAllExp) {
+                MetricsConfig.timerLoadAllExp = MetricsConfig.timerBLoadAllExp.tag("status", statusValue).register(MetricsConfig.meterRegistry());
+                timerLoadAllExp.stop(MetricsConfig.timerLoadAllExp);
+            }
+        }
+        return entries;
+    }
+
     @Override
     public List<KruizeResultsEntry> loadAllResults() throws Exception {
         // TODO: load only experimentStatus=inProgress , playback may not require completed experiments

diff --git a/src/main/java/com/autotune/database/helper/DBConstants.java b/src/main/java/com/autotune/database/helper/DBConstants.java
@@ -6,6 +6,7 @@ public class DBConstants {
 
     public static final class SQLQUERY {
         public static final String SELECT_FROM_EXPERIMENTS = "from KruizeExperimentEntry";
+        public static final String SELECT_FROM_LM_EXPERIMENTS = "from KruizeLMExperimentEntry";
         public static final String SELECT_FROM_EXPERIMENTS_BY_EXP_NAME = "from KruizeExperimentEntry k WHERE k.experiment_name = :experimentName";
         public static final String SELECT_FROM_LM_EXPERIMENTS_BY_EXP_NAME = "from KruizeLMExperimentEntry k WHERE k.experiment_name = :experimentName";
         public static final String SELECT_FROM_RESULTS = "from KruizeResultsEntry";

diff --git a/src/main/java/com/autotune/database/service/ExperimentDBService.java b/src/main/java/com/autotune/database/service/ExperimentDBService.java
@@ -77,6 +77,32 @@ public void loadAllExperiments(Map<String, KruizeObject> mainKruizeExperimentMap
         }
     }
 
+    public void loadAllLMExperiments(Map<String, KruizeObject> mainKruizeExperimentMap) throws Exception {
+        ExperimentInterface experimentInterface = new ExperimentInterfaceImpl();
+        List<KruizeLMExperimentEntry> entries = experimentDAO.loadAllLMExperiments();
+        if (null != entries && !entries.isEmpty()) {
+            List<CreateExperimentAPIObject> createExperimentAPIObjects = DBHelpers.Converters.KruizeObjectConverters.convertLMExperimentEntryToCreateExperimentAPIObject(entries);
+            if (null != createExperimentAPIObjects && !createExperimentAPIObjects.isEmpty()) {
+                List<KruizeObject> kruizeExpList = new ArrayList<>();
+
+                int failureThreshHold = createExperimentAPIObjects.size();
+                int failureCount = 0;
+                for (CreateExperimentAPIObject createExperimentAPIObject : createExperimentAPIObjects) {
+                    KruizeObject kruizeObject = Converters.KruizeObjectConverters.convertCreateExperimentAPIObjToKruizeObject(createExperimentAPIObject);
+                    if (null != kruizeObject) {
+                        kruizeExpList.add(kruizeObject);
+                    } else {
+                        failureCount++;
+                    }
+                }
+                if (failureThreshHold > 0 && failureCount == failureThreshHold) {
+                    throw new Exception("None of the experiments are able to load from DB.");
+                }
+                experimentInterface.addExperimentToLocalStorage(mainKruizeExperimentMap, kruizeExpList);
+            }
+        }
+    }
+
     public void loadAllResults(Map<String, KruizeObject> mainKruizeExperimentMap) throws Exception {
         ExperimentInterface experimentInterface = new ExperimentInterfaceImpl();
         KruizeObject kruizeObject;