diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot index 9b482db80..7809f4665 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot @@ -34,6 +34,9 @@ Test Tags PreUpgrade ${CUSTOM_CULLER_TIMEOUT} 60000 ${S_SIZE} 25 ${DW_PROJECT_CREATED} False +${CODE} while True: import time ; time.sleep(10); print ("Hello") +${UPGRADE_NS} upgrade +${UPGRADE_CONFIG_MAP} upgrade-config-map *** Test Cases *** @@ -246,8 +249,44 @@ Model Registry Pre Upgrade Set Up [Tags] Upgrade ModelRegistryUpgrade Model Registry Pre Upgrade Scenario +Long Running Jupyter Notebook + [Documentation] Launch a long running notebook before the upgrade + [Tags] Upgrade + Launch Notebook + Add And Run JupyterLab Code Cell In Active Notebook ${CODE} + + # Get the notebook pod creation timestamp + ${notebook_pod_name}= Set Variable jupyter-nb-ldap-2dadmin2-0 + ${return_code} ${ntb_creation_timestamp} = Run And Return Rc And Output + ... oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${ntb_creation_timestamp} + + # Save the timestamp to the OpenShift ConfigMap so it can be used in test in the next phase + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc create namespace ${UPGRADE_NS} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc create configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} --from-literal=ntb_creation_timestamp=${ntb_creation_timestamp} # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + + Close Browser + *** Keywords *** +Launch Notebook + [Documentation] Launch notebook for the suite + [Arguments] ${notebook_image}=minimal-notebook + ... ${username}=${TEST_USER2.USERNAME} + ... ${password}=${TEST_USER2.PASSWORD} + ... ${auth_type}=${TEST_USER2.AUTH_TYPE} + Begin Web Test username=${username} password=${password} auth_type=${auth_type} + Launch Jupyter From RHODS Dashboard Link + Spawn Notebook With Arguments + ... image=${notebook_image} + ... username=${username} + ... password=${password} + ... auth_type=${auth_type} + Dashboard Suite Setup [Documentation] Basic suite setup Set Library Search Order SeleniumLibrary diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot index b0c467722..226383bc7 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot @@ -14,23 +14,7 @@ Library JupyterLibrary Test Tags DuringUpgrade -*** Variables *** -${CODE} while True: import time ; time.sleep(10); print ("Hello") - - *** Test Cases *** -Long Running Jupyter Notebook - [Documentation] Launch a long running notebook before the upgrade - [Tags] Upgrade - Launch Notebook - Add And Run JupyterLab Code Cell In Active Notebook ${CODE} - # robocop:disable - ${return_code} ${timestamp} = Run And Return Rc And Output - ... oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' - Should Be Equal As Integers ${return_code} 0 - Set Global Variable ${timestamp} # robocop: disable - Close Browser - Upgrade RHODS [Documentation] Approve the install plan for the upgrade and make sure that upgrade has completed [Tags] ODS-1766 Upgrade @@ -79,7 +63,7 @@ PyTorch Image Workload Test *** Keywords *** Launch Notebook [Documentation] Launch notebook for the suite - [Arguments] ${notbook_image}=minimal-notebook + [Arguments] ${notebook_image}=minimal-notebook ... ${username}=${TEST_USER2.USERNAME} ... ${password}=${TEST_USER2.PASSWORD} ... ${auth_type}=${TEST_USER2.AUTH_TYPE} @@ -94,7 +78,7 @@ Launch Notebook Fix Spawner Status # robocop: disable Spawn Notebook With Arguments - ... image=${notbook_image} + ... image=${notebook_image} ... username=${username} ... password=${password} ... auth_type=${auth_type} diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot index d912e756a..a59e41a26 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot @@ -34,6 +34,8 @@ Test Tags PostUpgrade *** Variables *** ${S_SIZE} 25 ${DW_PROJECT_CREATED} False +${UPGRADE_NS} upgrade +${UPGRADE_CONFIG_MAP} upgrade-config-map *** Test Cases *** @@ -71,13 +73,26 @@ Verify Culler is Enabled END Verify Notebook Has Not Restarted - [Documentation] Verify Notbook pod has not restarted after the upgrade - [Tags] Upgrade AutomationBug RHOAIENG-14840 - # robocop:disable + [Documentation] Verify Notebook pod has not restarted after the upgrade + [Tags] Upgrade + ${notebook_name}= Set Variable jupyter-nb-ldap-2dadmin2 + ${notebook_pod_name}= Set Variable ${notebook_name}-0 + + # Get the running notebook creation timestamp ${return_code} ${new_timestamp} Run And Return Rc And Output - ... oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' - Should Be Equal As Integers ${return_code} 0 - Should Be Equal ${timestamp} ${new_timestamp} msg=Running notebook pod has restarted + ... oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${new_timestamp} + + # Get the running notebook creation timestamp from the upgrade ConfigMap safed in the previous + # phase (before the actual RHOAI upgrade) + ${return_code} ${ntb_creation_timestamp} Run And Return Rc And Output + ... oc get configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} -o jsonpath='{.data.ntb_creation_timestamp}' + Should Be Equal As Integers ${return_code} 0 msg=${ntb_creation_timestamp} + + # The timestamps should be equal + Should Be Equal ${ntb_creation_timestamp} ${new_timestamp} msg=Running notebook pod has restarted + + [Teardown] Terminate Running Notebook ${notebook_name} Verify Custom Image Is Present [Documentation] Verify Custom Noteboook is not deleted after the upgrade @@ -325,6 +340,13 @@ Delete OOTB Image IF not ${status} Fail Notebook image is deleted after the upgrade IF not ${IS_SELF_MANAGED} Managed RHOAI Upgrade Test Teardown +Terminate Running Notebook + [Documentation] Terminates the running notebook instance + [Arguments] ${notebook_name} + ${return_code} ${cmd_output} Run And Return Rc And Output + ... oc delete Notebook.kubeflow.org -n ${NOTEBOOKS_NAMESPACE} ${notebook_name} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + Managed RHOAI Upgrade Test Teardown # robocop: off=too-many-calls-in-keyword [Documentation] Check rhods_aggregate_availability metric when RHOAI is installed as managed