job-status now knows CIS internal error

antidote-cis

@@ -197,24 +197,33 @@ function cis_get_job_status () {
     jobid="$1"
     getjobhierarchy_apires=`curl_wrapped "https://beta-cps.trafficmanager.net/cis.client.svc/Public/$cis_namespace/GenericJob/$jobid/GetJobHierarchy"` && 
     subworkflows_text=`echo "$getjobhierarchy_apires" | json2table DisplayName,Id,DisplayStatus,JobType -p` || ! echo2 "API GetJobHierarchy failed. Server says '$getjobhierarchy_apires'" || return $?
-    callstack_formatted=`echo "$getjobhierarchy_apires" | json2table DisplayName,Id,DisplayStatus | grep -E '( InProgress )|( PlanError )'` # Ignore failure because it's only useful in InProgress and PlanError state. 
+        callstack_formatted=`echo "$getjobhierarchy_apires" | json2table DisplayName,Id,DisplayStatus | grep -E '( InProgress )|( PlanError )|( Internal Error )'` # Ignore failure because it's only useful in InProgress and PlanError state. 
     _mdcode='```'
 
+    echo "DEBUG: $subworkflows_text"
     if planerror_callstack=`echo "$subworkflows_text" | grep -F '|PlanError|'`; then
         # RemoteJob may have PlanError in subworkflow. The top-level workflow is still InProgress but bottom workflow got PlanError. 
-        [[ -z "$callstack_formatted" ]] && echo2 "API GetJobHierarchy result misinterpreted. No workflow in stat InProgress or PlanError. This is an unexpected non-fatal error and I did not catch it. " && return 2
+        [[ -z "$callstack_formatted" ]] && echo2 "API GetJobHierarchy result misinterpreted. No workflow in stat InProgress or PlanError or InternalError. This is an unexpected non-fatal error and I did not catch it. " && return 2
         [[ -z $ANTIDOTE_JOBSTAT_MARKDOWN ]] &&
             echo2 -e ">>> CIS reported PlanError:\nCallstack: \n$callstack_formatted\nCIS Log: https://beta-cps.trafficmanager.net/Public/$cis_namespace/JobDetails/$jobid" ||
             echo2 -e "## CIS reported PlanError\n\n[View CIS Log](https://beta-cps.trafficmanager.net/Public/$cis_namespace/JobDetails/$jobid)\n\n### Callstack\n\n$_mdcode\n$callstack_formatted\n$_mdcode\n"
         echo "Failed" && return 0
     fi
+    if planerror_callstack=`echo "$subworkflows_text" | grep -F '|Internal Error|'`; then
+        # RemoteJob may have PlanError in subworkflow. The top-level workflow is still InProgress but bottom workflow got PlanError. 
+        [[ -z "$callstack_formatted" ]] && echo2 "API GetJobHierarchy result misinterpreted. No workflow in stat InProgress or PlanError or InternalError. This is an unexpected non-fatal error and I did not catch it. " && return 2
+        [[ -z $ANTIDOTE_JOBSTAT_MARKDOWN ]] &&
+            echo2 -e ">>> CIS reported Internal Error:\nCallstack: \n$callstack_formatted\nCIS Log: https://beta-cps.trafficmanager.net/Public/$cis_namespace/JobDetails/$jobid" ||
+            echo2 -e "## CIS reported Internal Error\n\n[View CIS Log](https://beta-cps.trafficmanager.net/Public/$cis_namespace/JobDetails/$jobid)\n\n### Callstack\n\n$_mdcode\n$callstack_formatted\n$_mdcode\n"
+        echo "Failed" && return 0
+    fi
     if inprogress_callstack=`echo "$subworkflows_text" | grep -F '|InProgress|'`; then
         # There's some workflow in-progress. It maybe inprogress or blocked. 
         # Note that, there may be multiple workflows in "InProgress" state. In this scenario, 
         #   the last "InProgress" workflow is the bottom sub-workflow which we're interested in. 
         #   We just check the last sub-workflow, to see if there's any "Blocked" activity. 
 
-        [[ -z "$callstack_formatted" ]] && echo2 "API GetJobHierarchy result misinterpreted. No workflow in stat InProgress or PlanError. This is an unexpected non-fatal error and I did not catch it. " && return 2
+        [[ -z "$callstack_formatted" ]] && echo2 "API GetJobHierarchy result misinterpreted. No workflow in stat InProgress or PlanError or InternalError. This is an unexpected non-fatal error and I did not catch it. " && return 2
 
         # Remote workflow may have different namespace, so we need to get bottom_subworkflow_namespace. 
         bottom_subworkflow_id=`echo "$inprogress_callstack" | tail -n 1 | cut -d '|' -f 3` &&