fork download 
  1. def get_sys_log_data(request, exec_vo):
  2.     exec_vo_conf = exec_vo.config
  3.     model_vo_conf = exec_vo.model.config
  4.     logger.info(model_vo_conf)
  5.     fetched_data = dict()
  6.     is_hist_syslog_found = '0'
  7.     is_live_syslog_found = '0'
  8.     hist_syslog_time = 0
  9.     live_syslog_time = 0
  10.     hist_syslog_lines = 0
  11.     live_syslog_lines = 0
  12.     model_syslog_lines = 0
  13.     sys_log_lines = 0
  14.     sys_log_df = None
  15.     model_sys_log_df = None
  16.     syslog_range_start_time = None
  17.     syslog_range_end_time = None
  18.     syslog_start_time = None
  19.     syslog_end_time = None
  20.     sys_log = []
  21.     syslog_model_start_time = None
  22.     syslog_model_end_time = None
  23.     live_min_date = None
  24.     live_max_date = None
  25.     hist_min_date = None
  26.     hist_max_date = None
  27.     model_sys_logs = []
  28.     errors = []
  29.     log_errors = []
  30.     try:
  31.         no_of_days_logs = model_vo_conf['days_sys_log']
  32.         no_of_days_modellogs = model_vo_conf['model_sys_log']
  33.  
  34.         log_start_date, log_end_date = get_logfetch_daterange(request, no_of_days_logs)
  35.         logger.info('SysLog startDate:{}, endDate:{}'.format(log_start_date, log_end_date))
  36.  
  37.         # Adding Extracted date
  38.         model_start_date, model_end_date = get_model_log_daterange(request, no_of_days_modellogs)
  39.         logger.info('SysLog extractstartDate:{}, extractendDate:{}'.format(model_start_date,
  40.                                                                            model_end_date))
  41.  
  42.         # Adding 23 hrs 59 min 59 sec to the end date in time conversion
  43.         log_start_date_time = pd.to_datetime(log_start_date, format=constants.DATE_TIME_FORMAT)
  44.         log_end_date_time = pd.to_datetime(log_end_date, format=constants.DATE_TIME_FORMAT)
  45.         log_end_date_time += pd.Timedelta(hours=23, minutes=59, seconds=59)
  46.         logger.info('SysLog Time startDate:{}, endDate:{}'.format(log_start_date_time, log_end_date_time))
  47.  
  48.         # Adding 23 hrs 59 min 59 sec to the Extract end date in time conversion
  49.         model_start_date_time = pd.to_datetime(model_start_date, format=constants.DATE_TIME_FORMAT)
  50.         model_end_date_time = pd.to_datetime(model_end_date, format=constants.DATE_TIME_FORMAT)
  51.         model_end_date_time += pd.Timedelta(hours=23, minutes=59, seconds=59)
  52.         logger.info('SysLog Time extractstartDate:{}, extractendDate:{}'.format(model_start_date_time,
  53.                                                                                 model_end_date_time))
  54.  
  55.         # Set calculated value by default
  56.         syslog_range_start_time = log_start_date_time
  57.         syslog_range_end_time = log_end_date_time
  58.         # when execution id ref taken in consideration
  59.         if request.exec_id_ref is not None:
  60.             sys_log_df, syslog_metadata = get_sys_log_data_by_execId(request, exec_vo)
  61.             is_hist_syslog_found = syslog_metadata['isHistSyslogAvailable']
  62.             is_live_syslog_found = syslog_metadata['isLiveSyslogAvailable']
  63.             hist_syslog_lines = syslog_metadata['histSyslogLines']
  64.             live_syslog_lines = syslog_metadata['liveSyslogLines']
  65.             live_min_date = syslog_metadata['liveSyslogStartTime']
  66.             live_max_date = syslog_metadata['liveSyslogEndTime']
  67.             hist_min_date = syslog_metadata['histSyslogStartTime']
  68.             hist_max_date = syslog_metadata['histSyslogEndTime']
  69.             if not sys_log_df.empty:
  70.                 err_logs_df = sys_log_df
  71.                 err_logs_df[['msg_datetime']] = err_logs_df[['msg_datetime']].astype('datetime64[ns]')
  72.                 syslog_start_time = min(err_logs_df['msg_datetime']).strftime(
  73.                     constants.DATE_TIME_FORMAT)
  74.                 syslog_end_time = max(err_logs_df['msg_datetime']).strftime(
  75.                     constants.DATE_TIME_FORMAT)
  76.             else:
  77.                 sys_log_df = pd.DataFrame(columns=['msg_code', 'msg_datetime', 'msg_text'])
  78.                 err_logs_df = sys_log_df
  79.  
  80.         else:
  81.             if exec_vo_conf['isSysLogRequired'] is True:
  82.                 if hasattr(request, 'sr_id'):
  83.                     logger.info(
  84.                         'Getting SysLog data for SrId:{}, SystemId:{}, Date:{}'.format(request.sr_id, request.system_id,
  85.                                                                                        request.date))
  86.                     event_date = log_end_date
  87.                 else:
  88.                     logger.info('Getting SysLog data for SystemId:{}'.format(request.system_id, ))
  89.  
  90.                 if exec_vo_conf['isLiveSysLogRequired'] is True:
  91.                     current_date = (datetime.datetime.now()).date()
  92.                     if current_date-event_date<=datetime.timedelta(days=2):
  93.                         live_syslog_start = datetime.datetime.now()
  94.                         live_df = get_live_sys_log_data(request, log_errors)
  95.                         live_syslog_end = datetime.datetime.now()
  96.                         live_syslog_time = (live_syslog_end - live_syslog_start).total_seconds() * 1000
  97.                         logger.info('Getting Live log between model range :{} and  {}'.format(model_start_date,model_end_date ))
  98.                     else:
  99.                         live_df = pd.DataFrame(columns=['msg_code', 'msg_datetime', 'msg_text'])
  100.  
  101.                 hist_syslog_start = datetime.datetime.now()
  102.                 try:
  103.                     hist_df, time_taken = db_util.query_frame(db_type.PG, constants.GET_SYSLOG_QUERY
  104.                                                               .format(exec_vo_conf['sysLogTable'], request.system_id,
  105.                                                                       log_end_date, log_start_date))
  106.                     logger.info(
  107.                         'Got HistSysLog data for SystemId:{}. Time taken:{} ms.'.format(request.system_id, time_taken))
  108.  
  109.                     if hist_df.empty:
  110.                         log_errors.append("No historical logs present")
  111.  
  112.                 except MDLException as e:
  113.                     logger.error('Historical logs Fetch Failed from Onwatch DB!')
  114.                     log_errors.append('Historical logs Fetch Failed from Onwatch DB! {}'.format(str(e)))
  115.                 except Exception as e:
  116.                     logger.exception('Historical logs Fetch Failed from Onwatch DB!')
  117.                     log_errors.append('Historical logs Fetch Failed from Onwatch DB! {}'.format(str(e)))
  118.  
  119.                 live_syslog_lines = len(live_df.index)
  120.                 hist_syslog_lines = len(hist_df.index)
  121.                 logger.info('liveLength:{}, histLength:{}'.format(live_syslog_lines, hist_syslog_lines))
  122.  
  123.                 # calculating min max hist and live dates
  124.                 if not hist_df.empty:
  125.                     hist_df[['msg_datetime']] = hist_df[['msg_datetime']].astype('datetime64[ns]')
  126.                     hist_df[['msg_code']] = hist_df[['msg_code']].astype(str)
  127.  
  128.                     hist_min_date = min(hist_df['msg_datetime']).strftime(
  129.                         constants.DATE_TIME_FORMAT)
  130.                     hist_max_date = max(hist_df['msg_datetime']).strftime(
  131.                         constants.DATE_TIME_FORMAT)
  132.  
  133.                 if not live_df.empty:
  134.                     live_df[['msg_datetime']] = live_df[['msg_datetime']].astype('datetime64[ns]')
  135.                     live_df[['msg_code']] = live_df[['msg_code']].astype(str)
  136.  
  137.                     live_min_date = min(live_df['msg_datetime']).strftime(
  138.                         constants.DATE_TIME_FORMAT)
  139.                     live_max_date = max(live_df['msg_datetime']).strftime(
  140.                         constants.DATE_TIME_FORMAT)
  141.  
  142.                     # merging and filtering data
  143.                     try:
  144.                         # Getting history data within the live df range
  145.                         hist_live_range_df = hist_df[(hist_df['msg_datetime'] <= live_max_date) &
  146.                                                      (hist_df['msg_datetime'] >= live_min_date)]
  147.                         # Left Merge live + hist_live_range => if both has same(duplicate), then tar_systemid is not null
  148.                         live_hist_left_merge_df = live_df.merge(hist_live_range_df,
  149.                                                                 on=['msg_datetime', 'msg_code', 'msg_text'],
  150.                                                                 how='left', indicator=True)
  151.                         # Filtering only row with tar_systemid is NULL -> live not present in history
  152.                         live_wo_hist_df = live_hist_left_merge_df[live_hist_left_merge_df.tar_systemid.isnull()]
  153.                         # Merge history + live without history (no duplicate)
  154.                         live_hist_df = pd.concat([hist_df, live_wo_hist_df]).reset_index(drop=True)
  155.  
  156.                         logger.info('liveWithoutHistLength:{}, totalLength:{}'.format(len(live_wo_hist_df.index),
  157.                                                                                       len(live_hist_df.index)))
  158.                     except MDLException as e:
  159.                         logger.error('Failed to Filter the logs!')
  160.                         log_errors.append('Failed to Filter the logs! {}'.format(str(e)))
  161.                     except Exception as e:
  162.                         logger.exception('Failed to Filter the logs!')
  163.                         log_errors.append('Failed to Filter the logs! {}'.format(str(e)))
  164.  
  165.                 else:
  166.                     live_hist_df = hist_df
  167.                     logger.info('totalLength:{}'.format(len(live_hist_df.index)))
  168.  
  169.                 sys_log_df = live_hist_df
  170.  
  171.                 if len(hist_df.index) > 0:
  172.                     is_hist_syslog_found = '1'
  173.                 if len(live_df.index) > 0:
  174.                     is_live_syslog_found = '1'
  175.  
  176.                 hist_syslog_end = datetime.datetime.now()
  177.                 hist_syslog_time = (hist_syslog_end - hist_syslog_start).total_seconds() * 1000
  178.  
  179.                 if not live_hist_df.empty:
  180.                     err_logs_df = live_hist_df[(live_hist_df.msg_datetime.dt.date <= log_end_date) &
  181.                                                (live_hist_df.msg_datetime.dt.date >= log_start_date)]
  182.  
  183.                     if not err_logs_df.empty:
  184.  
  185.  
  186.                         filtered_syslog_start_time = min(err_logs_df['msg_datetime'])
  187.                         filtered_syslog_end_time = max(err_logs_df['msg_datetime'])
  188.  
  189.                         syslog_start_time = max(log_start_date_time, filtered_syslog_start_time).strftime(
  190.                             constants.DATE_TIME_FORMAT)
  191.                         syslog_end_time = min(log_end_date_time, filtered_syslog_end_time).strftime(
  192.                             constants.DATE_TIME_FORMAT)
  193.  
  194.                         logger.info('SysLog Filtered startDate:{}, endDate:{}'.format(filtered_syslog_start_time,
  195.                                                                                       filtered_syslog_end_time))
  196.                         logger.info('SysLog Final startDate:{}, endDate:{}'.format(syslog_start_time, syslog_end_time))
  197.  
  198.                     sys_log_df = err_logs_df[['msg_code', 'msg_datetime', 'msg_text']].copy()
  199.                     if (not live_df.empty or not hist_df.empty) and sys_log_df.empty:
  200.                         log_errors.append("No Logs present after filtering live logs and historical logs")
  201.  
  202.                 else:
  203.                     sys_log_df = pd.DataFrame(columns=['msg_code', 'msg_datetime', 'msg_text'])
  204.                     err_logs_df = sys_log_df
  205.             else:
  206.                 logger.info('SysLog NOT required.')
  207.  
  208.         if not err_logs_df.empty:
  209.             err_model_logs_df = err_logs_df[(err_logs_df.msg_datetime <= model_end_date_time) & (
  210.                     err_logs_df.msg_datetime >= model_start_date_time)]
  211.  
  212.             if not err_model_logs_df.empty:
  213.                 if exec_vo_conf['isExamNoFiltered'] is True:
  214.                     err_model_logs_df = filter_syslog_exam(request, exec_vo, err_model_logs_df)
  215.  
  216.                 if exec_vo_conf['isLogDtFiltered'] is True:
  217.                     err_model_logs_df = filter_syslog_time(request, exec_vo, err_model_logs_df)
  218.                 # null check is added if filtered data came as null(err_model_logs_df) the we get json serializable error  while performing aggregations(min and max) on null fields.
  219.                 if not err_model_logs_df.empty:
  220.                     syslog_model_start_time = min(err_model_logs_df['msg_datetime']).strftime(
  221.                         constants.DATE_TIME_FORMAT)
  222.                     syslog_model_end_time = max(err_model_logs_df['msg_datetime']).strftime(
  223.                         constants.DATE_TIME_FORMAT)
  224.                     logger.info(
  225.                         'SysLog Model extract startDate:{}, endDate:{}'.format(syslog_model_start_time,
  226.                                                                                syslog_model_end_time))
  227.  
  228.                 model_sys_log_df = err_model_logs_df[['msg_code', 'msg_datetime', 'msg_text']].copy()
  229.  
  230.                 try:
  231.                     model_sys_log_df['msg_datetime'] = model_sys_log_df.msg_datetime.astype(str)
  232.                     extract_out_sys_log = model_sys_log_df.rename(columns={'msg_code': 'code',
  233.                                                                            'msg_datetime': 'time',
  234.                                                                            'msg_text': 'message'})
  235.                     model_sys_logs = json.loads(extract_out_sys_log.to_json(orient='records'))
  236.                     model_syslog_lines = len(model_sys_log_df.index)
  237.  
  238.                 except Exception as e:
  239.                     logger.exception('Extrated SysLog to Json conversion failed!')
  240.                     errors.append(' Extracted SysLog to Json conversion failed! {}'.format(str(e)))
  241.  
  242.  
  243.         try:
  244.             sys_log_df['msg_datetime'] = sys_log_df.msg_datetime.astype(str)
  245.             out_sys_log = sys_log_df.rename(columns={'msg_code': 'code',
  246.                                                      'msg_datetime': 'time',
  247.                                                      'msg_text': 'message'})
  248.             sys_log = json.loads(out_sys_log.to_json(orient='records'))
  249.  
  250.         except Exception as e:
  251.             logger.exception('SysLog to Json conversion failed!')
  252.             errors.append('SysLog to Json conversion failed! {}'.format(str(e)))
  253.  
  254.         syslog_range_start_time = syslog_range_start_time.strftime(constants.DATE_TIME_FORMAT)
  255.         syslog_range_end_time = syslog_range_end_time.strftime(constants.DATE_TIME_FORMAT)
  256.         sys_log_lines = len(sys_log_df.index)
  257.  
  258.         logger.info('sys_log_df length:{}, sys_log length:{}, is_hist_syslog_found:{}, is_live_syslog_found:{}'
  259.                     .format(len(sys_log_df), len(sys_log), is_hist_syslog_found, is_live_syslog_found))
  260.  
  261.     except MDLException as e:
  262.         logger.error('SysLog Fetch Failed!')
  263.         errors.append('SysLog Fetch Failed! {}'.format(str(e)))
  264.     except Exception as e:
  265.         logger.exception('SysLog Fetch Failed!')
  266.         errors.append('SysLog Fetch Failed! {}'.format(str(e)))
  267.  
  268.     fetched_data['is_hist_syslog_found'] = is_hist_syslog_found
  269.     fetched_data['is_live_syslog_found'] = is_live_syslog_found
  270.     fetched_data['hist_syslog_lines'] = hist_syslog_lines
  271.     fetched_data['live_syslog_lines'] = live_syslog_lines
  272.     fetched_data['sys_log_lines'] = sys_log_lines
  273.     fetched_data['model_syslog_lines'] = model_syslog_lines
  274.     fetched_data['hist_syslog_time'] = hist_syslog_time
  275.     fetched_data['live_syslog_time'] = live_syslog_time
  276.     fetched_data['syslog_start_time'] = syslog_start_time
  277.     fetched_data['syslog_end_time'] = syslog_end_time
  278.     fetched_data['syslog_range_start_time'] = syslog_range_start_time
  279.     fetched_data['syslog_range_end_time'] = syslog_range_end_time
  280.     fetched_data['model_start_time'] = syslog_model_start_time
  281.     fetched_data['model_end_time'] = syslog_model_end_time
  282.     fetched_data['sys_log'] = sys_log
  283.     fetched_data['model_sys_logs'] = model_sys_logs
  284.     fetched_data['liveSyslogStartTime'] = live_min_date
  285.     fetched_data['liveSyslogEndTime'] = live_max_date
  286.     fetched_data['histSyslogStartTime'] = hist_min_date
  287.     fetched_data['histSyslogEndTime'] = hist_max_date
  288.  
  289.     if sys_log_df is None:
  290.         sys_log_df = pd.DataFrame(columns=['msg_code', 'msg_datetime', 'msg_text'])
  291.     fetched_data['sys_log_df'] = sys_log_df
  292.     fetched_data['errors'] = errors
  293.     fetched_data['log_errors'] = log_errors
  294.  
  295.     if model_sys_log_df is None:
  296.         model_sys_log_df = pd.DataFrame(columns=['msg_code', 'msg_datetime', 'msg_text'])
  297.     fetched_data['model_sys_log_df'] = model_sys_log_df
  298.  
  299.     return fetched_data
Success #stdin #stdout 0.02s 9212KB
comments (?)
stdin 
Standard input is empty
stdout 
Standard output is empty
https://ideone.com/y62XhC
language:
Python 3 (python  3.12)
created:
3 years ago
visibility:
public

Share or Embed source code

Discover > Sphere Engine API

The brand new service which powers Ideone!

Discover > IDE Widget

Widget for compiling and running the source code in a web browser!