diff --git a/odmtools/controller/frmDataTable.py b/odmtools/controller/frmDataTable.py index ce6f008..be6ccbe 100644 --- a/odmtools/controller/frmDataTable.py +++ b/odmtools/controller/frmDataTable.py @@ -13,17 +13,14 @@ def __init__(self, parent, **kwargs): self.memDB = None DataTable.__init__(self, parent, **kwargs) + def init(self, memDB): + self.memDB = memDB + self.olvDataTable.init(self.memDB) - def init_publishers(self): Publisher.subscribe(self.onChangeSelection, "changeTableSelection") Publisher.subscribe(self.onRefresh, "refreshTable") Publisher.subscribe(self.olvDataTable.onDeselectAll, "deselectAllDataTable") - def init(self, memDB): - self.memDB = memDB - self.olvDataTable.init(self.memDB) - self.init_publishers() - def onItemSelected(self, event): pass diff --git a/odmtools/controller/olvDataTable.py b/odmtools/controller/olvDataTable.py index 112f4d7..d3f0029 100644 --- a/odmtools/controller/olvDataTable.py +++ b/odmtools/controller/olvDataTable.py @@ -20,6 +20,8 @@ def __init__(self, parent, **kwargs): self.sortedColumnIndex = -1 self.currentItem = None self.dataframe = None + self.annotations = None + self.annotations_grouped = {} def init(self, memDB): self.memDB = memDB @@ -28,21 +30,59 @@ def init(self, memDB): self.oddRowsBackColor = wx.Colour(191, 217, 217) self.dataframe = self.memDB.getDataValuesDF() - sort_by_index = list(self.dataframe.columns).index("valuedatetime") + self.annotations = self.memDB.get_annotations() + + sort_by_index = self.dataframe.columns.tolist().index("valuedatetime") self.dataframe.sort_values(self.dataframe.columns[sort_by_index], inplace=True) - self.dataObjects = self.dataframe.values.tolist() + + self.annotations_grouped = self.__group_annotations() + self.dataObjects = self.__merge_dataframe_with_annotations() + + col = self.memDB.get_columns_with_annotations() + columns = \ [ColumnDefn(x.strip(), align="left", valueGetter=i, minimumWidth=125, width=125, - stringConverter='%Y-%m-%d %H:%M:%S' if "valuedatetime" == x.lower() else '%s') - for x, i in self.memDB.getEditColumns()] + stringConverter='%Y-%m-%d %H:%M:%S' if "valuedatetime" == x.lower() else '%s') + for x, i in col] + self.SetColumns(columns) self.SetObjectGetter(self.ObjectGetter) - self.SetItemCount(len(self.dataframe)) + self.SetItemCount(len(self.dataObjects)) + + def __merge_dataframe_with_annotations(self): + data_list = self.dataframe.values.tolist() + data = data_list + + for key, value in self.annotations_grouped.iteritems(): + for i in range(0, len(data_list)): + if key in data[i]: + data[i].append(value) + break + + return data + + def __group_annotations(self): + """ + Ideally, this method should only be called once. Use self.grouped_annotations after calling this method + :return: + """ + anno_list = self.annotations.values.tolist() + + anno = {} + for i in range(0, len(anno_list)): + value_id = anno_list[i][1] + annotation_code = anno_list[i][-1] + if value_id in anno: + anno[value_id].append(annotation_code) + else: + anno[value_id] = [annotation_code] + + return anno def EnableSorting(self): - self.Bind(wx.EVT_LIST_COL_CLICK, self.onColSelected) + self.Bind(wx.EVT_LIST_COL_CLICK, self.on_column_selected) if not self.smallImageList: self.SetImageLists() if (not self.smallImageList.HasName(ObjectListView.NAME_DOWN_IMAGE) and @@ -56,40 +96,58 @@ def ObjectGetter(self, index): """ return self.dataObjects[index % len(self.dataObjects)] - def onColSelected(self, evt): + def on_column_selected(self, event): """ Allows users to sort by clicking on columns """ - if isinstance(self.dataframe, pd.DataFrame): - if self.dataframe.empty: - return - else: - if not self.dataframe: - return + if not isinstance(self.dataframe, pd.DataFrame): + return + + if self.dataframe.empty: + return - logger.debug("Column: %s" % evt.m_col) - self.sortColumn(evt.m_col) + if not len(self.dataObjects): + return + + self.sortColumn(event.Column) def sortColumn(self, selected_column): + self.sortAscending = not self.sortAscending + oldSortColumnIndex = self.sortedColumnIndex self.sortedColumnIndex = selected_column - ascending = self.sortAscending - if ascending: - self.dataframe.sort_values(self.dataframe.columns[selected_column], inplace=True) - self.sortAscending = False - elif not ascending: - self.dataframe.sort_values(self.dataframe.columns[selected_column], ascending=False, inplace=True) - self.sortAscending = True self._UpdateColumnSortIndicators(selected_column, oldSortColumnIndex) - self.dataObjects = self.dataframe.values.tolist() - if self.GetItemCount: + if selected_column >= len(self.dataframe.columns): + self.dataObjects = self.sort_columns_by_annotation_code(reverse=self.sortAscending) + else: + self.dataframe.sort_values(self.dataframe.columns[selected_column], ascending=self.sortAscending, inplace=True) + self.dataObjects = self.__merge_dataframe_with_annotations() + + if self.GetItemCount(): itemFrom = self.GetTopItem() itemTo = self.GetTopItem() + 1 + self.GetCountPerPage() itemTo = min(itemTo, self.GetItemCount() - 1) self.RefreshItems(itemFrom, itemTo) + def sort_columns_by_annotation_code(self, reverse=False): + rows_with_annotation = [] + rows_without_annotation = [] + + column_number_of_dataframe = len(self.dataframe.columns) + + for i in self.dataObjects: + if len(i) > column_number_of_dataframe: + rows_with_annotation.append(i) + else: + rows_without_annotation.append(i) + + if reverse: + return rows_without_annotation + rows_with_annotation + else: + return rows_with_annotation + rows_without_annotation + def onItemSelected(self, event): """ diff --git a/odmtools/gui/mnuRibbon.py b/odmtools/gui/mnuRibbon.py index 6c21874..eede1aa 100644 --- a/odmtools/gui/mnuRibbon.py +++ b/odmtools/gui/mnuRibbon.py @@ -129,6 +129,7 @@ def _init_ctrls(self, prnt): # ------------------------------------------------------------------------------- editPage = RB.RibbonPage(self, wx.ID_ANY, "Edit") + # editPage.Bind(wx.EVT_ENTER_WINDOW, self.on_mouse_enter) main_panel = RB.RibbonPanel(editPage, wx.ID_ANY, "Main", wx.NullBitmap, wx.DefaultPosition, wx.DefaultSize, RB.RIBBON_PANEL_NO_AUTO_MINIMISE) @@ -204,14 +205,28 @@ def _init_ctrls(self, prnt): self.CurrPage = 1 self.SetActivePageByIndex(self.CurrPage) - self.bindEvents() + self.__bind_events() self.initPubSub() def __init__(self, parent, id, name): self.parent = parent self._init_ctrls(parent) - def bindEvents(self): + def on_mouse_enter(self, event): + ribbon_panel = event.GetEventObject().GetParent() + ribbon_panel._hovered = True + + self.Refresh() + event.Skip() + + def on_mouse_leave(self, event): + ribbon_panel = event.GetEventObject().GetParent() + ribbon_panel._hovered = False + + self.Refresh() + event.Skip() + + def __bind_events(self): ###Docking Window Selection self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onDocking, id=wxID_RIBBONVIEWTABLE) self.Bind(RB.EVT_RIBBONBUTTONBAR_CLICKED, self.onDocking, id=wxID_RIBBONVIEWSERIES) @@ -262,6 +277,24 @@ def bindEvents(self): ###Ribbon Event self.Bind(RB.EVT_RIBBONBAR_PAGE_CHANGED, self.onFileMenu, id=wxID_PANEL1) + # ENTER + self.main_bar.Bind(wx.EVT_ENTER_WINDOW, self.on_mouse_enter) # 1 + self.edit_bar.Bind(wx.EVT_ENTER_WINDOW, self.on_mouse_enter) # 2 + self.record_bar.Bind(wx.EVT_ENTER_WINDOW, self.on_mouse_enter) # 3 + self.PlotsOptions_bar.Bind(wx.EVT_ENTER_WINDOW, self.on_mouse_enter) # 4 + self.plots_bar.Bind(wx.EVT_ENTER_WINDOW, self.on_mouse_enter) # 5 + self.dateTime_buttonbar.Bind(wx.EVT_ENTER_WINDOW, self.on_mouse_enter) # 6 + self.scriptBar.Bind(wx.EVT_ENTER_WINDOW, self.on_mouse_enter) # 7 + + # LEAVE + self.main_bar.Bind(wx.EVT_LEAVE_WINDOW, self.on_mouse_leave) # 1 + self.edit_bar.Bind(wx.EVT_LEAVE_WINDOW, self.on_mouse_leave) # 2 + self.record_bar.Bind(wx.EVT_LEAVE_WINDOW, self.on_mouse_leave) # 3 + self.PlotsOptions_bar.Bind(wx.EVT_LEAVE_WINDOW, self.on_mouse_leave) # 4 + self.plots_bar.Bind(wx.EVT_LEAVE_WINDOW, self.on_mouse_leave) # 5 + self.dateTime_buttonbar.Bind(wx.EVT_LEAVE_WINDOW, self.on_mouse_leave) # 6 + self.scriptBar.Bind(wx.EVT_LEAVE_WINDOW, self.on_mouse_leave) # 7 + def initPubSub(self): Publisher.subscribe(self.toggleEditButtons, "EnableEditButtons") Publisher.subscribe(self.enableButtons, "EnablePlotButtons") diff --git a/odmtools/odmdata/memory_database.py b/odmtools/odmdata/memory_database.py index 813fbe9..2d9d24e 100644 --- a/odmtools/odmdata/memory_database.py +++ b/odmtools/odmdata/memory_database.py @@ -25,6 +25,7 @@ def __init__(self, taskserver=None): self.df = None # Series_Service handles remote database self.series_service = None + self.results_annotations = None # Memory_service handles in memory database sm = ServiceManager() @@ -43,18 +44,14 @@ def __init__(self, taskserver=None): #self.annotation_list = pd.DataFrame() columns =['ResultID', 'ValueDateTime', 'ValueID', 'AnnotationID') #send in engine - def reset_edit(self): sm = ServiceManager() self.mem_service = sm.get_series_service(conn_string="sqlite:///:memory:") setSchema(self.mem_service._session_factory.engine) - def set_series_service(self, service): self.series_service = service - - ############## # DB Queries ############## @@ -77,6 +74,13 @@ def getDataValuesDF(self): logging.debug("done updating memory dataframe") return self.df + def get_annotations(self): + result_id = self.df.resultid[0] + setSchema(self.series_service._session_factory.engine) + annotation = self.series_service.get_annotations_by_result(resultid=result_id) + self.results_annotations = annotation + return self.results_annotations + def getDataValues(self): # TODO: fix me! this commit location is only temoporarily. should be flushing so that we can restore self.mem_service._session.commit() @@ -94,6 +98,27 @@ def getEditColumns(self): return [(x, i) for (i, x) in enumerate(columns)] # return [(x, i) for (i, x) in enumerate(self.df.columns)] + def get_columns_with_annotations(self): + """ + If results_annotations has not been set then + :return: + """ + + if self.results_annotations is None or self.df is None: + print "self.df and self.results_annotations must be a pandas dataframe. Currently they are None" + return [] + + columns = [] + columns.extend(self.df.columns.tolist()) + + annotation_columns = self.results_annotations.columns.tolist() + index = annotation_columns.index("annotationcode") + annotation_code_column = annotation_columns[index] + + columns.append(annotation_code_column) + + return [(x, i) for (i, x) in enumerate(columns)] + def getDataValuesforGraph(self, seriesID, noDataValue, startDate=None, endDate=None): return self.series_service.get_plot_values(seriesID, noDataValue, startDate, endDate) @@ -250,7 +275,7 @@ def initEditValues(self, seriesID): logger.debug("Load series from db") self.series = self.series_service.get_series(seriesID) - self.df = self.series_service.get_values(series_id= seriesID) + self.df = self.series_service.get_values(series_id=seriesID) self.editLoaded = True diff --git a/odmtools/odmservices/series_service.py b/odmtools/odmservices/series_service.py index aec7c69..9a98d30 100644 --- a/odmtools/odmservices/series_service.py +++ b/odmtools/odmservices/series_service.py @@ -116,26 +116,6 @@ def get_variables_by_site_code(self, site_code): q = self._session.query(Variables).filter(Variables.VariableID.in_(var_ids)) return q.all() - # Data Value Methods - def get_values(self, series_id=None): - ''' - - :param series_id: Series id - :return: pandas dataframe - ''' - - setSchema(self._session_factory.engine) - q = self.read._session.query(TimeSeriesResultValues) - if series_id: - q = q.filter_by(ResultID=series_id) - q = q.order_by(TimeSeriesResultValues.ValueDateTime) - query = q.statement.compile(dialect=self._session_factory.engine.dialect) - data = pd.read_sql_query(sql=query, - con=self._session_factory.engine, - params=query.params) - data.set_index(data['valuedatetime'], inplace=True) - return data - # Series Catalog methods def get_series_by_site(self , site_id): # try: @@ -359,42 +339,45 @@ def get_series_from_filter(self): # Pass in probably a Series object, match it against the database pass -# + #Data Value Methods def get_values(self, series_id=None): - ''' - :param series_id: Series id + """ + :param series_id: :return: pandas dataframe - ''' - #series= self.get_series_by_id(series_id) - # if series: - # q = self._edit_session.query(DataValue).filter_by( - # site_id=series.site_id, - # variable_id=series.variable_id, - # method_id=series.method_id, - # source_id=series.source_id, - # quality_control_level_id=series.quality_control_level_id) - # - # query=q.statement.compile(dialect=self._session_factory.engine.dialect) - # data= pd.read_sql_query(sql= query, - # con = self._session_factory.engine, - # params = query.params ) - # #return data.set_index(data['LocalDateTime']) - # return data - # else: - # return None + """ + # see get_annotations_by_result around line 850 q = self.read._session.query(TimeSeriesResultValues) if series_id: - q=q.filter_by(ResultID=series_id) - q= q.order_by(TimeSeriesResultValues.ValueDateTime) + q = q.filter_by(ResultID=series_id) + q = q.order_by(TimeSeriesResultValues.ValueDateTime) query = q.statement.compile(dialect=self._session_factory.engine.dialect) data = pd.read_sql_query(sql=query, con=self._session_factory.engine, params=query.params) data.set_index(data['valuedatetime'], inplace=True) + + # if series_id: + # anno = self.get_annotations_by_result(series_id) + # q = pd.merge(data, anno, how="left", on='valueid', indicator=False) + # data.applymap(self.merge_annotation_with_timeseries_result) + + # if len(anno): + # # data.valueid.apply(self.merge_annotation_with_timeseries_result) + # # data.applymap(self.merge_annotation_with_timeseries_result) + # self.merge_annotation_with_timeseries_result(data, anno) + + + return data + # df.drop_duplicates(dup_cols, keep='last', inplace=True) + # newdf = pd.merge(df, pd.read_sql(query, engine), how='left', on=dup_cols, indicator=True) + # newdf = newdf[newdf['_merge'] == 'left_only'] + # newdf.drop(['_merge'], axis=1, inplace=True) + # return df[df['valuedatetime'].isin(newdf['valuedatetime'])] + def get_all_values_df(self): """ @@ -870,24 +853,31 @@ def get_quality_code(self): return self.read.getCVs(type="Quality Code") def get_annotation_by_code(self, code): - return self.read.getAnnotations(codes=[code])[0] + try: + return self.read.getAnnotations(codes=[code])[0] + except: + return None + def get_annotation_by_id(self, id): - return self.read.getAnnotations(ids=[id])[0] + try: + return self.read.getAnnotations(ids=[id])[0] + except: + return None + def get_all_annotations(self): - return self.read.getAnnotations(type=None) + try: + return self.read.getAnnotations(type=None) + except: + return None def get_annotations_by_result(self, resultid): - setSchema(self._session_factory.engine) - - # ids = [x[0] for x in self.read._session.query(TimeSeriesResultValues.ValueID)\ - # .filter(TimeSeriesResultValues.ResultID == resultid).all()] - # q = self.read._session.query(TimeSeriesResultValueAnnotations)\ - # .filter(TimeSeriesResultValueAnnotations.ValueID.in_(ids)).all() + resultid = int(resultid) - q =self.read._session.query(TimeSeriesResultValueAnnotations.AnnotationID, TimeSeriesResultValueAnnotations.ValueID, - TimeSeriesResultValues.ResultID, TimeSeriesResultValues.ValueDateTime)\ + q = self.read._session.query(TimeSeriesResultValueAnnotations.AnnotationID, TimeSeriesResultValueAnnotations.ValueID, + TimeSeriesResultValues.ResultID, TimeSeriesResultValues.ValueDateTime, Annotations.AnnotationCode)\ .filter(TimeSeriesResultValues.ResultID == resultid)\ - .filter(TimeSeriesResultValueAnnotations.ValueID == TimeSeriesResultValues.ValueID) + .filter(TimeSeriesResultValueAnnotations.ValueID == TimeSeriesResultValues.ValueID)\ + .filter(Annotations.AnnotationID==TimeSeriesResultValueAnnotations.AnnotationID) query = q.statement.compile(dialect=self._session_factory.engine.dialect) data = pd.read_sql_query(sql=query, con=self._session_factory.engine,