- added autodetection of columns with links

- tests adapted for new functionality

Author: andreygetmanov

fedot/core/constants.py

@@ -14,7 +14,7 @@
 
 MINIMAL_PIPELINE_NUMBER_FOR_EVALUATION = 100
 
-FRACTION_OF_UNIQUE_VALUES = 0.7
+FRACTION_OF_UNIQUE_VALUES = 0.6
 MIN_VOCABULARY_SIZE = 20
 
 default_data_split_ratio_by_task = {

fedot/core/data/data_detection.py

@@ -39,12 +39,29 @@ def define_text_columns(self, data_frame: pd.DataFrame) -> List[str]:
                 text_columns.append(column_name)
         return text_columns
 
+    def define_link_columns(self, data_frame: pd.DataFrame) -> List[str]:
+        """
+        :param data_frame: pandas dataframe with data
+        :return: list of link columns' names
+        """
+        link_columns = []
+        for column_name in data_frame.columns:
+            if self.is_link(data_frame[column_name]):
+                link_columns.append(column_name)
+        return link_columns
+
     @staticmethod
     def is_full_of_nans(text_data: np.array) -> bool:
         if np.sum(pd.isna(text_data)) / len(text_data) > ALLOWED_NAN_PERCENT:
             return True
         return False
 
+    @staticmethod
+    def is_link(text_data: np.array) -> bool:
+        if str(next(el for el in text_data if el is not None)).startswith('http'):
+            return True
+        return False
+
     @staticmethod
     def prepare_multimodal_data(dataframe: pd.DataFrame, columns: List[str]) -> dict:
         """ Prepares MultiModal text data in a form of dictionary
@@ -71,12 +88,18 @@ def _column_contains_text(self, column: pd.Series) -> bool:
         Column contains text if:
         1. it's not float or float compatible
         (e.g. ['1.2', '2.3', '3.4', ...] is float too)
-        2. fraction of unique values (except nans) is more than 0.95
+        2. fraction of unique values (except nans) is more than 0.6
+        3. size of tfidf vocabulary is more than 20
+
+        If size of tfidf vocabulary is less than 20, then it is probably
+        text column too, but it cannot be vectorized and used in model
 
         :param column: pandas series with data
-        :return: True if column contains text
+        :return: True if column contains text, False otherwise or if column contains links
         """
-        if column.dtype == object and not self._is_float_compatible(column) and self._has_unique_values(column):
+        if self.is_link(column):
+            return False
+        elif column.dtype == object and not self._is_float_compatible(column) and self._has_unique_values(column):
             try:
                 params = DefaultOperationParamsRepository().get_default_params_for_operation('tfidf')
                 tfidf_vectorizer = TfidfVectorizer(**params)

fedot/core/data/multi_modal.py

@@ -168,14 +168,16 @@ def from_csv(cls,
         if not text_columns:
             text_columns = text_data_detector.define_text_columns(data_frame)
 
+        link_columns = text_data_detector.define_link_columns(data_frame)
+        columns_to_drop = text_columns + link_columns
         data_text = text_data_detector.prepare_multimodal_data(data_frame, text_columns)
-        data_frame_table = data_frame.drop(columns=text_columns)
+        data_frame_table = data_frame.drop(columns=columns_to_drop)
         table_features, target = process_target_and_features(data_frame_table, target_columns)
 
         data_part_transformation_func = partial(array_to_input_data,
                                                 idx=idx, target_array=target, task=task)
 
-        # create labels for text data sources and remove source if there are many nans
+        # create labels for text data sources and remove source if there are many nans or text is link
         sources = dict((text_data_detector.new_key_name(data_part_key),
                         data_part_transformation_func(features_array=data_part, data_type=DataTypesEnum.text))
                        for (data_part_key, data_part) in data_text.items()

test/data/multimodal_data_with_complicated_types.csv

@@ -5,7 +5,7 @@
 3,3,,4,4,3,"make a type specimen book. It has survived not only five centuries, but also",4,  a  ,True,,,0,2,yes
 4,4,,5,5,0,"the leap into electronic typesetting, remaining essentially unchanged. It was",5,   b ,,,,0,3,no
 5,5,,6,6,0,popularised in the 1960s with the release of Letraset sheets containing Lorem ,6,   c  ,False,,,0,4,no
-6,6,inf,7,7,0,"Ipsum passages, and more recently with desktop publishing software like Aldus ",7,    a  ,True,sample text,sample text,1,5,no
+6,6,inf,7,7,0,"Ipsum passages, and more recently with desktop publishing software like Aldus ",7,    a  ,True,"Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum. Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of ""de Finibus Bonorum et Malorum"" (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, ""Lorem ipsum dolor sit amet.."", comes from a line in section 1.10.32.","Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.",1,5,no
 7,7,inf,8,8,1,PageMaker including versions of Lorem Ipsum.,1, b   ,,,4,0,6,no
 8,inf,inf,9,9,2,"Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots ",2,,True,,,1,7,no
 9,9,inf,10,10,2,"in a piece of classical Latin literature from 45 BC, making it over 2000 years ",3, c  ,False,,,0,8,yes
@@ -16,4 +16,4 @@
 14,14,,3,3,2,"1.10.32 and 1.10.33 of ""de Finibus Bonorum et Malorum"" (The Extremes of Good ",1,a,False,,,,error,no
 15,15,,4,4,1,"and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of",2,a  ,False,,,,13,no
 16,16,2,5,12,0,"ethics, very popular during the Renaissance. The first line of Lorem Ipsum,",3,   d       ,True,,,1,16,yes
-17,17,3,6,13,0,"""Lorem ipsum dolor sit amet.."", comes from a line in section 1.10.32.",4,  d      ,False,,another sample text,0,17,no
+17,17,3,6,13,0,"""Lorem ipsum dolor sit amet.."", comes from a line in section 1.10.32.",4,  d      ,False,,"Contrary to popular belief, Lorem Ipsum is not simply random text. It has roots in a piece of classical Latin literature from 45 BC, making it over 2000 years old. Richard McClintock, a Latin professor at Hampden-Sydney College in Virginia, looked up one of the more obscure Latin words, consectetur, from a Lorem Ipsum passage, and going through the cites of the word in classical literature, discovered the undoubtable source. Lorem Ipsum comes from sections 1.10.32 and 1.10.33 of ""de Finibus Bonorum et Malorum"" (The Extremes of Good and Evil) by Cicero, written in 45 BC. This book is a treatise on the theory of ethics, very popular during the Renaissance. The first line of Lorem Ipsum, ""Lorem ipsum dolor sit amet.."", comes from a line in section 1.10.32.",0,17,no

test/unit/data/test_multimodal_data.py

@@ -43,7 +43,7 @@ def test_multi_modal_data():
     multi_modal.target = new_target
     assert np.array_equal(multi_modal.target, new_target)
 
-
+# TODO make test of text columns autodetection
 def test_multimodal_data_from_csv():
     """
     Checking correctness of MultiModalData import from csv file.
@@ -54,7 +54,7 @@ def test_multimodal_data_from_csv():
     text_data = np.array(df['description'])
     table_data = np.array(df.drop(columns=['id', 'description', 'variety']))
     target = np.array(df['variety']).reshape(-1, 1)
-    actual_data = MultiModalData.from_csv(path)
+    actual_data = MultiModalData.from_csv(path, text_columns=['description'])
     actual_text_features = actual_data['data_source_text/description'].features
     actual_table_features = actual_data['data_source_table'].features
     actual_target = actual_data.target