Add xfail to classes with failing tests

tensorflow_model_analysis/api/model_eval_lib_test.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 """Test for using the model_eval_lib API."""
 
+import pytest
 import json
 import os
 import tempfile
@@ -65,6 +66,8 @@
 _TF_MAJOR_VERSION = int(tf.version.VERSION.split('.')[0])
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class EvaluateTest(
     test_util.TensorflowModelAnalysisTest, parameterized.TestCase
 ):

tensorflow_model_analysis/evaluators/analysis_table_evaluator_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Tests for analysis_table_evaluator."""
 
+
+import pytest
 import apache_beam as beam
 from apache_beam.testing import util
 import tensorflow as tf
@@ -21,6 +23,8 @@
 from tensorflow_model_analysis.utils import test_util
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class AnalysisTableEvaulatorTest(test_util.TensorflowModelAnalysisTest):
 
   def testIncludeFilter(self):

tensorflow_model_analysis/evaluators/confidence_intervals_util_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Tests for confidence_intervals_util."""
 
+
+import pytest
 from absl.testing import absltest
 from absl.testing import parameterized
 import apache_beam as beam
@@ -35,6 +37,8 @@ def extract_output(
     return self._validate_accumulator(accumulator)
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class ConfidenceIntervalsUtilTest(parameterized.TestCase):
 
   @parameterized.named_parameters(

tensorflow_model_analysis/evaluators/jackknife_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Tests for evaluators.jackknife."""
 
+
+import pytest
 import functools
 
 from absl.testing import absltest
@@ -66,6 +68,8 @@ def add_input(self, accumulator, element):
     )
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class JackknifeTest(absltest.TestCase):
 
   def test_accumulate_only_combiner(self):

tensorflow_model_analysis/evaluators/metrics_plots_and_validations_evaluator_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for MetricsPlotsAndValidationsEvaluator with different metrics."""
 
+
+import pytest
 import os
 
 from absl.testing import parameterized
@@ -50,6 +52,8 @@
 _TF_MAJOR_VERSION = int(tf.version.VERSION.split('.')[0])
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class MetricsPlotsAndValidationsEvaluatorTest(
     testutil.TensorflowModelAnalysisTest, parameterized.TestCase
 ):

tensorflow_model_analysis/evaluators/poisson_bootstrap_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for using the poisson bootstrap API."""
 
+
+import pytest
 from absl.testing import absltest
 import apache_beam as beam
 from apache_beam.testing import util
@@ -24,6 +26,8 @@
 from tensorflow_model_analysis.metrics import metric_types
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class PoissonBootstrapTest(absltest.TestCase):
 
   def test_bootstrap_combine_fn(self):

tensorflow_model_analysis/extractors/counterfactual_predictions_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Tests for counterfactual_predictions_extactor."""
 
+
+import pytest
 import os
 import tempfile
 
@@ -51,6 +53,8 @@ def call(self, serialized_example):
     return parsed[self._feature_key]
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class CounterfactualPredictionsExtactorTest(
     test_util.TensorflowModelAnalysisTest, parameterized.TestCase
 ):

tensorflow_model_analysis/extractors/example_weights_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for example weights extractor."""
 
+
+import pytest
 from absl.testing import parameterized
 import apache_beam as beam
 from apache_beam.testing import util
@@ -30,6 +32,8 @@
 from tensorflow_metadata.proto.v0 import schema_pb2
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class ExampleWeightsExtractorTest(
     test_util.TensorflowModelAnalysisTest, parameterized.TestCase
 ):

tensorflow_model_analysis/extractors/extractor_test.py

@@ -13,13 +13,17 @@
 # limitations under the License.
 """Test for extractor."""
 
+
+import pytest
 import apache_beam as beam
 from apache_beam.testing import util
 import tensorflow as tf
 from tensorflow_model_analysis.extractors import extractor
 from tensorflow_model_analysis.utils import test_util
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class ExtractorTest(test_util.TensorflowModelAnalysisTest):
 
   def testFilterRaisesValueError(self):

tensorflow_model_analysis/extractors/features_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for features extractor."""
 
+
+import pytest
 from absl.testing import parameterized
 import apache_beam as beam
 from apache_beam.testing import util
@@ -29,6 +31,8 @@
 from tensorflow_metadata.proto.v0 import schema_pb2
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class FeaturesExtractorTest(
     test_util.TensorflowModelAnalysisTest, parameterized.TestCase
 ):

tensorflow_model_analysis/extractors/inference_base_test.py

@@ -17,6 +17,8 @@
 tfx_bsl_predictions_extractor_test.py.
 """
 
+
+import pytest
 import os
 
 import tensorflow as tf
@@ -35,6 +37,8 @@
 from tensorflow_serving.apis import prediction_log_pb2
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class TfxBslPredictionsExtractorTest(testutil.TensorflowModelAnalysisTest):
 
   def setUp(self):

tensorflow_model_analysis/extractors/labels_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for labels extractor."""
 
+
+import pytest
 from absl.testing import parameterized
 import apache_beam as beam
 from apache_beam.testing import util
@@ -30,6 +32,8 @@
 from tensorflow_metadata.proto.v0 import schema_pb2
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class LabelsExtractorTest(
     test_util.TensorflowModelAnalysisTest, parameterized.TestCase
 ):

tensorflow_model_analysis/extractors/legacy_input_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Tests for input extractor."""
 
+
+import pytest
 import apache_beam as beam
 from apache_beam.testing import util
 import numpy as np
@@ -24,6 +26,8 @@
 from tensorflow_model_analysis.utils import test_util
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class InputExtractorTest(test_util.TensorflowModelAnalysisTest):
 
   def testInputExtractor(self):

tensorflow_model_analysis/extractors/legacy_meta_feature_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for using the MetaFeatureExtractor as part of TFMA."""
 
+
+import pytest
 import apache_beam as beam
 from apache_beam.testing import util
 import numpy as np
@@ -70,6 +72,8 @@ def get_num_interests(fpl):
   return new_features
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class MetaFeatureExtractorTest(test_util.TensorflowModelAnalysisTest):
 
   def testMetaFeatures(self):

tensorflow_model_analysis/extractors/materialized_predictions_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for batched materialized predictions extractor."""
 
+
+import pytest
 import apache_beam as beam
 from apache_beam.testing import util
 import numpy as np
@@ -30,6 +32,8 @@
 from tensorflow_metadata.proto.v0 import schema_pb2
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class MaterializedPredictionsExtractorTest(
     testutil.TensorflowModelAnalysisTest
 ):

tensorflow_model_analysis/extractors/predictions_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for batched predict extractor."""
 
+
+import pytest
 import os
 
 from absl.testing import parameterized
@@ -34,6 +36,8 @@
 from tensorflow_metadata.proto.v0 import schema_pb2
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class PredictionsExtractorTest(
     test_util.TensorflowModelAnalysisTest, parameterized.TestCase
 ):

tensorflow_model_analysis/extractors/slice_key_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for slice_key_extractor."""
 
+
+import pytest
 from absl.testing import parameterized
 import apache_beam as beam
 from apache_beam.testing import util
@@ -64,6 +66,8 @@ def wrap_fpl(fpl):
   }
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class SliceTest(test_util.TensorflowModelAnalysisTest, parameterized.TestCase):
 
   @parameterized.named_parameters(

tensorflow_model_analysis/extractors/sql_slice_key_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Tests for tensorflow_model_analysis.google.extractors.sql_slice_key_extractor."""
 
+
+import pytest
 import apache_beam as beam
 from apache_beam.testing import util
 import numpy as np
@@ -50,6 +52,8 @@
 )
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class SqlSliceKeyExtractorTest(test_util.TensorflowModelAnalysisTest):
 
   def testSqlSliceKeyExtractor(self):

tensorflow_model_analysis/extractors/tfjs_predict_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Tests for tfjs predict extractor."""
 
+
+import pytest
 import tempfile
 
 from absl.testing import parameterized
@@ -39,6 +41,8 @@
   _TFJS_IMPORTED = False
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class TFJSPredictExtractorTest(
     testutil.TensorflowModelAnalysisTest, parameterized.TestCase
 ):

tensorflow_model_analysis/extractors/transformed_features_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for transformed features extractor."""
 
+
+import pytest
 import tempfile
 import unittest
 
@@ -36,6 +38,8 @@
 _TF_MAJOR_VERSION = int(tf.version.VERSION.split('.')[0])
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class TransformedFeaturesExtractorTest(
     testutil.TensorflowModelAnalysisTest, parameterized.TestCase
 ):

tensorflow_model_analysis/extractors/unbatch_extractor_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Test for unbatch extractor."""
 
+
+import pytest
 import apache_beam as beam
 from apache_beam.testing import util
 import numpy as np
@@ -32,6 +34,8 @@
 from tensorflow_metadata.proto.v0 import schema_pb2
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class UnbatchExtractorTest(testutil.TensorflowModelAnalysisTest):
 
   def testExtractUnbatchedInputsRaisesChainedException(self):

tensorflow_model_analysis/metrics/aggregation_test.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 """Tests for aggregation metrics."""
 
+
+import pytest
 import copy
 import apache_beam as beam
 from apache_beam.testing import util
@@ -25,6 +27,8 @@
 from tensorflow_model_analysis.utils import test_util
 
 
+@pytest.mark.xfail(run=False, reason="PR 183 This class contains tests that fail and needs to be fixed. "
+"If all tests pass, please remove this mark.")
 class AggregationMetricsTest(test_util.TensorflowModelAnalysisTest):
 
   def testOutputAverage(self):