From 0e6fb2118b6a3d127703b56ade1ca7c8469117c9 Mon Sep 17 00:00:00 2001 From: Evgeniia Nugmanova Date: Tue, 12 Nov 2024 11:19:09 +0400 Subject: [PATCH 1/2] Partial Value optimization Signed-off-by: Evgeniia Nugmanova --- .../symbol_optimization.cpp | 182 +++++++++++++----- .../symbol_optimization.cpp | 25 +-- 2 files changed, 139 insertions(+), 68 deletions(-) diff --git a/src/common/transformations/src/transformations/symbolic_transformations/symbol_optimization.cpp b/src/common/transformations/src/transformations/symbolic_transformations/symbol_optimization.cpp index 55f0794e0ee008..07583ae486ae74 100644 --- a/src/common/transformations/src/transformations/symbolic_transformations/symbol_optimization.cpp +++ b/src/common/transformations/src/transformations/symbolic_transformations/symbol_optimization.cpp @@ -18,6 +18,7 @@ #include "openvino/op/squeeze.hpp" #include "openvino/op/util/multi_subgraph_base.hpp" #include "openvino/op/util/op_types.hpp" +#include "transformations/symbolic_transformations/utils.hpp" #include "transformations/utils/utils.hpp" namespace { @@ -84,27 +85,28 @@ int64_t get_idx_of_symbol_in_source(const ov::Output& source, const st } ov::Output alternative_source_from_existing_value(const std::shared_ptr& symbol, - const ov::Output& original_output, + const ov::Shape& original_shape, + const ov::element::Type& original_et, + const std::shared_ptr& node_to_copy_rt_info, STS_map& symbol_value_source) { auto alternative_source = ov::Output(); if (symbol_value_source.count(symbol)) { alternative_source = symbol_value_source[symbol]; - const auto &original_shape = original_output.get_shape(), &alternative_shape = alternative_source.get_shape(); - const auto &original_et = original_output.get_element_type(), - &alternative_et = alternative_source.get_element_type(); + const auto& alternative_shape = alternative_source.get_shape(); + const auto& alternative_et = alternative_source.get_element_type(); if (alternative_shape != original_shape && (original_shape.empty() || original_shape == ov::Shape{0})) { auto squeeze = std::make_shared(alternative_source); - ov::copy_runtime_info(original_output.get_node_shared_ptr(), squeeze); + ov::copy_runtime_info(node_to_copy_rt_info, squeeze); alternative_source = squeeze->output(0); } else if (alternative_shape != original_shape) { auto shape = ov::op::v0::Constant::create(ov::element::i64, {original_shape.size()}, original_shape); auto reshape = std::make_shared(alternative_source, shape, false); - ov::copy_runtime_info(original_output.get_node_shared_ptr(), reshape); + ov::copy_runtime_info(node_to_copy_rt_info, reshape); alternative_source = reshape->output(0); } if (alternative_et != original_et) { auto convert = std::make_shared(alternative_source, original_et); - ov::copy_runtime_info(original_output.get_node_shared_ptr(), convert); + ov::copy_runtime_info(node_to_copy_rt_info, convert); alternative_source = convert->output(0); } } @@ -113,7 +115,9 @@ ov::Output alternative_source_from_existing_value(const std::shared_pt ov::Output alternative_source_from_shape_source(const STS_map& symbol_shape_source, const std::shared_ptr& symbol, - const ov::Output& original_output, + const ov::Shape& original_shape, + const ov::element::Type& original_et, + const std::shared_ptr& node_to_copy_rt_info, STS_map& symbol_value_source) { auto alternative_source = ov::Output(); if (symbol_shape_source.count(symbol)) { @@ -122,39 +126,61 @@ ov::Output alternative_source_from_shape_source(const STS_map& symbol_ const int64_t& idx = get_idx_of_symbol_in_source(source, symbol); if (idx == -1) return alternative_source; - const auto& original_et = original_output.get_element_type(); std::shared_ptr shape; if (original_et == ov::element::i32 || original_et == ov::element::i64) { shape = std::make_shared(source, original_et); } else { shape = std::make_shared(source); - ov::copy_runtime_info(original_output.get_node_shared_ptr(), shape); + ov::copy_runtime_info(node_to_copy_rt_info, shape); shape = std::make_shared(shape, original_et); } - auto indices = ov::op::v0::Constant::create(ov::element::i64, original_output.get_shape(), {idx}); + auto indices = ov::op::v0::Constant::create(ov::element::i64, original_shape, {idx}); auto axis = ov::op::v0::Constant::create(ov::element::i64, {}, {0}); auto gather = std::make_shared(shape, indices, axis); - ov::copy_runtime_info(original_output.get_node_shared_ptr(), {shape, indices, axis, gather}); + ov::copy_runtime_info(node_to_copy_rt_info, {shape, indices, axis, gather}); alternative_source = gather; symbol_value_source[symbol] = alternative_source; } return alternative_source; } -ov::Output get_alternative_source_from_value_or_shape_source(const STS_map& symbol_shape_source, - const std::shared_ptr& symbol, - const ov::Output& original_output, - STS_map& symbol_value_source) { +ov::Output get_alternative_source_from_value_or_shape_source( + const STS_map& symbol_shape_source, + const std::shared_ptr& symbol, + const ov::Shape& original_shape, + const ov::element::Type& original_et, + const std::shared_ptr& node_to_copy_rt_info, + STS_map& symbol_value_source) { auto alternative_source = ov::Output(); if (symbol == nullptr) return alternative_source; - alternative_source = alternative_source_from_existing_value(symbol, original_output, symbol_value_source); + alternative_source = alternative_source_from_existing_value(symbol, + original_shape, + original_et, + node_to_copy_rt_info, + symbol_value_source); if (!alternative_source.get_node_shared_ptr()) - alternative_source = - alternative_source_from_shape_source(symbol_shape_source, symbol, original_output, symbol_value_source); + alternative_source = alternative_source_from_shape_source(symbol_shape_source, + symbol, + original_shape, + original_et, + node_to_copy_rt_info, + symbol_value_source); return alternative_source; } +ov::Output get_alternative_source_from_value_or_shape_source(const STS_map& symbol_shape_source, + const std::shared_ptr& symbol, + const ov::Output& original_output, + STS_map& symbol_value_source) { + return get_alternative_source_from_value_or_shape_source(symbol_shape_source, + symbol, + original_output.get_shape(), + original_output.get_element_type(), + original_output.get_node_shared_ptr(), + symbol_value_source); +} + ov::Output alternative_source_from_concat_input_sources(const STS_map& symbol_shape_source, const std::shared_ptr& symbol, const ov::Output& original_output, @@ -198,7 +224,9 @@ ov::Output alternative_source_from_concat_input_sources(const STS_map& return alternative_source; } -void optimize_value_usage(ov::Output& output, STS_map& symbol_shape_source, STS_map& symbol_value_source) { +void optimize_single_value_usage(ov::Output& output, + STS_map& symbol_shape_source, + STS_map& symbol_value_source) { auto value_symbols = output.get_tensor().get_value_symbol(); if (value_symbols.size() != 1) return; @@ -316,16 +344,16 @@ std::vector> topological_order(const std::shared_ptr& op, STS_map& symbol_shape_source) { - if (ov::is_type(op) || ov::is_type(op)) { - const auto& output = op->input_value(0); + bool is_shape = ov::is_type(op) || ov::is_type(op); + bool is_parameter = ov::is_type(op); + if (is_shape || is_parameter) { + const auto& output = is_shape ? op->input_value(0) : op->output(0); if (output.get_partial_shape().rank().is_dynamic()) return; for (const auto& d : output.get_partial_shape()) { - if (d.is_static()) - continue; - auto symbol = d.get_symbol(); - if (symbol == nullptr) + if (d.is_static() || d.get_symbol() == nullptr) continue; + auto symbol = ov::symbol::ancestor_of(d.get_symbol()); if (symbol_shape_source.count(symbol)) continue; symbol_shape_source[symbol] = output; @@ -344,11 +372,9 @@ void save_shape_sources(const std::shared_ptr& op, STS_map& symbol_sha if (input.get_partial_shape().rank().is_dynamic()) continue; const auto dimension = input.get_partial_shape()[axis]; - if (dimension.is_static()) - continue; - auto symbol = dimension.get_symbol(); - if (symbol == nullptr) + if (dimension.is_static() || dimension.get_symbol() == nullptr) continue; + auto symbol = ov::symbol::ancestor_of(dimension.get_symbol()); if (symbol_shape_source.count(symbol)) continue; symbol_shape_source[symbol] = input; @@ -402,27 +428,74 @@ struct OutputValue { } }; -void save_and_update_value_sources(const std::shared_ptr& op, - std::map>& multi_symbol_source) { - for (auto& output : op->outputs()) { - if (output.get_tensor().get_value_symbol().size() < 2) - continue; // singular values are handled by optimize_value_usage helper - - if (auto result = OutputValue::make(output)) { - if (multi_symbol_source.count(*result)) { - auto alternative_source = multi_symbol_source[*result]; - if (output.get_element_type() != alternative_source.get_element_type()) { - auto convert = std::make_shared(alternative_source, output.get_element_type()); - ov::copy_runtime_info(output.get_node_shared_ptr(), convert); - alternative_source = convert->output(0); - } - if (output.get_partial_shape().is_dynamic() || - output.get_partial_shape() != alternative_source.get_partial_shape()) - continue; - output.replace(alternative_source); +void optimize_multi_value_usage(ov::Output& output, + std::map>& multi_symbol_source, + STS_map& symbol_shape_source, + STS_map& symbol_value_source) { + if (output.get_tensor().get_value_symbol().size() < 2) + return; // singular values are handled by optimize_single_value_usage helper + auto result = OutputValue::make(output); + if (!result) + return; + if (multi_symbol_source.count(*result)) { + // multiple value source have been seen before + auto alternative_source = multi_symbol_source[*result]; + if (output.get_element_type() != alternative_source.get_element_type()) { + auto convert = std::make_shared(alternative_source, output.get_element_type()); + // should this be a convert like? no, output should be erased ideally. or should we just avoid dynamic data type? + ov::copy_runtime_info(output.get_node_shared_ptr(), convert); + alternative_source = convert->output(0); + } + if (output.get_partial_shape() != alternative_source.get_partial_shape()) { + const auto& shape = ov::op::v0::Constant::create(ov::element::i32, + ov::Shape{output.get_shape().size()}, + output.get_shape()); + alternative_source = std::make_shared(alternative_source, shape, false)->output(0); + } + output.replace(alternative_source); + } else { + // new instance of multiple value source + ov::OutputVector to_be_concated; + for (const auto& el : result->value) { + if (el.is()) { + const auto& value = el.as(); + const auto& constant = ov::op::v0::Constant::create(output.get_element_type(), ov::Shape{1}, {value}); + to_be_concated.push_back(constant->output(0)); + } else if (el.is>()) { + const auto& symbol = el.as>(); + auto alternative_output = + get_alternative_source_from_value_or_shape_source(symbol_shape_source, + symbol, + ov::Shape{1}, + output.get_element_type(), + output.get_node_shared_ptr(), + symbol_value_source); + if (alternative_output.get_node_shared_ptr()) + to_be_concated.push_back(alternative_output); + else + break; } else { - multi_symbol_source[*result] = output; + break; + } + } + if (to_be_concated.size() != ov::shape_size(output.get_shape())) { + multi_symbol_source[*result] = output; + } else { + auto alternative_output = std::make_shared(to_be_concated, 0)->output(0); + ov::copy_runtime_info(output.get_node_shared_ptr(), alternative_output.get_node_shared_ptr()); + if (output.get_partial_shape() != alternative_output.get_partial_shape()) { + alternative_output = std::make_shared( + alternative_output, + ov::op::v0::Constant::create(ov::element::i32, + ov::Shape{output.get_shape().size()}, + output.get_shape()), + false) + ->output(0); + ov::copy_runtime_info(output.get_node_shared_ptr(), alternative_output.get_node_shared_ptr()); } + ov::util::evaluate_both_bounds(alternative_output); + output.replace(alternative_output); + multi_symbol_source[*result] = alternative_output; } } } @@ -434,6 +507,8 @@ bool ov::pass::OptimizeSymbolsUsedAsValues::run_on_model(const std::shared_ptr> multi_symbol_source; + for (const auto& parameter : m->get_parameters()) + save_shape_sources(parameter, symbol_shape_source); for (const auto& op : topological_order(m)) { // Result has output port which has shared (during validate_and_infer_type) tensor with input port. // Transformations may replace input of Result. After replacement and before Result::validate_and_infer_type -- @@ -443,13 +518,14 @@ bool ov::pass::OptimizeSymbolsUsedAsValues::run_on_model(const std::shared_ptr(op)) continue; - // LTS maps aren't shared with sub-graphs because inner graph can not access outer graph for label sources + // LTS maps aren't shared with sub-graphs because inner graph can not access outer graph for symbol sources ov::op::util::process_subgraph(*this, op); - for (auto& output : op->outputs()) - optimize_value_usage(output, symbol_shape_source, symbol_value_source); + for (auto& output : op->outputs()) { + optimize_single_value_usage(output, symbol_shape_source, symbol_value_source); + optimize_multi_value_usage(output, multi_symbol_source, symbol_shape_source, symbol_value_source); + } save_shape_sources(op, symbol_shape_source); - save_and_update_value_sources(op, multi_symbol_source); } return true; } diff --git a/src/common/transformations/tests/symbolic_transformations/symbol_optimization.cpp b/src/common/transformations/tests/symbolic_transformations/symbol_optimization.cpp index e4653ec084bafb..4da19a3c896ca6 100644 --- a/src/common/transformations/tests/symbolic_transformations/symbol_optimization.cpp +++ b/src/common/transformations/tests/symbolic_transformations/symbol_optimization.cpp @@ -19,7 +19,6 @@ #include "openvino/pass/visualize_tree.hpp" #include "transformations/common_optimizations/shared_ops_optimization.hpp" #include "transformations/symbolic_transformations/symbolic_optimizations.hpp" -#include "transformations/symbolic_transformations/utils.hpp" using namespace ov; using namespace ov::op; @@ -174,7 +173,7 @@ TEST_F(TransformationTestsF, ValueOptimizationDoubleValue) { auto input = make_shared(element::f32, PartialShape::dynamic(4)); auto dim_0 = get_dim_by_idx(input, {-1, -2}, element::i64); - auto dim_1 = get_dim_by_idx(input, {3, 2}, element::i32); + auto dim_1 = get_dim_by_idx(input, {3, 2}, element::i64); auto reshape_0 = make_shared( input, @@ -182,28 +181,25 @@ TEST_F(TransformationTestsF, ValueOptimizationDoubleValue) { false); auto reshape_1 = make_shared( input, - make_shared(OutputVector{v0::Constant::create(element::i32, {1}, {0}), dim_1}, 0), + make_shared(OutputVector{v0::Constant::create(element::i64, {1}, {0}), dim_1}, 0), false); model = make_shared(NodeVector{reshape_0, reshape_1}, ParameterVector{input}); manager.set_per_pass_validation(false); - manager.register_pass(); - manager.register_pass(); - manager.register_pass(); + manager.register_pass(); } { auto input = make_shared(element::f32, PartialShape::dynamic(4)); - auto dim_0 = get_dim_by_idx(input, {3, 2}, element::i32); - auto dim_1 = std::make_shared(dim_0, element::i64); + auto dim_0 = get_dim_by_idx(input, {3, 2}, element::i64); auto reshape_0 = make_shared( input, - make_shared(OutputVector{v0::Constant::create(element::i64, {1}, {-1}), dim_1}, 0), + make_shared(OutputVector{v0::Constant::create(element::i64, {1}, {-1}), dim_0}, 0), false); auto reshape_1 = make_shared( input, - make_shared(OutputVector{v0::Constant::create(element::i32, {1}, {0}), dim_0}, 0), + make_shared(OutputVector{v0::Constant::create(element::i64, {1}, {0}), dim_0}, 0), false); model_ref = make_shared(NodeVector{reshape_0, reshape_1}, ParameterVector{input}); @@ -216,7 +212,7 @@ TEST_F(TransformationTestsF, ValueOptimizationSymbolAndValue) { auto input = make_shared(element::f32, PartialShape({-1, -1, 4, -1})); auto dim_0 = get_dim_by_idx(input, {-1, -2}, element::i64); - auto dim_1 = get_dim_by_idx(input, {3, 2}, element::i32); + auto dim_1 = get_dim_by_idx(input, {3, 2}, element::i64); auto reshape_0 = make_shared( input, @@ -224,7 +220,7 @@ TEST_F(TransformationTestsF, ValueOptimizationSymbolAndValue) { false); auto reshape_1 = make_shared( input, - make_shared(OutputVector{v0::Constant::create(element::i32, {1}, {-1}), dim_1}, 0), + make_shared(OutputVector{v0::Constant::create(element::i64, {1}, {-1}), dim_1}, 0), false); model = make_shared(NodeVector{reshape_0, reshape_1}, ParameterVector{input}); @@ -237,11 +233,10 @@ TEST_F(TransformationTestsF, ValueOptimizationSymbolAndValue) { { auto input = make_shared(element::f32, PartialShape({-1, -1, 4, -1})); auto dim_0 = make_shared( - OutputVector{v0::Constant::create(element::i32, {1}, {-1}), get_dim_by_idx(input, {3, 2}, element::i32)}, + OutputVector{v0::Constant::create(element::i64, {1}, {-1}), get_dim_by_idx(input, {3}, element::i64), v0::Constant::create(element::i64, {1}, {4})}, 0); - auto dim_1 = std::make_shared(dim_0, element::i64); - auto reshape_0 = make_shared(input, dim_1, false); + auto reshape_0 = make_shared(input, dim_0, false); auto reshape_1 = make_shared(input, dim_0, false); model_ref = make_shared(NodeVector{reshape_0, reshape_1}, ParameterVector{input}); From 6110c677b52ed135a1d00f14a0bf6d445788fa85 Mon Sep 17 00:00:00 2001 From: Evgeniia Nugmanova Date: Thu, 14 Nov 2024 14:13:46 +0400 Subject: [PATCH 2/2] style Signed-off-by: Evgeniia Nugmanova --- .../symbolic_transformations/symbol_optimization.cpp | 3 ++- .../tests/symbolic_transformations/symbol_optimization.cpp | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/common/transformations/src/transformations/symbolic_transformations/symbol_optimization.cpp b/src/common/transformations/src/transformations/symbolic_transformations/symbol_optimization.cpp index 07583ae486ae74..71463bc56cbd54 100644 --- a/src/common/transformations/src/transformations/symbolic_transformations/symbol_optimization.cpp +++ b/src/common/transformations/src/transformations/symbolic_transformations/symbol_optimization.cpp @@ -442,7 +442,8 @@ void optimize_multi_value_usage(ov::Output& output, auto alternative_source = multi_symbol_source[*result]; if (output.get_element_type() != alternative_source.get_element_type()) { auto convert = std::make_shared(alternative_source, output.get_element_type()); - // should this be a convert like? no, output should be erased ideally. or should we just avoid dynamic data type? + // should this be a convert like? no, output should be erased ideally. or should we just avoid dynamic data + // type? ov::copy_runtime_info(output.get_node_shared_ptr(), convert); alternative_source = convert->output(0); } diff --git a/src/common/transformations/tests/symbolic_transformations/symbol_optimization.cpp b/src/common/transformations/tests/symbolic_transformations/symbol_optimization.cpp index 4da19a3c896ca6..2070a2bce7d349 100644 --- a/src/common/transformations/tests/symbolic_transformations/symbol_optimization.cpp +++ b/src/common/transformations/tests/symbolic_transformations/symbol_optimization.cpp @@ -232,9 +232,10 @@ TEST_F(TransformationTestsF, ValueOptimizationSymbolAndValue) { } { auto input = make_shared(element::f32, PartialShape({-1, -1, 4, -1})); - auto dim_0 = make_shared( - OutputVector{v0::Constant::create(element::i64, {1}, {-1}), get_dim_by_idx(input, {3}, element::i64), v0::Constant::create(element::i64, {1}, {4})}, - 0); + auto dim_0 = make_shared(OutputVector{v0::Constant::create(element::i64, {1}, {-1}), + get_dim_by_idx(input, {3}, element::i64), + v0::Constant::create(element::i64, {1}, {4})}, + 0); auto reshape_0 = make_shared(input, dim_0, false); auto reshape_1 = make_shared(input, dim_0, false);