Add closure segmenter option to populate the table keyed segments in its output.

One table keyed segment is added per merge group.

Author: garretrieger

ift/encoder/closure_glyph_segmenter.cc

@@ -193,11 +193,10 @@ struct SegmentOrdering {
 };
 
 static std::vector<Segment> PreGroupSegments(
-  const btree_map<SegmentSet, MergeStrategy>& merge_groups,
-  const std::vector<SegmentOrdering>& ordering,
-  const std::vector<SubsetDefinition>& subset_definitions,
-  std::vector<uint32_t>& segment_index_map
-) {
+    const btree_map<SegmentSet, MergeStrategy>& merge_groups,
+    const std::vector<SegmentOrdering>& ordering,
+    const std::vector<SubsetDefinition>& subset_definitions,
+    std::vector<uint32_t>& segment_index_map) {
   segment_index_map.resize(subset_definitions.size());
   std::vector<Segment> segments;
 
@@ -206,9 +205,10 @@ static std::vector<Segment> PreGroupSegments(
   auto merge_group_it = merge_groups.begin();
   auto ordering_it = ordering.begin();
 
-  while (ordering_it != ordering.end())  {
+  while (ordering_it != ordering.end()) {
     const auto& o = *ordering_it;
-    if (o.group_index != last_group_index && merge_group_it != merge_groups.end()) {
+    if (o.group_index != last_group_index &&
+        merge_group_it != merge_groups.end()) {
       merge_group_it++;
     }
 
@@ -217,11 +217,11 @@ static std::vector<Segment> PreGroupSegments(
       strategy = &(merge_group_it->second);
     }
 
-    Segment segment = Segment{subset_definitions[o.original_index], o.probability};
+    Segment segment =
+        Segment{subset_definitions[o.original_index], o.probability};
     ordering_it++;
 
-    if (strategy == nullptr ||
-        strategy->PreClosureGroupSize() <= 1 ||
+    if (strategy == nullptr || strategy->PreClosureGroupSize() <= 1 ||
         o.probability.Max() > strategy->PreClosureProbabilityThreshold()) {
       segment_index_map[o.original_index] = i;
     } else {
@@ -232,15 +232,18 @@ static std::vector<Segment> PreGroupSegments(
           break;
         }
 
-        segment.Definition().Union(subset_definitions[ordering_it->original_index]);
+        segment.Definition().Union(
+            subset_definitions[ordering_it->original_index]);
         segment_index_map[ordering_it->original_index] = i;
 
         ordering_it++;
         remaining--;
       }
 
       if (strategy->UseCosts()) {
-        segment.SetProbability(strategy->ProbabilityCalculator()->ComputeProbability(segment.Definition()));
+        segment.SetProbability(
+            strategy->ProbabilityCalculator()->ComputeProbability(
+                segment.Definition()));
       }
     }
 
@@ -328,7 +331,8 @@ static StatusOr<std::vector<Segment>> ToOrderedSegments(
 
   // maps from index in subset_definitions to the new ordering.
   std::vector<uint32_t> segment_index_map;
-  std::vector<Segment> segments = PreGroupSegments(merge_groups, ordering, subset_definitions, segment_index_map);
+  std::vector<Segment> segments = PreGroupSegments(
+      merge_groups, ordering, subset_definitions, segment_index_map);
   VLOG(0) << segments.size() << " segments after pregrouping.";
 
   btree_map<SegmentSet, MergeStrategy> new_merge_groups;
@@ -344,8 +348,8 @@ static StatusOr<std::vector<Segment>> ToOrderedSegments(
       remapped_full.insert(s_prime);
     }
 
-
-    VLOG(0) << "  Merge group " << group_index << " has " << remapped.size() << " segments.";
+    VLOG(0) << "  Merge group " << group_index << " has " << remapped.size()
+            << " segments.";
     group_index++;
 
     if (!new_merge_groups.insert(std::make_pair(remapped, std::move(strategy)))

ift/encoder/closure_glyph_segmenter_test.cc

@@ -1427,19 +1427,11 @@ if (s0 AND s2) then p2
 )");
 }
 
-
 TEST_F(ClosureGlyphSegmenterTest, MultipleMergeGroups_PreGrouping) {
   UnicodeFrequencies freq{
-    {{' ', ' '}, 100},
-    {{'d', 'd'}, 100},
-    {{'a', 'a'}, 60},
-    {{'e', 'e'}, 30},
-    {{'b', 'b'}, 29},
-    {{'f', 'f'}, 28},
-    {{'c', 'c'}, 10},
-    {{'g', 'g'}, 9},
-    {{'h', 'h'}, 5},
-    {{'i', 'i'}, 1}, // 8
+      {{' ', ' '}, 100}, {{'d', 'd'}, 100}, {{'a', 'a'}, 60}, {{'e', 'e'}, 30},
+      {{'b', 'b'}, 29},  {{'f', 'f'}, 28},  {{'c', 'c'}, 10}, {{'g', 'g'}, 9},
+      {{'h', 'h'}, 5},   {{'i', 'i'}, 1},  // 8
   };
 
   MergeStrategy costs = *MergeStrategy::CostBased(std::move(freq), 0, 1);
@@ -1473,8 +1465,8 @@ TEST_F(ClosureGlyphSegmenterTest, MultipleMergeGroups_PreGrouping) {
       // Group 1
       {'d'},
       {'a'},
-      {'e', 'b', 'f'}, // pre merge
-      {'c', 'g'},      // pre merge
+      {'e', 'b', 'f'},  // pre merge
+      {'c', 'g'},       // pre merge
       // Shared
       {'h'},
       {'i'},

ift/encoder/estimated_patch_size_cache.cc

@@ -3,8 +3,8 @@
 #include "common/int_set.h"
 #include "common/try.h"
 
-using absl::StatusOr;
 using absl::flat_hash_set;
+using absl::StatusOr;
 using common::GlyphSet;
 
 namespace ift::encoder {

ift/encoder/estimated_patch_size_cache.h

@@ -2,6 +2,7 @@
 #define IFT_ENCODER_ESTIMATED_PATCH_SIZE_CACHE_H_
 
 #include <memory>
+
 #include "absl/status/statusor.h"
 #include "common/font_data.h"
 #include "common/int_set.h"
@@ -18,14 +19,13 @@ class EstimatedPatchSizeCache : public PatchSizeCache {
  public:
   static absl::StatusOr<std::unique_ptr<PatchSizeCache>> New(hb_face_t* face) {
     double compression_ratio = TRY(EstimateCompressionRatio(face));
-    return std::unique_ptr<PatchSizeCache>(new EstimatedPatchSizeCache(face, compression_ratio));
+    return std::unique_ptr<PatchSizeCache>(
+        new EstimatedPatchSizeCache(face, compression_ratio));
   }
 
   absl::StatusOr<uint32_t> GetPatchSize(const common::GlyphSet& gids) override;
 
-  double CompressionRatio() const {
-    return compression_ratio_;
-  }
+  double CompressionRatio() const { return compression_ratio_; }
 
  private:
   explicit EstimatedPatchSizeCache(hb_face_t* original_face,

ift/encoder/estimated_patch_size_cache_test.cc

@@ -1,17 +1,16 @@
 #include "ift/encoder/estimated_patch_size_cache.h"
 
+#include "common/font_data.h"
 #include "common/font_helper.h"
 #include "common/int_set.h"
 #include "gtest/gtest.h"
 
-#include "common/font_data.h"
-
-using common::hb_face_unique_ptr;
-using common::make_hb_face;
-using common::hb_blob_unique_ptr;
-using common::make_hb_blob;
 using common::FontHelper;
 using common::GlyphSet;
+using common::hb_blob_unique_ptr;
+using common::hb_face_unique_ptr;
+using common::make_hb_blob;
+using common::make_hb_face;
 
 namespace ift::encoder {
 
@@ -23,24 +22,24 @@ class EstimatedPatchSizeCacheTest : public ::testing::Test {
     roboto = make_hb_face(hb_face_create(blob.get(), 0));
   }
 
-
   double CompressionRatio(GlyphSet gids, double expected_compression_ratio) {
-    uint32_t raw_outline_size =
-      *FontHelper::TotalGlyphData(roboto.get(), gids);
-    double fixed_size = 1 + 7 * 4; // header
-    fixed_size += (double) (5 + gids.size() * 2 + 4 + (gids.size() + 1)*4) * expected_compression_ratio; // glyph patches header
+    uint32_t raw_outline_size = *FontHelper::TotalGlyphData(roboto.get(), gids);
+    double fixed_size = 1 + 7 * 4;  // header
+    fixed_size += (double)(5 + gids.size() * 2 + 4 + (gids.size() + 1) * 4) *
+                  expected_compression_ratio;  // glyph patches header
     auto estimated = *EstimatedPatchSizeCache::New(roboto.get());
     uint32_t compressed_size = *estimated->GetPatchSize(gids);
-    return (double) (compressed_size - fixed_size) / (double) raw_outline_size;
+    return (double)(compressed_size - fixed_size) / (double)raw_outline_size;
   }
 
   hb_face_unique_ptr roboto;
 };
 
 TEST_F(EstimatedPatchSizeCacheTest, PatchSize) {
   // There should be a consistent compression ratio between patches.
-  ASSERT_NEAR(this->CompressionRatio(GlyphSet {44, 47, 49}, 0.457), 0.46, 0.01);
-  ASSERT_NEAR(CompressionRatio(GlyphSet {45, 48, 50, 51, 52, 53}, 0.457), 0.46, 0.01);
+  ASSERT_NEAR(this->CompressionRatio(GlyphSet{44, 47, 49}, 0.457), 0.46, 0.01);
+  ASSERT_NEAR(CompressionRatio(GlyphSet{45, 48, 50, 51, 52, 53}, 0.457), 0.46,
+              0.01);
 }
 
 }  // namespace ift::encoder

ift/encoder/glyph_segmentation.cc

@@ -2,13 +2,11 @@
 
 #include <cstdint>
 #include <cstdio>
-#include <optional>
 #include <sstream>
 
 #include "absl/container/btree_map.h"
 #include "absl/container/btree_set.h"
 #include "absl/container/flat_hash_map.h"
-#include "absl/container/flat_hash_set.h"
 #include "absl/status/statusor.h"
 #include "common/font_helper.h"
 #include "common/int_set.h"
@@ -144,17 +142,22 @@ ProtoType TagsToSetProto(const btree_set<hb_tag_t>& set) {
   return values;
 }
 
+void GlyphSegmentation::SubsetDefinitionToSegment(const SubsetDefinition& def,
+                                                  SegmentProto& segment_proto) {
+  (*segment_proto.mutable_codepoints()) =
+      ToSetProto<Codepoints>(def.codepoints);
+  (*segment_proto.mutable_features()) =
+      TagsToSetProto<Features>(def.feature_tags);
+}
+
 SegmentationPlan GlyphSegmentation::ToSegmentationPlanProto() const {
   SegmentationPlan config;
 
   uint32_t set_index = 0;
   for (const auto& s : Segments()) {
     if (!s.Empty()) {
       SegmentProto segment_proto;
-      (*segment_proto.mutable_codepoints()) =
-          ToSetProto<Codepoints>(s.codepoints);
-      (*segment_proto.mutable_features()) =
-          TagsToSetProto<Features>(s.feature_tags);
+      SubsetDefinitionToSegment(s, segment_proto);
       (*config.mutable_segments())[set_index++] = segment_proto;
     } else {
       set_index++;

ift/encoder/glyph_segmentation.h

@@ -89,6 +89,9 @@ class GlyphSegmentation {
     return init_font_segment_;
   };
 
+  static void SubsetDefinitionToSegment(const SubsetDefinition& def,
+                                        SegmentProto& segment_proto);
+
   SegmentationPlan ToSegmentationPlanProto() const;
 
   static absl::Status GroupsToSegmentation(

ift/encoder/merge_strategy.h

@@ -145,9 +145,7 @@ class MergeStrategy {
     return init_font_merge_probability_threshold_;
   }
 
-  uint32_t PreClosureGroupSize() const {
-    return pre_closure_group_size_;
-  }
+  uint32_t PreClosureGroupSize() const { return pre_closure_group_size_; }
 
   double PreClosureProbabilityThreshold() const {
     return pre_closure_probability_threshold_;

util/closure_glyph_keyed_segmenter_util.cc

@@ -127,6 +127,39 @@ static Status Analysis(hb_face_t* font,
   return absl::OkStatus();
 }
 
+static void AddTableKeyedSegments(
+    SegmentationPlan& plan,
+    const btree_map<SegmentSet, MergeStrategy>& merge_groups,
+    const std::vector<SubsetDefinition>& segments,
+    const SubsetDefinition& init_segment) {
+  std::vector<SubsetDefinition> table_keyed_segments;
+  for (const auto& [segment_ids, _] : merge_groups) {
+    SubsetDefinition new_segment;
+    for (uint32_t s : segment_ids) {
+      new_segment.Union(segments.at(s));
+    }
+    new_segment.Subtract(init_segment);
+    table_keyed_segments.push_back(new_segment);
+  }
+
+  uint32_t max_id = 0;
+  for (const auto& [id, _] : plan.segments()) {
+    if (id > max_id) {
+      max_id = id;
+    }
+  }
+
+  uint32_t next_id = max_id + 1;
+  auto* plan_segments = plan.mutable_segments();
+  for (const SubsetDefinition& def : table_keyed_segments) {
+    GlyphSegmentation::SubsetDefinitionToSegment(def,
+                                                 (*plan_segments)[next_id]);
+    SegmentsProto* segment_ids = plan.add_non_glyph_segments();
+    segment_ids->add_values(next_id);
+    next_id++;
+  }
+}
+
 static Status Main(const std::vector<char*> args) {
   hb_face_unique_ptr font =
       TRY(LoadFont(absl::GetFlag(FLAGS_input_font).c_str()));
@@ -137,12 +170,14 @@ static Status Main(const std::vector<char*> args) {
   CodepointSet font_codepoints = FontHelper::ToCodepointsSet(font.get());
   SubsetDefinition init_segment =
       config_util.SegmentProtoToSubsetDefinition(config.initial_segment());
+
   std::vector<SubsetDefinition> segments;
   btree_map<SegmentSet, MergeStrategy> merge_groups =
       TRY(config_util.ConfigToMergeGroups(config, font_codepoints, segments));
 
-  ClosureGlyphSegmenter segmenter(config.brotli_quality(),
-                                  config.brotli_quality_for_initial_font_merging());
+  ClosureGlyphSegmenter segmenter(
+      config.brotli_quality(),
+      config.brotli_quality_for_initial_font_merging());
   GlyphSegmentation segmentation = TRY(segmenter.CodepointToGlyphSegments(
       font.get(), init_segment, segments, merge_groups,
       config.move_fallback_glyphs_into_initial_font()));
@@ -154,6 +189,10 @@ static Status Main(const std::vector<char*> args) {
       plan.clear_initial_codepoints();
     }
 
+    if (config.generate_table_keyed_segments()) {
+      AddTableKeyedSegments(plan, merge_groups, segments, init_segment);
+    }
+
     // TODO(garretrieger): assign a basic (single segment) table keyed config.
     // Later on the input to this util should include information on how the
     // segments should be grouped together for the table keyed portion of the