Support recursive normalizer in v2 API write (#2736)

#### Reference Issues/PRs
<!--Example: Fixes #1234. See also #3456.-->
https://man312219.monday.com/boards/7852509418/pulses/18298965201
#### What does this implement or fix?
1. Makes `def write` and `def write_pickle` in v2 API support recursive
normalizer
- Recursive normalizer enables ArcricDB to write and read nest data
structures (`dict`, `list`, `tuple`) of dataframes and arrays without
having to pickling the entire structure
- The data stored is not filterable, as is pickled data
- Please refer to
https://docs.arcticdb.io/latest/notebooks/arcticdb_demo_recursive_normalizers
for more details
3. Minor changes on formatter to support formatting individual files

Per discussed, in v2 API, recursive normalizer setting in LibraryOption
will be respected. Default library option of recursive normalizer will
be `False`. Existing libraries are unaffected.
#### Any other comments?
`def batch_write` in v1 API doesn't support recursive normalizer.
Ticket: https://man312219.monday.com/boards/7852509418/pulses/7855436309
Therefore the support of recursive normalizer in corresponding v2 API
will not be covered in this PR.

##### Pickling
`pickling` is a bit of a mess in V1 API. 
For `arrow` and `pandas` data, if the normalization fails, it can
fallback to msgpack and pickling, depending on whether
`pickle_on_failure` is `True`.
However, for other kinds of data, it almost certainly fallback to
msgpack and pickling. The only option to prevent pickling is a library
config `strict_mode`. But I don't see there is any API to enable this
option, in V1/V2 nor internally.

#### Checklist

<details>
  <summary>
   Checklist for code changes...
  </summary>
 
- [ ] Have you updated the relevant docstrings, documentation and
copyright notice?
- [ ] Is this contribution tested against [all ArcticDB's
features](../docs/mkdocs/docs/technical/contributing.md)?
- [ ] Do all exceptions introduced raise appropriate [error
messages](https://docs.arcticdb.io/error_messages/)?
 - [ ] Are API changes highlighted in the PR description?
- [ ] Is the PR labelled as enhancement or bug so it appears in
autogenerated release notes?
</details>

<!--
Thanks for contributing a Pull Request to ArcticDB! Please ensure you
have taken a look at:
- ArcticDB's Code of Conduct:
https://github.com/man-group/ArcticDB/blob/master/CODE_OF_CONDUCT.md
- ArcticDB's Contribution Licensing:
https://github.com/man-group/ArcticDB/blob/master/docs/mkdocs/docs/technical/contributing.md#contribution-licensing
-->

Author: phoebusm

build_tooling/format.py

@@ -31,30 +31,38 @@ def install_tools():
     return black or clang
 
 
-def lint_python(in_place: bool):
+def lint_python(in_place: bool, specific_file: str = None):
     try:
         import black
         assert black.__version__ == black_version
     except ImportError:
         raise RuntimeError("black not installed. Run this script with --install-tools then try again")
 
+    if specific_file:
+        path = specific_file
+    else:
+        path = "python/"
+
     if in_place:
-        return subprocess.run(["black", "-l", "120", "python/"]).returncode
+        return subprocess.run(["black", "-l", "120", path]).returncode
     else:
-        return subprocess.run(["black", "-l", "120", "--check", "python/"]).returncode
+        return subprocess.run(["black", "-l", "120", "--check", path]).returncode
 
 
-def lint_cpp(in_place: bool):
+def lint_cpp(in_place: bool, specific_file: str = None):
     try:
         import clang_format
     except ImportError:
         raise RuntimeError("clang-format not installed. Run this script with --install-tools then try again")
 
     files = []
-    root = pathlib.Path("cpp", "arcticdb")
-    for e in ("*.cpp", "*.hpp"):
-        for f in root.rglob(e):
-            files.append(str(f))
+    if specific_file:
+        files.append(specific_file)
+    else:
+        root = pathlib.Path("cpp", "arcticdb")
+        for e in ("*.cpp", "*.hpp"):
+            for f in root.rglob(e):
+                files.append(str(f))
 
     args = ["clang-format"]
     if in_place:
@@ -69,11 +77,11 @@ def lint_cpp(in_place: bool):
     return subprocess.run(args).returncode
 
 
-def main(type: str, in_place: bool):
+def main(type: str, in_place: bool, specific_file: str):
     if type == "python":
-        return lint_python(in_place)
+        return lint_python(in_place, specific_file)
     elif type == "cpp":
-        return lint_cpp(in_place)
+        return lint_cpp(in_place, specific_file)
     else:
         return lint_python(in_place) or lint_cpp(in_place)
 
@@ -105,6 +113,11 @@ def main(type: str, in_place: bool):
         action='store_true',
         help="Apply linting rules to your working copy. Changes files."
     )
+    parser.add_argument(
+        "-f", 
+        "--file", 
+        help="Apply linting rules to a specific file."
+    )
     args = parser.parse_args()
 
     if args.install_tools:
@@ -122,6 +135,7 @@ def main(type: str, in_place: bool):
     return_code = main(
         type=args.type,
         in_place=args.in_place,
+        specific_file=args.file,
     )
 
     sys.exit(return_code)

cpp/arcticdb/storage/library_manager.cpp

@@ -155,6 +155,9 @@ void LibraryManager::modify_library_option(
                 case ModifiableLibraryOption::COLUMNS_PER_SEGMENT:
                     mutable_write_options->set_column_group_size(get_positive_int(new_value));
                     break;
+                case ModifiableLibraryOption::RECURSIVE_NORMALIZERS:
+                    mutable_write_options->set_recursive_normalizers(get_bool(new_value));
+                    break;
                 default:
                     throw UnsupportedLibraryOptionValue(fmt::format("Invalid library option: {}", option));
                 }

cpp/arcticdb/storage/library_manager.hpp

@@ -16,7 +16,12 @@
 #include <arcticdb/entity/protobufs.hpp>
 
 namespace arcticdb::storage {
-enum class ModifiableLibraryOption { DEDUP = 1, ROWS_PER_SEGMENT = 2, COLUMNS_PER_SEGMENT = 3 };
+enum class ModifiableLibraryOption {
+    DEDUP = 1,
+    ROWS_PER_SEGMENT = 2,
+    COLUMNS_PER_SEGMENT = 3,
+    RECURSIVE_NORMALIZERS = 4
+};
 enum class ModifiableEnterpriseLibraryOption { REPLICATION = 1, BACKGROUND_DELETION = 2 };
 using LibraryOptionValue = std::variant<bool, int64_t>;
 
@@ -101,6 +106,8 @@ struct formatter<arcticdb::storage::ModifiableLibraryOption> {
             return fmt::format_to(ctx.out(), "ROWS_PER_SEGMENT");
         case arcticdb::storage::ModifiableLibraryOption::COLUMNS_PER_SEGMENT:
             return fmt::format_to(ctx.out(), "COLUMNS_PER_SEGMENT");
+        case arcticdb::storage::ModifiableLibraryOption::RECURSIVE_NORMALIZERS:
+            return fmt::format_to(ctx.out(), "RECURSIVE_NORMALIZERS");
         default:
             arcticdb::util::raise_rte("Unrecognized modifiable option {}", int(o));
         }

cpp/arcticdb/storage/python_bindings.cpp

@@ -148,7 +148,8 @@ void register_bindings(py::module& storage, py::exception<arcticdb::ArcticExcept
      )pbdoc")
             .value("DEDUP", ModifiableLibraryOption::DEDUP)
             .value("ROWS_PER_SEGMENT", ModifiableLibraryOption::ROWS_PER_SEGMENT)
-            .value("COLUMNS_PER_SEGMENT", ModifiableLibraryOption::COLUMNS_PER_SEGMENT);
+            .value("COLUMNS_PER_SEGMENT", ModifiableLibraryOption::COLUMNS_PER_SEGMENT)
+            .value("RECURSIVE_NORMALIZERS", ModifiableLibraryOption::RECURSIVE_NORMALIZERS);
 
     py::enum_<ModifiableEnterpriseLibraryOption>(storage, "ModifiableEnterpriseLibraryOption", R"pbdoc(
          Enterprise library options that can be modified after library creation.