Merge pull request #271 from abs-tudelft/arrow-3.0

Migrate Arrow to 3.0.0

Author: johanpel

.github/workflows/assets.yml

@@ -8,7 +8,7 @@ on:
 
 env:
   CMAKE_VERSION: '3.19.2'
-  ARROW_VERSION: '1.0.1'
+  ARROW_VERSION: '3.0.0'
 
 jobs:
   archive:
@@ -172,6 +172,7 @@ jobs:
         - 'cp36-cp36m'
         - 'cp37-cp37m'
         - 'cp38-cp38'
+        - 'cp39-cp39'
     steps:
     - uses: actions/checkout@v2
       with:

.github/workflows/documentation.yml

@@ -18,7 +18,7 @@ jobs:
     name: Book
     runs-on: ubuntu-latest
     env:
-      MDBOOK_VERSION: '0.4.1'
+      MDBOOK_VERSION: '0.4.6'
     steps:
     - uses: actions/checkout@v2
     - name: Install mdbook

.github/workflows/examples.yml

@@ -13,8 +13,8 @@ jobs:
     container: quay.io/pypa/manylinux2014_x86_64:latest
     env:
       CMAKE_VERSION: '3.17.3'
-      ARROW_VERSION: '1.0.1'
-      CPYTHON_VERSION: 'cp35-cp35m'
+      ARROW_VERSION: '3.0.0'
+      CPYTHON_VERSION: 'cp36-cp36m'
     steps:
     - uses: actions/checkout@v2
     - name: Install Apache Arrow
@@ -40,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: prepare
     env:
-      PYTHON_VERSION: '3.5.x'
+      PYTHON_VERSION: '3.6.x'
     strategy:
       fail-fast: false
       matrix:

.github/workflows/test.yml

@@ -7,7 +7,7 @@ on:
   pull_request:
 
 env:
-  ARROW_VERSION: '1.0.1'
+  ARROW_VERSION: '3.0.0'
 
 jobs:
   cpp:
@@ -60,16 +60,21 @@ jobs:
         sudo apt-get install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
         sudo apt-get update
         sudo apt-get install -y libarrow-dev=$ARROW_VERSION-1 libarrow-python-dev=$ARROW_VERSION-1
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
     - name: Install pyarrow
-      run: pip3 install pyarrow==$ARROW_VERSION setuptools wheel
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install pyarrow==$ARROW_VERSION
     - name: Build and install
       working-directory: ${{ matrix.source }}
       run: |
-        python3 setup.py build
-        python3 setup.py bdist_wheel
-        pip3 install build/dist/*.whl
+        python setup.py build
+        python setup.py bdist_wheel
+        python -m pip install build/dist/*.whl
     - name: Import
-      run: python3 -c "import ${{ matrix.package }}"
+      run: python -c "import ${{ matrix.package }}"
 
   vhdl:
     name: VHDL

README.md

@@ -50,8 +50,7 @@ generates the following:
   - Structs
   - Validity bitmaps
 
-- Once the Arrow reference implementation and format specific reaches ensured
-  stability (i.e. version 1.0), we would like to support:
+- In the future we would like to support:
   - Sparse and dense unions
   - Dictionaries
   - Chunked tabular structures (`Arrow::Table`)
@@ -128,15 +127,18 @@ External projects using Fletcher:
 - [Posit PairHMM](https://github.com/lvandam/pairhmm_posit_hdl_arrow)
 
 ## Publications
+
 If you used or studied Fletcher, please cite:
+
 - J. Peltenburg, J. van Straten, L. Wijtemans, L. van Leeuwen, Z. Al-Ars, and
   H.P. Hofstee, Fletcher: A Framework to Efficiently Integrate FPGA Accelerators
-  with Apache Arrow*, in 29th International Conference on Field Programmable 
+  with Apache Arrow\*, in 29th International Conference on Field Programmable
   Logic and Applications (FPL) (2019) pp. 270–277.
 
 Additional publications:
+
 - J. Peltenburg, J. van Straten, M. Brobbel, H.P. Hofstee, and Z. Al-Ars,
   Supporting Columnar In-memory Formats on FPGA: The Hardware Design of Fletcher
-  for Apache Arrow*, in Applied Reconfigurable Computing, edited by 
-  C. Hochberger, B. Nelson, A. Koch, R. Woods, and P. Diniz (Springer 
+  for Apache Arrow\*, in Applied Reconfigurable Computing, edited by
+  C. Hochberger, B. Nelson, A. Koch, R. Woods, and P. Diniz (Springer
   International Publishing, Cham, 2019) pp. 32–47

codegen/cpp/fletchgen/CMakeLists.txt

@@ -7,7 +7,7 @@ project(fletchgen
   LANGUAGES CXX
 )
 
-find_package(Arrow 1.0 CONFIG REQUIRED)
+find_package(Arrow 3.0.0 CONFIG REQUIRED)
 
 include(FindThreads)
 include(FetchContent)
@@ -197,7 +197,7 @@ set(CPACK_DEBIAN_PACKAGE_MAINTAINER "${CPACK_PACKAGE_VENDOR}")
 set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
 set(CPACK_RPM_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION}")
 set(CPACK_RPM_PACKAGE_LICENSE "ASL 2.0")
-set(CPACK_RPM_PACKAGE_REQUIRES "arrow-libs >= 1.0.1, arrow-libs < 2.0.0")
+set(CPACK_RPM_PACKAGE_REQUIRES "arrow-libs >= 3.0.0, arrow-libs < 4.0.0")
 
 set(CPACK_ARCHIVE_BINARY_FILE_NAME "${CMAKE_PROJECT_NAME}-${fletchgen_VERSION}-${CMAKE_SYSTEM_NAME}")
 

codegen/cpp/fletchgen/README.md

@@ -37,7 +37,7 @@ It currently supports only two top-level platforms.
 # Prerequisites
 
 - [C++17 compliant compiler](https://clang.llvm.org/)
-- [Apache Arrow 1.0+](https://github.com/apache/arrow)
+- [Apache Arrow 3.0+](https://github.com/apache/arrow)
 - [CMake 3.14+](https://cmake.org/)
 
 # Build & install
@@ -95,6 +95,7 @@ your kernel implementation.
 | fletcher_tag_width | 1 / 2 / 3 / ... | 1       | Width of the `tag` field of commands and unlock streams of RecordBatchReaders/Writers. Can be used to identify commands.              |
 
 # Custom MMIO registers
+
 You can add custom MMIO registers to your kernel using `--reg`.
 More information [can be found here](../../../docs/mmio.md).
 
@@ -114,32 +115,35 @@ bit.
 - record:
   name: platform
   fields:
-  - record:
-    name: complete
-    fields:
-    - field:
-      name: req
-      width: 1
-    - field:
-      name: ack
-      width: 1
-      reverse: true
+    - record:
+      name: complete
+      fields:
+        - field:
+          name: req
+          width: 1
+        - field:
+          name: ack
+          width: 1
+          reverse: true
 ```
 
 This will result in the following signals appearing at the top-level:
+
 ```vhdl
 ext_platform_complete_req : out std_logic;
 ext_platform_complete_ack : in  std_logic
 ```
 
-* The signals are assumed to be driven by the kernel. To drive them from the top
+- The signals are assumed to be driven by the kernel. To drive them from the top
   level, use:
+
 ```yaml
 reverse: true
 ```
 
-* Fields with a width of 1 can be forced to be `std_logic_vector` instead of 
+- Fields with a width of 1 can be forced to be `std_logic_vector` instead of
   `std_logic` by using:
+
 ```yaml
 vector: true
 ```

codegen/cpp/fletchgen/test/fletchgen/srec/test_srec.cc

@@ -38,16 +38,22 @@ namespace fletchgen::srec {
  */
 
 TEST(SREC, ToString) {
-  uint8_t data0[16] = {0x28, 0x5F, 0x24, 0x5F, 0x22, 0x12, 0x22, 0x6A, 0x00, 0x04, 0x24, 0x29, 0x00, 0x08, 0x23, 0x7C};
-  uint8_t data1[16] = {0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x26, 0x29, 0x00, 0x18, 0x53, 0x81, 0x23, 0x41, 0x00, 0x18};
-  uint8_t data2[16] = {0x41, 0xE9, 0x00, 0x08, 0x4E, 0x42, 0x23, 0x43, 0x00, 0x18, 0x23, 0x42, 0x00, 0x08, 0x24, 0xA9};
+  uint8_t data0[16] = {0x28, 0x5F, 0x24, 0x5F, 0x22, 0x12, 0x22, 0x6A,
+                       0x00, 0x04, 0x24, 0x29, 0x00, 0x08, 0x23, 0x7C};
+  uint8_t data1[16] = {0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x26, 0x29,
+                       0x00, 0x18, 0x53, 0x81, 0x23, 0x41, 0x00, 0x18};
+  uint8_t data2[16] = {0x41, 0xE9, 0x00, 0x08, 0x4E, 0x42, 0x23, 0x43,
+                       0x00, 0x18, 0x23, 0x42, 0x00, 0x08, 0x24, 0xA9};
   uint8_t data3[4] = {0x00, 0x14, 0x4E, 0xD4};
   // Check header record. Default header is similar to example.
   ASSERT_EQ(Record::Header().ToString(), "S00600004844521B");
   // Check other records
-  ASSERT_EQ(Record::Data<16>(0x00, data0, 16).ToString(), "S1130000285F245F2212226A000424290008237C2A");
-  ASSERT_EQ(Record::Data<16>(0x10, data1, 16).ToString(), "S11300100002000800082629001853812341001813");
-  ASSERT_EQ(Record::Data<16>(0x20, data2, 16).ToString(), "S113002041E900084E42234300182342000824A952");
+  ASSERT_EQ(Record::Data<16>(0x00, data0, 16).ToString(),
+            "S1130000285F245F2212226A000424290008237C2A");
+  ASSERT_EQ(Record::Data<16>(0x10, data1, 16).ToString(),
+            "S11300100002000800082629001853812341001813");
+  ASSERT_EQ(Record::Data<16>(0x20, data2, 16).ToString(),
+            "S113002041E900084E42234300182342000824A952");
   ASSERT_EQ(Record::Data<16>(0x30, data3, 4).ToString(), "S107003000144ED492");
 }
 
@@ -64,9 +70,12 @@ TEST(SREC, FromString) {
 
 TEST(SREC, File) {
   // Test a round trip via a file.
-  uint8_t data[52] = {0x28, 0x5F, 0x24, 0x5F, 0x22, 0x12, 0x22, 0x6A, 0x00, 0x04, 0x24, 0x29, 0x00, 0x08, 0x23, 0x7C,
-                      0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x26, 0x29, 0x00, 0x18, 0x53, 0x81, 0x23, 0x41, 0x00, 0x18,
-                      0x41, 0xE9, 0x00, 0x08, 0x4E, 0x42, 0x23, 0x43, 0x00, 0x18, 0x23, 0x42, 0x00, 0x08, 0x24, 0xA9,
+  uint8_t data[52] = {0x28, 0x5F, 0x24, 0x5F, 0x22, 0x12, 0x22, 0x6A,
+                      0x00, 0x04, 0x24, 0x29, 0x00, 0x08, 0x23, 0x7C,
+                      0x00, 0x02, 0x00, 0x08, 0x00, 0x08, 0x26, 0x29,
+                      0x00, 0x18, 0x53, 0x81, 0x23, 0x41, 0x00, 0x18,
+                      0x41, 0xE9, 0x00, 0x08, 0x4E, 0x42, 0x23, 0x43,
+                      0x00, 0x18, 0x23, 0x42, 0x00, 0x08, 0x24, 0xA9,
                       0x00, 0x14, 0x4E, 0xD4};
   // Check header record. Default header is similar to example.
   auto sro = File(0, data, 52);
@@ -75,7 +84,7 @@ TEST(SREC, File) {
   ofs.close();
   auto ifs = std::ifstream("srec_file_test.srec");
   auto sri = File(&ifs);
-  uint8_t* result;
+  uint8_t *result;
   size_t size;
   sri.ToBuffer(&result, &size);
   ASSERT_EQ(memcmp(data, result, 52), 0);
@@ -86,12 +95,14 @@ TEST(SREC, RecordBatchRoundTrip) {
   // Get a recordbatch with some integers
   auto rb = fletcher::GetStringRB();
   // Open an Arrow FileOutputStream
-  arrow::Result<std::shared_ptr<arrow::io::OutputStream>> result = arrow::io::FileOutputStream::Open("test.rbf");
+  arrow::Result<std::shared_ptr<arrow::io::OutputStream>>
+      result = arrow::io::FileOutputStream::Open("test.rbf");
 
   EXPECT_TRUE(result.ok());
   std::shared_ptr<arrow::io::OutputStream> aos = result.ValueOrDie();
   // Create a RecordBatchFile writer
-  arrow::Result<std::shared_ptr<arrow::ipc::RecordBatchWriter>> afw = arrow::ipc::NewFileWriter(aos.get(), rb->schema());
+  arrow::Result<std::shared_ptr<arrow::ipc::RecordBatchWriter>>
+      afw = arrow::ipc::MakeFileWriter(aos.get(), rb->schema());
   EXPECT_TRUE(afw.ok());
 
   // Write the RecordBatch to the FileOutputStream

codegen/python/setup.py

@@ -127,12 +127,12 @@ def initialize_options(self):
     install_requires=[
         'numpy >= 1.14',
         'pandas',
-        'pyarrow == 1.0.1',
+        'pyarrow == 3.0.0',
     ],
     setup_requires=[
         'cython',
         'numpy',
-        'pyarrow == 1.0.1',
+        'pyarrow == 3.0.0',
         'plumbum'
     ],
     classifiers=[

common/cpp/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 
 project(fletcher_common VERSION 0.0.0 LANGUAGES CXX)
 
-find_package(Arrow 1.0 CONFIG REQUIRED)
+find_package(Arrow 3.0.0 CONFIG REQUIRED)
 
 include(FetchContent)