[WIP] Python binding

CMakeLists.txt

@@ -28,6 +28,7 @@ option(SUPPORT_TTL "whether support TTL" OFF)
 option(OPT_SUPPORT_ZSTD_TRACE "whether support zstd trace" ON)
 option(ENABLE_LRB "enable LRB" OFF)
 option(ENABLE_3L_CACHE "enable 3LCache" OFF)
+option(BUILD_SHARED_LIBS "Build shared library" ON)
 set(LOG_LEVEL NONE CACHE STRING "change the logging level")
 set_property(CACHE LOG_LEVEL PROPERTY STRINGS INFO WARN ERROR DEBUG VERBOSE VVERBOSE VVVERBOSE)
 
@@ -354,9 +355,12 @@ set(LIB_SOURCE ${LIB_SOURCE} ${cache_source} ${reader_source} ${dataStructure_so
 # # https://stackoverflow.com/questions/32469953/why-is-cmake-designed-so-that-it-removes-runtime-path-when-installing
 SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
-add_library(${PROJECT_NAME} ${LIB_SOURCE})
+if(BUILD_SHARED_LIBS)
+    add_library(${PROJECT_NAME} SHARED ${LIB_SOURCE})
+else()
+    add_library(${PROJECT_NAME} ${LIB_SOURCE})
+endif()
 
-# add_library(${PROJECT_NAME} SHARED ${LIB_SOURCE})
 set_target_properties(${PROJECT_NAME} PROPERTIES VERSION ${${PROJECT_NAME}_VERSION})
 set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE C)
 set_target_properties(${PROJECT_NAME} PROPERTIES PUBLIC_HEADER ${PROJECT_SOURCE_DIR}/libCacheSim/include/libCacheSim.h)
@@ -383,3 +387,8 @@ if(ENABLE_TESTS)
 else()
     message(STATUS "Building without test")
 endif()
+
+if (BUILD_SHARED_LIBS)
+    target_link_libraries(${PROJECT_NAME} PUBLIC ${LIBS})
+    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/libCacheSim/python)
+endif()

libCacheSim/bin/cachesim/CMakeLists.txt

@@ -3,5 +3,8 @@ add_executable(cachesim main.c cli_parser.c sim.c ../cli_reader_utils.c)
 target_link_libraries(cachesim ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils)
 install(TARGETS cachesim RUNTIME DESTINATION bin)
 
-# add_executable(flash flash.cpp cli_parser.c sim.c ../cli_reader_utils.c)
-# target_link_libraries(flash ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils)
+if (BUILD_SHARED_LIBS)
+    set(source cli_parser.c sim.c ../cli_reader_utils.c)
+    add_library(lcachesim ${source})
+    target_link_libraries(lcachesim ${ALL_MODULES} ${LIBS} ${CMAKE_THREAD_LIBS_INIT} utils)
+endif()

libCacheSim/bin/cachesim/cache_init.h

@@ -1,6 +1,8 @@
 
 #include <strings.h>
+#ifndef _GNU_SOURCE
 #define _GNU_SOURCE
+#endif
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -72,7 +74,7 @@ static inline cache_t *create_cache(const char *trace_path, const char *eviction
     } else {
       const char *window_size = strstr(eviction_params, "window-size=");
       if (window_size == NULL) {
-        char *new_params = malloc(strlen(eviction_params) + 20);
+        char *new_params = (char *)malloc(strlen(eviction_params) + 20);
         sprintf(new_params, "%s,window-size=0.01", eviction_params);
         cache = WTinyLFU_init(cc_params, new_params);
       } else {

libCacheSim/include/python/api.h

@@ -0,0 +1,115 @@
+#ifndef LIBCACHESIM_API_H_
+#define LIBCACHESIM_API_H_
+
+// TODO(haocheng): add more headers
+#include "../../bin/cachesim/cache_init.h"
+#include "../../bin/cli_reader_utils.h"
+#include "../libCacheSim/simulator.h"
+
+#ifdef __cplusplus
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#define LIBCACHESIM_C_EXPORT extern "C"
+#else
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#define LIBCACHESIM_C_EXPORT
+#endif
+
+typedef void* CacheHandle;    /*!< \brief Handle of cache. */
+typedef void* ReaderHandle;   /*!< \brief Handle of reader. */
+typedef void* CacheObjHandle; /*!< \brief Handle of cache object. */
+typedef void* RequestHandle;  /*!< \brief Handle of request. */
+
+#define C_API_DTYPE_FLOAT32 (0) /*!< \brief float32 (single precision float). */
+#define C_API_DTYPE_FLOAT64 (1) /*!< \brief float64 (double precision float). */
+#define C_API_DTYPE_INT32 (2)   /*!< \brief int32. */
+#define C_API_DTYPE_INT64 (3)   /*!< \brief int64. */
+
+LIBCACHESIM_C_EXPORT int LCS_Simulate(const char* trace_path, const char* trace_type_str, const char* eviction_algo,
+                                      float cache_size, bool ignore_obj_size, const char* eviction_params,
+                                      int64_t* n_req,  // TODO(haocheng): add more results
+                                      int64_t* n_miss);
+
+/* --- start Cache interface */
+/*!
+ * \brief Create a reader from a trace file.
+ * \param filename The name of the trace file
+ * \param parameters Additional parameters
+ * \param[out] out A reader
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIBCACHESIM_C_EXPORT int LCS_ReaderCreateFromFile(const char* filename, const char* parameters,
+                                                   ReaderHandle* out);
+
+/*!
+ * \brief Free a reader.
+ * \param reader The reader
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIBCACHESIM_C_EXPORT int LCS_ReaderFree(const ReaderHandle reader);
+
+/*!
+ * \brief Get the next request from the reader.
+ * \param reader The reader
+ * \param[out] out A request
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIBCACHESIM_C_EXPORT int LCS_ReaderGetNextRequest(const ReaderHandle reader, RequestHandle* out);
+
+/*!
+ * \brief Get the number of requests in the reader.
+ * \param reader The reader
+ * \param[out] out The number of requests
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIBCACHESIM_C_EXPORT int LCS_ReaderGetReqNum(const ReaderHandle reader, int64_t* out);
+
+/*!
+ * \brief Get the number of objects in the reader.
+ * \param reader The reader
+ * \param[out] out The number of objects
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIBCACHESIM_C_EXPORT int LCS_ReaderGetObjNum(const ReaderHandle reader, int64_t* out);
+
+/*!
+ * \brief Free a request.
+ * \param request The request
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIBCACHESIM_C_EXPORT int LCS_RequestFree(const RequestHandle request);
+
+/*!
+ * \brief Create a cache.
+ * \param reader The reader
+ * \param eviction_algo The name of the eviction algorithm
+ * \param cache_size The size of the cache
+ * \param ignore_obj_size Whether to ignore the object size
+ * \param eviction_parameters Additional parameters
+ * \param[out] out A cache
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIBCACHESIM_C_EXPORT int LCS_CacheCreate(const ReaderHandle reader, const char* eviction_algo, float cache_size, bool ignore_obj_size,
+                                        const char* eviction_parameters, CacheHandle* out);
+
+/*!
+ * \brief Free a cache.
+ * \param cache The cache
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIBCACHESIM_C_EXPORT int LCS_CacheFree(const CacheHandle cache);
+
+/*!
+ * \brief Get a cache object.
+ * \param cache The cache
+ * \param request The request
+ * \param[out] hit Whether the request is hit
+ * \return 0 when succeed, -1 when failure happens
+ */
+LIBCACHESIM_C_EXPORT int LCS_CacheGet(const CacheHandle cache, const RequestHandle request,
+                                      bool* hit);
+
+#endif  // LIBCACHESIM_API_H_

libCacheSim/python/CMakeLists.txt

@@ -0,0 +1,3 @@
+aux_source_directory(. source)
+add_library (python ${source})
+target_link_libraries(python lcachesim)

libCacheSim/python/api.cpp

@@ -0,0 +1,169 @@
+#include "../include/python/api.h"
+
+#include <exception>
+#include <iostream>
+
+// namespace libCacheSim {
+
+#define API_BEGIN() try {
+#define API_END()                                       \
+  }                                                     \
+  catch (std::exception & ex) {                         \
+    return LCS_APIHandleException(ex.what());           \
+  }                                                     \
+  catch (...) {                                         \
+    return LCS_APIHandleException("unknown exception"); \
+  }                                                     \
+  return 0;
+
+inline int LCS_APIHandleException(const char* ex) {
+  // dump exception
+  std::cerr << ex << std::endl;
+  return -1;
+}
+
+uint64_t _convert_cache_size_to_bytes(float cache_size, ReaderHandle reader, bool ignore_obj_size) {
+  int64_t wss_obj = 0, wss_byte = 0;
+  cal_working_set_size(static_cast<reader_t*>(reader), &wss_obj, &wss_byte);
+  uint64_t wss = ignore_obj_size ? wss_obj : wss_byte;
+  uint64_t cache_size_bytes = 0;
+  if (cache_size <= 0) {
+    cache_size_bytes = wss;
+  } else if (cache_size < 1) {
+    cache_size_bytes = (uint64_t)(wss * cache_size);
+  } else {
+    cache_size_bytes = (uint64_t)cache_size;
+  }
+  return cache_size_bytes;
+}
+
+int LCS_Simulate(const char* trace_path, const char* trace_type_str, const char* eviction_algo, float cache_size,
+                 bool ignore_obj_size, const char* eviction_params,
+                 // results
+                 int64_t* n_req,  // TODO(haocheng): add more results
+                 int64_t* n_miss) {
+  API_BEGIN();
+  trace_type_e trace_type = trace_type_str_to_enum(trace_type_str, trace_path);
+  reader_t* reader = setup_reader(trace_path, trace_type, NULL);
+  uint64_t cache_size_bytes = _convert_cache_size_to_bytes(cache_size, reader, ignore_obj_size);
+
+  // TODO(haocheng): consider obj metadata is false by default, make it changeable
+  cache_t* cache = create_cache(trace_path, eviction_algo, cache_size_bytes, eviction_params, false);
+
+  int num_of_sizes = 1;
+  uint64_t cache_sizes[1] = {cache_size_bytes};
+  reader_t* warmup_reader = NULL;
+  double warmup_frac = 0;
+  int warmup_sec = 0;
+  int num_of_threads = 1;
+  bool use_random_seed = false;
+  // simulate
+  cache_stat_t* stat = simulate_at_multi_sizes(reader, cache, num_of_sizes, cache_sizes, warmup_reader, warmup_frac,
+                                               warmup_sec, num_of_threads, use_random_seed);
+  // extract results
+  *n_req = stat[0].n_req;
+  *n_miss = stat[0].n_miss;
+  API_END();
+  return 0;
+}
+
+// ----------------------------------------------------------------------------
+char* _extract_trace_type_str(const char* trace_path) {
+  // TODO(haocheng): add more trace types
+  // 'csv', 'oracleGeneral', 'vscsi', 'txt'
+  // match string in trace_path
+  if (strstr(trace_path, "csv")) {
+    return "csv";
+  } else if (strstr(trace_path, "oracleGeneral")) {
+    return "oracleGeneral";
+  } else if (strstr(trace_path, "vscsi")) {
+    return "vscsi";
+  } else if (strstr(trace_path, "txt")) {
+    return "txt";
+  } else {
+    // exception
+    throw LCS_APIHandleException("Unknown trace type");
+  }
+}
+
+int LCS_ReaderCreateFromFile(const char* filename, const char* parameters, ReaderHandle* out) {
+  API_BEGIN();
+  // TODO(haocheng): add more parameters
+  // initialize reader by filename
+  trace_type_e trace_type = trace_type_str_to_enum(_extract_trace_type_str(filename), filename);
+  reader_t* reader = setup_reader(filename, trace_type, NULL);
+  if (reader == NULL) {
+    printf("Failed to create reader\n");
+  }
+  *out = (ReaderHandle)reader;
+  API_END();
+}
+
+int LCS_ReaderFree(const ReaderHandle reader) {
+  API_BEGIN();
+  close_reader(static_cast<reader_t*>(reader));
+  API_END();
+}
+
+int LCS_ReaderGetNextRequest(const ReaderHandle reader, RequestHandle* out) {
+  API_BEGIN();
+  request_t* request = (request_t*)malloc(sizeof(request_t));
+  int status = read_one_req(static_cast<reader_t*>(reader), request);
+  if (status == 0) {
+    *out = request;
+  } else if (status == 1) {
+    *out = NULL;
+  } else {
+    printf("Failed to get next request\n");
+    return -1;
+  }
+  API_END();
+}
+
+int LCS_ReaderGetObjNum(const ReaderHandle reader, int64_t* out) {
+  API_BEGIN();
+  int64_t wss_obj = 0, wss_byte = 0;
+  cal_working_set_size(static_cast<reader_t*>(reader), &wss_obj, &wss_byte);
+  *out = wss_obj;
+  API_END();
+}
+
+int LCS_ReaderGetReqNum(const ReaderHandle reader, int64_t* out) {
+  API_BEGIN();
+  *out = (int64_t)(static_cast<reader_t*>(reader)->n_total_req);
+  API_END();
+}
+
+int LCS_CacheCreate(const ReaderHandle reader, const char* eviction_algo, float cache_size, bool ignore_obj_size,
+                    const char* eviction_parameters, CacheHandle* out) {
+  API_BEGIN();
+  // TODO(haocheng): add more parameters
+  const char* trace_path = static_cast<reader_t*>(reader)->trace_path;
+  cache_t* cache =
+      create_cache(trace_path, eviction_algo, _convert_cache_size_to_bytes(cache_size, reader, ignore_obj_size),
+                   eviction_parameters, false);
+  *out = cache;
+  API_END();
+}
+
+int LCS_CacheFree(const CacheHandle cache) {
+  API_BEGIN();
+  cache_t* cache_ptr = static_cast<cache_t*>(cache);
+  cache_ptr->cache_free(cache_ptr);
+  API_END();
+}
+
+int LCS_CacheGet(const CacheHandle cache, const RequestHandle request, bool* hit) {
+  API_BEGIN();
+  *hit = cache_get_base(static_cast<cache_t*>(cache), static_cast<request_t*>(request));
+  // printf("hit from c side: %d\n", *hit);
+  API_END();
+}
+
+int LCS_RequestFree(const RequestHandle request) {
+  API_BEGIN();
+  free_request(static_cast<request_t*>(request));
+  API_END();
+}
+
+// }

libCacheSim/python/design.md

@@ -0,0 +1,7 @@
+Expose the cache interface to the user.
+
+```python
+cache = FIFO(cache_size, **kwargs)
+for req in reader: 
+    hit = cache.get(req)
+```

python-package/README.md

@@ -0,0 +1,18 @@
+libCacheSim Python-package
+=======================
+
+## Installation
+
+### Install from [PyPI](https://pypi.org/project/libCacheSim)
+
+note: coming soon
+
+```sh
+pip install libcachesim
+```
+
+### Build from Sources
+
+```sh
+pip install --no-binary libcachesim libcachesim
+```