diff --git a/analysis/ec-placement.ipynb b/analysis/ec-placement.ipynb new file mode 100644 index 0000000..55f7301 --- /dev/null +++ b/analysis/ec-placement.ipynb @@ -0,0 +1,359 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 56, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0d654a31aa8842bea08f69d17e1e9283", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(Label(value='EC Layout and Placement'), VBox(children=(IntSlider(value=2, description='K', max=…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "577931878dbf419d85441e582a1760f8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from __future__ import print_function\n", + "from ipywidgets import interact, interactive, fixed, interact_manual\n", + "import ipywidgets as widgets\n", + "import itertools as it\n", + "import tabulate as tb\n", + "from functools import reduce\n", + "from operator import concat\n", + "from collections import OrderedDict\n", + "import math\n", + "\n", + "DEFAULT_BLOCKS = 12\n", + "DEFAULT_NODES = 3\n", + "DEFAULT_K = 2\n", + "DEFAULT_M = 1\n", + "DEFAULT_LAYOUT = \"Inline\"\n", + "DEFAULT_DIRECTION = \"Horizontal\"\n", + "\n", + "\n", + "def roundBlocks(factor, ob):\n", + " return (factor - (ob % factor)) if (ob % factor) > 0 else 0\n", + "\n", + "\n", + "def makeEcGroups(\n", + " blocks,\n", + " k, m,\n", + " direction,\n", + " offset=0):\n", + "\n", + " offset = len(blocks) if offset == 0 else offset\n", + " res = []\n", + "\n", + " groups = (len(blocks) // k)\n", + " steps = len(blocks) // groups\n", + " if direction == \"Horizontal\":\n", + " for b in range(0, len(blocks), steps):\n", + " res.append(\n", + " blocks[b:b + steps] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + " else:\n", + " for s in range(0, groups):\n", + " res.append(\n", + " blocks[s:s+groups+offset:groups] +\n", + " list(range(offset + 1, (offset + 1 + m))))\n", + " offset += m\n", + "\n", + " return res\n", + "\n", + "\n", + "def mapNodes(blocks, hosts, offset=0):\n", + " width = getZeros(len(blocks) or BC.value)\n", + " formatBlock = getBlockFormat(width)\n", + " res = OrderedDict()\n", + "\n", + " for i, _ in enumerate(blocks):\n", + " key = \"node\" + str((i % hosts)+1)\n", + " res[key] = (key in res and res[key]) or []\n", + " res[key].append(formatBlock(blocks[i]))\n", + "\n", + " return res\n", + "\n", + "\n", + "def getBlockFormat(width):\n", + " return lambda b: \"{b:0{fmt}}\".format(b=b, fmt=width)\n", + "\n", + "\n", + "def getZeros(width):\n", + " return int(math.log(width, 10) + 1)\n", + "\n", + "\n", + "def makeReadSeq(K, M, ecGroups, direction, layout):\n", + " readSequence = []\n", + " if layout == \"Append\":\n", + " if direction == \"Horizontal\":\n", + " readSequence = reduce(\n", + " concat,\n", + " [g[:K] for g in ecGroups] + [g[-M:] for g in ecGroups])\n", + " else:\n", + " readSequence = reduce(concat, zip(*ecGroups))\n", + " else:\n", + " readSequence = reduce(concat, ecGroups)\n", + "\n", + " return readSequence\n", + "\n", + "\n", + "class ECPlacement:\n", + " blocks = []\n", + " nodesMap = OrderedDict()\n", + " blockSeq = []\n", + " ecGroups = []\n", + " rb = 0\n", + "\n", + " def __init__(\n", + " self,\n", + " k=DEFAULT_K, m=DEFAULT_M,\n", + " layout=DEFAULT_LAYOUT,\n", + " direction=DEFAULT_LAYOUT,\n", + " nodes=DEFAULT_NODES,\n", + " bc=DEFAULT_BLOCKS):\n", + " self.blocks = list(range(1, bc + 1))\n", + " self.k = k\n", + " self.m = m\n", + " self.layout = layout\n", + " self.direction = direction\n", + " self.nodes = nodes\n", + " self.bc = bc\n", + " self.blocks = []\n", + " self.nodesMap = OrderedDict()\n", + " self.blockSeq = []\n", + " self.ecGroups = []\n", + " self.rb = 0\n", + "\n", + " def reset(self):\n", + " self.k = DEFAULT_K\n", + " self.m = DEFAULT_M\n", + " self.layout = DEFAULT_LAYOUT\n", + " self.direction = DEFAULT_DIRECTION\n", + " self.nodes = DEFAULT_NODES\n", + " self.bc = DEFAULT_BLOCKS\n", + " self.rb = 0\n", + "\n", + " self.blocks = list(range(1, self.bc + 1))\n", + " self.nodesMap = OrderedDict()\n", + " self.blockSeq = []\n", + " self.ecGroups = []\n", + "\n", + " def update(self, k, m, bc, direction, layout, nodes, append):\n", + " print(k, m, bc, direction, layout, nodes, append)\n", + "\n", + " if append:\n", + " if bc > self.bc:\n", + " appendBlocks = bc - self.bc\n", + " appendBlocks = appendBlocks + roundBlocks(k, appendBlocks)\n", + " self.blocks += list(\n", + " range(self.bc + 1, bc + appendBlocks + 1))\n", + "\n", + " if len(self.blocks) > 0:\n", + " blockSeqLen = len(self.blockSeq)\n", + " print(blockSeqLen)\n", + "\n", + " newBlocks = list(\n", + " range(blockSeqLen + 1, blockSeqLen + appendBlocks + 1))\n", + "\n", + " self.ecGroups += makeEcGroups(\n", + " newBlocks, k, m, direction, blockSeqLen + appendBlocks) # Calculate EC groups\n", + "\n", + " self.blockSeq = makeReadSeq(\n", + " k, m, self.ecGroups, direction, layout)\n", + "\n", + " self.nodesMap = mapNodes(\n", + " self.blockSeq, nodes).items()\n", + " else:\n", + " # Calculate geometry\n", + " self.rb = roundBlocks(k, bc)\n", + " # Generate source blocks\n", + " self.blocks = list(\n", + " range(1, bc + 1 + self.rb))\n", + "\n", + " if len(self.blocks) > 0:\n", + " self.ecGroups = makeEcGroups(\n", + " self.blocks, k, m, direction) # Calculate EC groups\n", + "\n", + " self.blockSeq = makeReadSeq(\n", + " k, m, self.ecGroups, direction, layout)\n", + "\n", + " self.nodesMap = mapNodes(\n", + " self.blockSeq, nodes).items()\n", + "\n", + " self.bc = bc\n", + " self.k = k\n", + " self.m = m\n", + " self.direction = dir\n", + " self.layout = layout\n", + " self.nodes = nodes\n", + "\n", + " self.render()\n", + "\n", + " def render(self):\n", + " width = getZeros(len(self.blockSeq) or self.bc)\n", + " formatBlock = getBlockFormat(width)\n", + "\n", + " print(\"\\n\")\n", + " print(\"Rounded Blocks:\", self.rb)\n", + " print(\"Encoding Groups:\", len(self.ecGroups))\n", + " print(\"K + M = \", self.k + self.m)\n", + " print(\"Source Blocks:\", \", \".join(map(formatBlock, self.blocks)))\n", + " print(\"Read Layout:\", \", \".join(map(formatBlock, self.blockSeq)))\n", + "\n", + " groupsTable = tb.tabulate(\n", + " [[\", \".join(map(formatBlock, g[:self.k])), \", \".join(map(formatBlock, g[-self.m:]))]\n", + " for g in self.ecGroups], [\"Source\", \"Parity\"])\n", + " print(\"\\n\", groupsTable)\n", + "\n", + " nodesTable = tb.tabulate(\n", + " list(map(list, self.nodesMap)), [\"Nodes\", \"Blocks\"])\n", + " print(\"\\n\", nodesTable)\n", + "\n", + "\n", + "ecPlacement = ECPlacement()\n", + "\n", + "\n", + "def init(k, m, bc, dir, layout, nodes, append):\n", + " # K - The number of source blocks to encode\n", + " # BC - Initial block count\n", + " # Dir - Coding direction\n", + " # Layout - Block layout\n", + " # Nodes - Nodes count\n", + " ##\n", + "\n", + " ecPlacement.update(k, m, bc, dir, layout, nodes, append)\n", + "\n", + "\n", + "MAX_BLOCKS = 256\n", + "K = widgets.IntSlider(description=\"K\", min=2, max=MAX_BLOCKS)\n", + "M = widgets.IntSlider(description=\"M\", min=1, max=MAX_BLOCKS)\n", + "BC = widgets.IntSlider(description=\"Origina blocks count\",\n", + " min=1, max=10000, value=DEFAULT_BLOCKS)\n", + "\n", + "D = widgets.Dropdown(options=['Horizontal', 'Vertical'],\n", + " description=\"EC Coding Direction\")\n", + "\n", + "BlocksLayout = widgets.Dropdown(\n", + " options=['Inline', 'Append'],\n", + " description=\"Blocks Manifest Layout\")\n", + "\n", + "Clear = widgets.Button(description='Clear')\n", + "Nodes = widgets.IntSlider(\n", + " description=\"Nodes\", min=K.value + M.value, max=MAX_BLOCKS)\n", + "\n", + "Append = widgets.Checkbox(\n", + " value=False,\n", + " description='Append',\n", + " disabled=False,\n", + " indent=False)\n", + "\n", + "ui = widgets.VBox([\n", + " widgets.Label(value=\"EC Layout and Placement\"),\n", + " widgets.VBox([K, M, BC, D, BlocksLayout, Nodes, Append]),\n", + " Clear])\n", + "\n", + "\n", + "def clearModelClick(button):\n", + " ecPlacement.reset()\n", + " K.value = K.min\n", + " M.value = M.min\n", + " Nodes.value = Nodes.min = K.value + M.value\n", + " BC.value = 12\n", + " D.value = \"Horizontal\"\n", + " BlocksLayout.value = \"Inline\"\n", + " appendBlocks.value = 0\n", + " appendNodes = 0\n", + " Append.value = False\n", + "\n", + "\n", + "Clear.on_click(clearModelClick)\n", + "\n", + "\n", + "def updateKRange(*args):\n", + " K.max = MAX_BLOCKS - M.value\n", + " Nodes.value = K.value + M.value\n", + "\n", + "\n", + "M.observe(updateKRange, 'value')\n", + "\n", + "\n", + "def updateMRange(*args):\n", + " M.max = MAX_BLOCKS - K.value\n", + " Nodes.value = K.value + M.value\n", + "\n", + "\n", + "K.observe(updateMRange, 'value')\n", + "\n", + "\n", + "def updateNodesRange(*args):\n", + " Nodes.min = K.value + M.value\n", + "\n", + "\n", + "Nodes.observe(updateNodesRange, 'value')\n", + "\n", + "init = widgets.interactive_output(\n", + " init,\n", + " {\n", + " 'k': K,\n", + " 'm': M,\n", + " 'bc': BC,\n", + " 'dir': D,\n", + " 'layout': BlocksLayout,\n", + " 'nodes': Nodes,\n", + " 'append': Append})\n", + "\n", + "display(ui, init)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.9.5 64-bit ('3.9.5')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.5" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "f341111545ab039e59db4a6307f8d37d8b30a2d09f0bbfe377cad911d3b9e229" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}