docs/source/examples/joblib.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using IPython Parallel as a joblib backend\n",
    "\n",
    "[joblib][] is a tool for running tasks, which includes support for implementing custom parallel backends.\n",
    "IPython defines one such backend, so you can use IPython parallel with joblib.\n",
    "\n",
    "[joblib]: https://joblib.readthedocs.io\n",
    "\n",
    "The simplest way to set this up is a single call:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import ipyparallel as ipp\n",
    "\n",
    "ipp.register_joblib_backend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This registers the 'ipyparallel' backend with all the defaults."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using existing profile dir: '/Users/minrk/.ipython/profile_default'\n",
      "Using existing profile dir: '/Users/minrk/.ipython/profile_default'\n",
      "Starting 4 engines with <class 'ipyparallel.cluster.launcher.LocalEngineSetLauncher'>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be0e2704e2204783ae9435d63d6facff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?engine/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopping engine(s): 1630490654\n",
      "engine set stopped 1630490654: {'0': {'exit_code': 0, 'pid': 14275, 'identifier': '0'}, '1': {'exit_code': 0, 'pid': 14276, 'identifier': '1'}, '2': {'exit_code': 0, 'pid': 14277, 'identifier': '2'}, '3': {'exit_code': 0, 'pid': 14278, 'identifier': '3'}}\n",
      "Stopping controller\n",
      "Controller stopped: {'exit_code': 0, 'pid': 14261, 'identifier': 'ipcontroller-1630490653-glt9-14256'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " -1,\n",
       " -2,\n",
       " -3,\n",
       " -4,\n",
       " -5,\n",
       " -6,\n",
       " -7,\n",
       " -8,\n",
       " -9,\n",
       " -10,\n",
       " -11,\n",
       " -12,\n",
       " -13,\n",
       " -14,\n",
       " -15,\n",
       " -16,\n",
       " -17,\n",
       " -18,\n",
       " -19,\n",
       " -20,\n",
       " -21,\n",
       " -22,\n",
       " -23,\n",
       " -24,\n",
       " -25,\n",
       " -26,\n",
       " -27,\n",
       " -28,\n",
       " -29,\n",
       " -30,\n",
       " -31]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from joblib import Parallel, delayed\n",
    "\n",
    "\n",
    "def neg(x):\n",
    "    return -x\n",
    "\n",
    "\n",
    "Parallel(backend='ipyparallel')(delayed(neg)(i) for i in range(32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can also configure your own View, and register it explicitly, or even as the default:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using existing profile dir: '/Users/minrk/.ipython/profile_default'\n",
      "Starting 4 engines with <class 'ipyparallel.cluster.launcher.LocalEngineSetLauncher'>\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dea6ba0c1c7b419eae13a54f7b6c66fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?engine/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rc = ipp.Cluster(n=4).start_and_connect_sync()\n",
    "\n",
    "rc[:].use_cloudpickle()\n",
    "view = rc.load_balanced_view()\n",
    "view.register_joblib_backend(make_default=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[14317, 14316, 14315, 14306, 14306, 14306, 14306, 14315, 14306, 14306]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "Parallel()(delayed(os.getpid)() for i in range(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Joblib also provides a context manager for selecting a particular backend:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[14316, 14306, 14316, 14306, 14316]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "even = rc.load_balanced_view(targets=rc.ids[::2])\n",
    "even.register_joblib_backend('even')\n",
    "\n",
    "from joblib import parallel_backend\n",
    "\n",
    "with parallel_backend('even'):\n",
    "    result = Parallel()(delayed(os.getpid)() for i in range(5))\n",
    "result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}