examples/gitlogs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run-but-hide\n",
    "import sys\n",
    "sys.path.insert(0, \"..\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from elastipy import Search, query"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "They always say: *put the imports at the top!*\n",
    "\n",
    "\n",
    "# git commit analytics\n",
    "\n",
    "Below we use a lot of :link:`pandas` and plotting to get insight into the community of an open source project. \n",
    "\n",
    "To explore a repository of your choice move to `elastipy/examples/` and call:\n",
    "\n",
    "```bash\n",
    "python gitlogs.py <project-name> path/to/git-repo\n",
    "```\n",
    "\n",
    "If you are `cloning` a repository and are just interested in commits you can somewhat limit the size on disk with:\n",
    "\n",
    "```bash\n",
    "git clone <repo-url> --no-checkout\n",
    "```\n",
    "\n",
    "Replace the `<project-name>` with the name of the project and change the value below in the `notebook`:   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROJECT = \"pandas\"\n",
    "\n",
    "def search():\n",
    "    return Search(f\"elastipy-example-commits-{PROJECT}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run-but-hide\n",
    "\n",
    "# make sure that we actually have results when building\n",
    "# the export for the documentation\n",
    "assert search().execute().total_hits"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## activity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### commits per week"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = search()\n",
    "agg = s.agg_date_histogram(\"date\", calendar_interval=\"week\")\n",
    "df = agg.execute().df(to_index=True)\n",
    "df[\"commits/week\"] = df.pop(\"date.doc_count\")\n",
    "df[\"smooth\"] = df.rolling(window=50).mean()\n",
    "df.plot(figsize=(15,4), color=[\"lightblue\", \"blue\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### additions/deletions per week"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = search()\n",
    "agg = s.agg_date_histogram(\"date\", calendar_interval=\"month\")\n",
    "agg.metric_sum(\"add\", field=\"changes.additions\")\n",
    "agg.metric_sum(\"del\", field=\"changes.deletions\")\n",
    "df = agg.execute().df(to_index=True, exclude=\"*doc_count\")\n",
    "#df = df.rolling(window=10).mean()[[\"add\", \"del\"]]\n",
    "df.plot.line(color=[\"green\", \"pink\"], figsize=(15,4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### commits per weekday/hour for each year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def commits_per(field, interval=\"year\"):\n",
    "    s = search()\n",
    "    agg = s.agg_date_histogram(interval, calendar_interval=interval)\n",
    "    #agg = s.agg_terms(\"author\", field=\"author\")\n",
    "    agg = agg.agg_terms(\"weekday\", field=field, size=100)\n",
    "    agg.execute().plot.heatmap(\n",
    "        sort=True, transpose=True, \n",
    "        annot=False, fmt=\".0f\", cmap=\"gray_r\", figsize=(15, .3), \n",
    "    )\n",
    "commits_per(\"timestamp_weekday\")\n",
    "commits_per(\"timestamp_hour\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## authors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### top 3 authors per year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = search()\n",
    "agg = s.agg_date_histogram(\"date\", calendar_interval=\"year\")\n",
    "agg = agg.agg_terms(\"author\", field=\"author\", size=3)\n",
    "agg_top3_authors = agg\n",
    "agg.execute().df(to_index=True, flat=\"author\", exclude=\"*doc_count\").plot.bar(figsize=(15,4), stacked=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### commits of all top 3 authors  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_authors = set(k[1] for k in agg_top3_authors.keys())\n",
    "\n",
    "s = search()\n",
    "agg = s.agg_filters(\"author\", filters={key: query.Term(\"author\", key) for key in top_authors})\n",
    "agg = agg.agg_date_histogram(\"date\", calendar_interval=\"year\")\n",
    "agg.execute().plot.heatmap(\n",
    "    sort=True, replace={0: np.nan},\n",
    "    annot=True, fmt=\".0f\", figsize=(15, .6), cmap=\"gray_r\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### top 3 average-additions per author per year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = search()\n",
    "agg = s.agg_filters(\"author\", filters={key: query.Term(\"author\", key) for key in top_authors})\n",
    "agg = agg.agg_date_histogram(\"date\", calendar_interval=\"year\")\n",
    "agg = agg.metric_avg(\"avg-add\", field=\"changes.additions\", return_self=True)\n",
    "agg.execute().plot.heatmap(\n",
    "    sort=True, replace={0: np.nan},\n",
    "    annot=True, fmt=\".0f\", figsize=(15, .6), cmap=\"gray_r\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### number of authors per year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = search()\n",
    "global_authors = s.metric_cardinality(field=\"author\", return_self=True)\n",
    "agg = s.agg_date_histogram(\"year\", calendar_interval=\"year\")\n",
    "agg = agg.metric_cardinality(\"authors\", field=\"author\")\n",
    "agg.execute().plot.bar(\"year\", \"authors\", figsize=(15, 4))\n",
    "print(next(global_authors.values()), \"authors at all\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## commit messages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### the first ten commit messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = search().sort(\"timestamp\")\n",
    "# s = s.range(\"timestamp\", gte=\"2020\")\n",
    "for d in s.execute().documents:\n",
    "    print((\"-- %(timestamp)s %(hash)s\\n%(message)s\" % d).strip() + \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### significant terms by year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def significant_terms_by_year(s, field, size=4, shard_size=100):\n",
    "    agg = s.copy().agg_date_histogram(\"year\", calendar_interval=\"year\")\n",
    "    agg = agg.agg_significant_terms(field=field, size=size, shard_size=shard_size)\n",
    "    keywords = set(k[-1] for k in agg.execute().keys())\n",
    "\n",
    "    agg = s.agg_date_histogram(\"date\", calendar_interval=\"year\")\n",
    "    agg = agg.agg_filters(\"word\", filters={key: query.Term(field, key) for key in keywords})\n",
    "    agg.execute().plot.heatmap(\n",
    "        sort=True, replace={0: np.nan},\n",
    "        transpose=True, annot=True, fmt=\".0f\", figsize=(.3, .7), cmap=\"gray_r\"\n",
    "    )\n",
    "    \n",
    "significant_terms_by_year(search(), \"message\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### significant terms by author"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def significant_terms_by_terms(s, split_field, terms_field, split_size=30, size=3, shard_size=100):\n",
    "    agg = s.copy().agg_terms(split_field, field=split_field, size=split_size)\n",
    "    agg = agg.agg_significant_terms(\"term\", field=terms_field, size=size, shard_size=shard_size)\n",
    "    df = agg.execute().df(include=[\"term\", \"term.doc_count\"])\n",
    "    \n",
    "    # find max count of all significant terms\n",
    "    df = df.groupby(\"term\").max()\n",
    "    # print(df.describe())\n",
    "    \n",
    "    # and drop everything above a high percentile \n",
    "    df = df[df < df.quantile(.8)].dropna()\n",
    "    keywords = list(df.index)\n",
    "\n",
    "    agg = s.agg_terms(split_field, field=split_field, size=split_size)\n",
    "    agg = agg.agg_filters(\"term\", filters={key: query.Term(terms_field, key) for key in keywords})\n",
    "    agg.execute().plot.heatmap(\n",
    "        sort=True, transpose=True, replace={0: np.nan}, \n",
    "        annot=True, fmt=\".0f\", figsize=(.23, .6), cmap=\"gray_r\"\n",
    "    )\n",
    "    \n",
    "significant_terms_by_terms(search(), \"author\", \"message\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### overall top 50 edited files per year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = search()\n",
    "agg = s.agg_terms(field=\"changes.file\", size=50)\n",
    "agg = agg.agg_date_histogram(\"date\", calendar_interval=\"year\")\n",
    "df = agg.execute().plot.heatmap(\n",
    "    sort=True, replace={0: np.nan},\n",
    "    annot=True, fmt=\".0f\", figsize=(.3, 1.5), cmap=\"gray_r\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### significant changed files by year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = search().param(rest_total_hits_as_int=True)\n",
    "# remove version specific files\n",
    "s = ~s.query_string(\"changes.file: *.txt *.rst\")\n",
    "significant_terms_by_year(s, \"changes.file\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### significant changed files by author"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "significant_terms_by_terms(search(), \"author\", \"changes.file\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  which files get edited together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = search()\n",
    "s = s.query_string(\"changes.file: __init__.py\")\n",
    "\n",
    "agg = s.agg_terms(field=\"changes.file\", size=50)\n",
    "agg = agg.agg_date_histogram(\"date\", calendar_interval=\"year\")\n",
    "try:\n",
    "    agg.execute().plot.heatmap(figsize=(.3, 1.5), cmap=\"gray_r\")\n",
    "except ValueError:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}