Skip to content

Commit d20fbd1

Browse files
vivekjoshyvivekjoshy
authored
Break ties in rank calculation (#168)
* Break ties in rank calculation Fix #166 * Add changelog fragment
1 parent 3d23bc6 commit d20fbd1

24 files changed

+938
-661
lines changed

‎benchmark/draw.ipynb

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,8 @@
156156
"Requirement already satisfied: arrow>=0.15.0 in /opt/conda/lib/python3.10/site-packages (from isoduration->jsonschema>=2.6->nbformat->notebook>=4.4.1->widgetsnbextension~=3.6.0->ipywidgets) (1.2.3)\r\n",
157157
"Downloading rbo-0.1.3-py3-none-any.whl (7.8 kB)\r\n",
158158
"Downloading openskill-6.0.0-py3-none-any.whl (50 kB)\r\n",
159-
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.9/50.9 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
160-
"\u001b[?25hInstalling collected packages: rbo, openskill\r\n",
159+
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m50.9/50.9 kB\u001B[0m \u001B[31m1.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
160+
"\u001B[?25hInstalling collected packages: rbo, openskill\r\n",
161161
"Successfully installed openskill-6.0.0 rbo-0.1.3\r\n",
162162
"Note: you may need to restart the kernel to use updated packages.\n"
163163
]
@@ -210,14 +210,17 @@
210210
"source": [
211211
"import gc\n",
212212
"from dataclasses import dataclass\n",
213+
"from enum import Enum\n",
214+
"from pathlib import Path\n",
213215
"from typing import Dict, List\n",
214216
"\n",
215217
"import ipywidgets as widgets\n",
216-
"import numpy as np\n",
217218
"import pandas as pd\n",
218219
"import rich\n",
220+
"from pooch import DOIDownloader\n",
219221
"from rbo import rbo\n",
220222
"from rich.table import Table\n",
223+
"from sklearn.model_selection import train_test_split\n",
221224
"from tqdm.notebook import tqdm\n",
222225
"\n",
223226
"import openskill\n",
@@ -227,12 +230,7 @@
227230
" PlackettLuce,\n",
228231
" ThurstoneMostellerFull,\n",
229232
" ThurstoneMostellerPart,\n",
230-
")\n",
231-
"\n",
232-
"from pooch import DOIDownloader\n",
233-
"from pathlib import Path\n",
234-
"from sklearn.model_selection import train_test_split\n",
235-
"from enum import Enum"
233+
")"
236234
]
237235
},
238236
{
@@ -1139,12 +1137,12 @@
11391137
"</pre>\n"
11401138
],
11411139
"text/plain": [
1142-
"\u001b[3m Benchmark Results \u001b[0m\n",
1140+
"\u001B[3m Benchmark Results \u001B[0m\n",
11431141
"┏━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓\n",
1144-
"┃\u001b[1m \u001b[0m\u001b[1m Information\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mValue \u001b[0m\u001b[1m \u001b[0m┃\n",
1142+
"┃\u001B[1m \u001B[0m\u001B[1m Information\u001B[0m\u001B[1m \u001B[0m┃\u001B[1m \u001B[0m\u001B[1mValue \u001B[0m\u001B[1m \u001B[0m┃\n",
11451143
"┡━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩\n",
1146-
"│\u001b[36m \u001b[0m\u001b[36m Available Matches\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m20064 \u001b[0m\u001b[35m \u001b[0m│\n",
1147-
"│\u001b[36m \u001b[0m\u001b[36mPlackettLuce Accuracy\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m19335/729 [ 96.37%]\u001b[0m\u001b[35m \u001b[0m│\n",
1144+
"│\u001B[36m \u001B[0m\u001B[36m Available Matches\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m20064 \u001B[0m\u001B[35m \u001B[0m│\n",
1145+
"│\u001B[36m \u001B[0m\u001B[36mPlackettLuce Accuracy\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m19335/729 [ 96.37%]\u001B[0m\u001B[35m \u001B[0m│\n",
11481146
"└───────────────────────┴─────────────────────┘\n"
11491147
]
11501148
},

‎benchmark/rank.ipynb

Lines changed: 30 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,8 @@
5454
"Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (1.26.4)\r\n",
5555
"Collecting numpy\r\n",
5656
" Downloading numpy-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (60 kB)\r\n",
57-
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.9/60.9 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
58-
"\u001b[?25hRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (4.66.4)\r\n",
57+
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m60.9/60.9 kB\u001B[0m \u001B[31m1.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
58+
"\u001B[?25hRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (4.66.4)\r\n",
5959
"Requirement already satisfied: pooch in /opt/conda/lib/python3.10/site-packages (1.8.2)\r\n",
6060
"Requirement already satisfied: ipywidgets in /opt/conda/lib/python3.10/site-packages (7.7.1)\r\n",
6161
"Collecting ipywidgets\r\n",
@@ -96,16 +96,16 @@
9696
"Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.10/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\r\n",
9797
"Downloading rbo-0.1.3-py3-none-any.whl (7.8 kB)\r\n",
9898
"Downloading rich-13.7.1-py3-none-any.whl (240 kB)\r\n",
99-
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m240.7/240.7 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
100-
"\u001b[?25hDownloading ipywidgets-8.1.3-py3-none-any.whl (139 kB)\r\n",
101-
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.4/139.4 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
102-
"\u001b[?25hDownloading openskill-6.0.0-py3-none-any.whl (50 kB)\r\n",
103-
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.9/50.9 kB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
104-
"\u001b[?25hDownloading jupyterlab_widgets-3.0.11-py3-none-any.whl (214 kB)\r\n",
105-
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.4/214.4 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
106-
"\u001b[?25hDownloading widgetsnbextension-4.0.11-py3-none-any.whl (2.3 MB)\r\n",
107-
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m42.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
108-
"\u001b[?25hInstalling collected packages: widgetsnbextension, rbo, openskill, jupyterlab-widgets, rich, ipywidgets\r\n",
99+
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m240.7/240.7 kB\u001B[0m \u001B[31m9.1 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
100+
"\u001B[?25hDownloading ipywidgets-8.1.3-py3-none-any.whl (139 kB)\r\n",
101+
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m139.4/139.4 kB\u001B[0m \u001B[31m6.3 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
102+
"\u001B[?25hDownloading openskill-6.0.0-py3-none-any.whl (50 kB)\r\n",
103+
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m50.9/50.9 kB\u001B[0m \u001B[31m2.2 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
104+
"\u001B[?25hDownloading jupyterlab_widgets-3.0.11-py3-none-any.whl (214 kB)\r\n",
105+
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m214.4/214.4 kB\u001B[0m \u001B[31m8.9 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
106+
"\u001B[?25hDownloading widgetsnbextension-4.0.11-py3-none-any.whl (2.3 MB)\r\n",
107+
"\u001B[2K \u001B[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001B[0m \u001B[32m2.3/2.3 MB\u001B[0m \u001B[31m42.2 MB/s\u001B[0m eta \u001B[36m0:00:00\u001B[0m\r\n",
108+
"\u001B[?25hInstalling collected packages: widgetsnbextension, rbo, openskill, jupyterlab-widgets, rich, ipywidgets\r\n",
109109
" Attempting uninstall: widgetsnbextension\r\n",
110110
" Found existing installation: widgetsnbextension 3.6.7\r\n",
111111
" Uninstalling widgetsnbextension-3.6.7:\r\n",
@@ -122,10 +122,10 @@
122122
" Found existing installation: ipywidgets 7.7.1\r\n",
123123
" Uninstalling ipywidgets-7.7.1:\r\n",
124124
" Successfully uninstalled ipywidgets-7.7.1\r\n",
125-
"\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
125+
"\u001B[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
126126
"tensorflow-decision-forests 1.8.1 requires wurlitzer, which is not installed.\r\n",
127-
"tensorflow 2.15.0 requires keras<2.16,>=2.15.0, but you have keras 3.4.1 which is incompatible.\u001b[0m\u001b[31m\r\n",
128-
"\u001b[0mSuccessfully installed ipywidgets-8.1.3 jupyterlab-widgets-3.0.11 openskill-6.0.0 rbo-0.1.3 rich-13.7.1 widgetsnbextension-4.0.11\r\n",
127+
"tensorflow 2.15.0 requires keras<2.16,>=2.15.0, but you have keras 3.4.1 which is incompatible.\u001B[0m\u001B[31m\r\n",
128+
"\u001B[0mSuccessfully installed ipywidgets-8.1.3 jupyterlab-widgets-3.0.11 openskill-6.0.0 rbo-0.1.3 rich-13.7.1 widgetsnbextension-4.0.11\r\n",
129129
"Note: you may need to restart the kernel to use updated packages.\n"
130130
]
131131
}
@@ -175,15 +175,17 @@
175175
},
176176
"outputs": [],
177177
"source": [
178-
"import time\n",
179178
"import gc\n",
179+
"import time\n",
180180
"from dataclasses import dataclass\n",
181+
"from pathlib import Path\n",
181182
"from typing import Dict\n",
182183
"\n",
183184
"import ipywidgets as widgets\n",
184185
"import numpy as np\n",
185186
"import polars as pl\n",
186187
"import rich\n",
188+
"from pooch import DOIDownloader\n",
187189
"from rbo import rbo\n",
188190
"from rich.table import Table\n",
189191
"from tqdm.notebook import tqdm\n",
@@ -195,10 +197,7 @@
195197
" PlackettLuce,\n",
196198
" ThurstoneMostellerFull,\n",
197199
" ThurstoneMostellerPart,\n",
198-
")\n",
199-
"\n",
200-
"from pooch import DOIDownloader\n",
201-
"from pathlib import Path"
200+
")"
202201
]
203202
},
204203
{
@@ -1307,13 +1306,13 @@
13071306
"</pre>\n"
13081307
],
13091308
"text/plain": [
1310-
"\u001b[3m Benchmark Results \u001b[0m\n",
1309+
"\u001B[3m Benchmark Results \u001B[0m\n",
13111310
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
1312-
"┃\u001b[1m \u001b[0m\u001b[1m Information\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mValue \u001b[0m\u001b[1m \u001b[0m┃\n",
1311+
"┃\u001B[1m \u001B[0m\u001B[1m Information\u001B[0m\u001B[1m \u001B[0m┃\u001B[1m \u001B[0m\u001B[1mValue \u001B[0m\u001B[1m \u001B[0m┃\n",
13131312
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
1314-
"│\u001b[36m \u001b[0m\u001b[36m Available Matches\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m145994 \u001b[0m\u001b[35m \u001b[0m│\n",
1315-
"│\u001b[36m \u001b[0m\u001b[36m PlackettLuce Accuracy\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m134323/11671 [ 92.01%]\u001b[0m\u001b[35m \u001b[0m│\n",
1316-
"│\u001b[36m \u001b[0m\u001b[36m Rank-Biased Overlap Score:\u001b[0m\u001b[36m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m 64.46 \u001b[0m\u001b[35m \u001b[0m│\n",
1313+
"│\u001B[36m \u001B[0m\u001B[36m Available Matches\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m145994 \u001B[0m\u001B[35m \u001B[0m│\n",
1314+
"│\u001B[36m \u001B[0m\u001B[36m PlackettLuce Accuracy\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m134323/11671 [ 92.01%]\u001B[0m\u001B[35m \u001B[0m│\n",
1315+
"│\u001B[36m \u001B[0m\u001B[36m Rank-Biased Overlap Score:\u001B[0m\u001B[36m \u001B[0m│\u001B[35m \u001B[0m\u001B[35m 64.46 \u001B[0m\u001B[35m \u001B[0m│\n",
13171316
"└─────────────────────────────┴────────────────────────┘\n"
13181317
]
13191318
},
@@ -1624,13 +1623,13 @@
16241623
"description": "",
16251624
"description_allow_html": false,
16261625
"layout": "IPY_MODEL_0d1a94edd9e4441f800b85c272cd4174",
1627-
"max": 51704546.0,
1626+
"max": 5.1704546E7,
16281627
"min": 0.0,
16291628
"orientation": "horizontal",
16301629
"style": "IPY_MODEL_c05b6b0fe0b14f8dba8c8532d5becad1",
16311630
"tabbable": null,
16321631
"tooltip": null,
1633-
"value": 51704546.0
1632+
"value": 5.1704546E7
16341633
}
16351634
},
16361635
"1fa24287923243499d42735aa608dff0": {
@@ -2076,13 +2075,13 @@
20762075
"description": "",
20772076
"description_allow_html": false,
20782077
"layout": "IPY_MODEL_5f83e32b011b4503b1e67a736ec1153f",
2079-
"max": 11487236.0,
2078+
"max": 1.1487236E7,
20802079
"min": 0.0,
20812080
"orientation": "horizontal",
20822081
"style": "IPY_MODEL_e87634fce8704dd3824c2a8685c5fd3b",
20832082
"tabbable": null,
20842083
"tooltip": null,
2085-
"value": 11487236.0
2084+
"value": 1.1487236E7
20862085
}
20872086
},
20882087
"6eb3fa3d7ca84770ab2d24b2f6a694f0": {
@@ -2666,13 +2665,13 @@
26662665
"description": "",
26672666
"description_allow_html": false,
26682667
"layout": "IPY_MODEL_f2d82cacecc9493295741ae970381eea",
2669-
"max": 51704546.0,
2668+
"max": 5.1704546E7,
26702669
"min": 0.0,
26712670
"orientation": "horizontal",
26722671
"style": "IPY_MODEL_651e7062e6e94d4495ccd2f06cb05c2a",
26732672
"tabbable": null,
26742673
"tooltip": null,
2675-
"value": 51703476.0
2674+
"value": 5.1703476E7
26762675
}
26772676
},
26782677
"bdfb5a775ba04c458828718e6926edc7": {

0 commit comments

Comments
 (0)