{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convolutional Neural Network\n",
    "\n",
    "We benchmark the convolutional neural networks provided by the [Gluon modelzoo](https://mxnet.incubator.apache.org/api/python/gluon/model_zoo.html).\n",
    "\n",
    "## Inference\n",
    "\n",
    "### Throughput on various batch size\n",
    "\n",
    "Given network `net` and batch size `b`, we feed `b` images, denoted by `X`, into `net` to measture the time `t` to complete `net(X)`. We then calculate the throughput as `b/t`. We first load the benchmark resutls and print all network and devices names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "1"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(['AlexNet', 'DensetNet-121', 'DensetNet-161', 'DensetNet-169',\n",
       "        'DensetNet-201', 'ResNet-v1-101', 'ResNet-v1-152', 'ResNet-v1-18',\n",
       "        'ResNet-v1-34', 'ResNet-v1-50', 'ResNet-v2-18', 'ResNet-v2-34',\n",
       "        'ResNet-v2-50', 'SqueezeNet-1.0', 'SqueezeNet-1.1', 'VGG-11',\n",
       "        'VGG-13', 'VGG-16', 'VGG-19'], dtype=object),\n",
       " array(['Tesla K80', 'Tesla V100-SXM2-16GB', 'Tesla M60'], dtype=object))"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import dlmark as dm\n",
    "\n",
    "thr = dm.benchmark.load_results('cnn.py__benchmark_throughput*json')\n",
    "\n",
    "models = thr.model.unique()\n",
    "devices = thr.device.unique()\n",
    "(models, devices)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we visualize the throughput for each network when increasing the batch sizes. We only use the results on the first device and show a quater of networks:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "2"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div class=\"bk-root\">\n",
       "        <a href=\"https://bokeh.pydata.org\" target=\"_blank\" class=\"bk-logo bk-logo-small bk-logo-notebook\"></a>\n",
       "        <span id=\"637620dc-1d4b-42a6-8b09-2741425877ff\">Loading BokehJS ...</span>\n",
       "    </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  var force = true;\n",
       "\n",
       "  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "  var JS_MIME_TYPE = 'application/javascript';\n",
       "  var HTML_MIME_TYPE = 'text/html';\n",
       "  var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  var CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    var script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    var cell = handle.cell;\n",
       "\n",
       "    var id = cell.output_area._bokeh_element_id;\n",
       "    var server_id = cell.output_area._bokeh_server_id;\n",
       "    // Clean up Bokeh references\n",
       "    if (id !== undefined) {\n",
       "      Bokeh.index[id].model.document.clear();\n",
       "      delete Bokeh.index[id];\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            var element_id = msg.content.text.trim();\n",
       "            Bokeh.index[element_id].model.document.clear();\n",
       "            delete Bokeh.index[element_id];\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    var output_area = handle.output_area;\n",
       "    var output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[0].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      var bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      var script_attrs = bk_div.children[0].attributes;\n",
       "      for (var i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[0].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      var toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[0]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    var events = require('base/js/events');\n",
       "    var OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  \n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  var NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded() {\n",
       "    var el = document.getElementById(\"637620dc-1d4b-42a6-8b09-2741425877ff\");\n",
       "    if (el != null) {\n",
       "      el.textContent = \"BokehJS is loading...\";\n",
       "    }\n",
       "    if (root.Bokeh !== undefined) {\n",
       "      if (el != null) {\n",
       "        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(display_loaded, 100)\n",
       "    }\n",
       "  }\n",
       "\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n",
       "    }\n",
       "    finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.info(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(js_urls, callback) {\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = js_urls.length;\n",
       "    for (var i = 0; i < js_urls.length; i++) {\n",
       "      var url = js_urls[i];\n",
       "      var s = document.createElement('script');\n",
       "      s.src = url;\n",
       "      s.async = false;\n",
       "      s.onreadystatechange = s.onload = function() {\n",
       "        root._bokeh_is_loading--;\n",
       "        if (root._bokeh_is_loading === 0) {\n",
       "          console.log(\"Bokeh: all BokehJS libraries loaded\");\n",
       "          run_callbacks()\n",
       "        }\n",
       "      };\n",
       "      s.onerror = function() {\n",
       "        console.warn(\"failed to load library \" + url);\n",
       "      };\n",
       "      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "    }\n",
       "  };var element = document.getElementById(\"637620dc-1d4b-42a6-8b09-2741425877ff\");\n",
       "  if (element == null) {\n",
       "    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '637620dc-1d4b-42a6-8b09-2741425877ff' but no matching script tag was found. \")\n",
       "    return false;\n",
       "  }\n",
       "\n",
       "  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.13.min.js\"];\n",
       "\n",
       "  var inline_js = [\n",
       "    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "    \n",
       "    function(Bokeh) {\n",
       "      \n",
       "    },\n",
       "    function(Bokeh) {\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n",
       "      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n",
       "      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    \n",
       "    if ((root.Bokeh !== undefined) || (force === true)) {\n",
       "      for (var i = 0; i < inline_js.length; i++) {\n",
       "        inline_js[i].call(root, root.Bokeh);\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      var cell = $(document.getElementById(\"637620dc-1d4b-42a6-8b09-2741425877ff\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(js_urls, function() {\n",
       "      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n  function now() {\n    return new Date();\n  }\n\n  var force = true;\n\n  if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n    root._bokeh_onload_callbacks = [];\n    root._bokeh_is_loading = undefined;\n  }\n\n  \n\n  \n  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n    root._bokeh_timeout = Date.now() + 5000;\n    root._bokeh_failed_load = false;\n  }\n\n  var NB_LOAD_WARNING = {'data': {'text/html':\n     \"<div style='background-color: #fdd'>\\n\"+\n     \"<p>\\n\"+\n     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n     \"</p>\\n\"+\n     \"<ul>\\n\"+\n     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n     \"</ul>\\n\"+\n     \"<code>\\n\"+\n     \"from bokeh.resources import INLINE\\n\"+\n     \"output_notebook(resources=INLINE)\\n\"+\n     \"</code>\\n\"+\n     \"</div>\"}};\n\n  function display_loaded() {\n    var el = document.getElementById(\"637620dc-1d4b-42a6-8b09-2741425877ff\");\n    if (el != null) {\n      el.textContent = \"BokehJS is loading...\";\n    }\n    if (root.Bokeh !== undefined) {\n      if (el != null) {\n        el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n      }\n    } else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(display_loaded, 100)\n    }\n  }\n\n\n  function run_callbacks() {\n    try {\n      root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n    }\n    finally {\n      delete root._bokeh_onload_callbacks\n    }\n    console.info(\"Bokeh: all callbacks have finished\");\n  }\n\n  function load_libs(js_urls, callback) {\n    root._bokeh_onload_callbacks.push(callback);\n    if (root._bokeh_is_loading > 0) {\n      console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n      return null;\n    }\n    if (js_urls == null || js_urls.length === 0) {\n      run_callbacks();\n      return null;\n    }\n    console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n    root._bokeh_is_loading = js_urls.length;\n    for (var i = 0; i < js_urls.length; i++) {\n      var url = js_urls[i];\n      var s = document.createElement('script');\n      s.src = url;\n      s.async = false;\n      s.onreadystatechange = s.onload = function() {\n        root._bokeh_is_loading--;\n        if (root._bokeh_is_loading === 0) {\n          console.log(\"Bokeh: all BokehJS libraries loaded\");\n          run_callbacks()\n        }\n      };\n      s.onerror = function() {\n        console.warn(\"failed to load library \" + url);\n      };\n      console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n      document.getElementsByTagName(\"head\")[0].appendChild(s);\n    }\n  };var element = document.getElementById(\"637620dc-1d4b-42a6-8b09-2741425877ff\");\n  if (element == null) {\n    console.log(\"Bokeh: ERROR: autoload.js configured with elementid '637620dc-1d4b-42a6-8b09-2741425877ff' but no matching script tag was found. \")\n    return false;\n  }\n\n  var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.13.min.js\"];\n\n  var inline_js = [\n    function(Bokeh) {\n      Bokeh.set_log_level(\"info\");\n    },\n    \n    function(Bokeh) {\n      \n    },\n    function(Bokeh) {\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n      console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n      Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n    }\n  ];\n\n  function run_inline_js() {\n    \n    if ((root.Bokeh !== undefined) || (force === true)) {\n      for (var i = 0; i < inline_js.length; i++) {\n        inline_js[i].call(root, root.Bokeh);\n      }if (force === true) {\n        display_loaded();\n      }} else if (Date.now() < root._bokeh_timeout) {\n      setTimeout(run_inline_js, 100);\n    } else if (!root._bokeh_failed_load) {\n      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n      root._bokeh_failed_load = true;\n    } else if (force !== true) {\n      var cell = $(document.getElementById(\"637620dc-1d4b-42a6-8b09-2741425877ff\")).parents('.cell').data().cell;\n      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n    }\n\n  }\n\n  if (root._bokeh_is_loading === 0) {\n    console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n    run_inline_js();\n  } else {\n    load_libs(js_urls, function() {\n      console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n      run_inline_js();\n    });\n  }\n}(window));"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "<div class=\"bk-root\">\n",
       "    <div class=\"bk-plotdiv\" id=\"af91d39a-b87c-4801-af15-a8c8842abd2a\"></div>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"1b4d8739-f3aa-475d-b0b2-5b9ed7debcdb\":{\"roots\":{\"references\":[{\"attributes\":{\"grid_line_color\":{\"value\":null},\"plot\":{\"id\":\"3541a191-5b4e-4938-9edb-c0a2ed1e9844\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"33484252-8ab2-477c-bc1c-c14cf049070a\",\"type\":\"LogTicker\"}},\"id\":\"e9fe79fd-5401-4cc0-9734-ca8113d88cfc\",\"type\":\"Grid\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"782060d5-6a60-41ba-b104-a9ba3dc1e45b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"aca7420c-4880-426c-94da-060b0f5728eb\",\"type\":\"BasicTicker\"}},\"id\":\"cd819b44-ee0e-422c-8a9a-854ba7cf06a9\",\"type\":\"Grid\"},{\"attributes\":{\"num_minor_ticks\":10},\"id\":\"959cc564-b067-4456-b705-57474928e39a\",\"type\":\"LogTicker\"},{\"attributes\":{\"axis_label\":\"#examples / sec\",\"formatter\":{\"id\":\"a2bc0a1a-2166-42a1-a20b-eccbfd69252d\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"3541a191-5b4e-4938-9edb-c0a2ed1e9844\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"4f42179a-d1cc-4856-a65e-9ddc6cf41a64\",\"type\":\"BasicTicker\"}},\"id\":\"aa11d5c4-2304-495e-844a-620297377e0e\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"4f42179a-d1cc-4856-a65e-9ddc6cf41a64\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"3541a191-5b4e-4938-9edb-c0a2ed1e9844\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"4f42179a-d1cc-4856-a65e-9ddc6cf41a64\",\"type\":\"BasicTicker\"}},\"id\":\"05f56e47-127c-44e9-af52-22405e285aa1\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"d5e8bced-9419-4e0a-b920-cf8df70d743b\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"ticks\":[1,2,4,8,16,32,64,128,256]},\"id\":\"2dfe26e2-340e-4d61-82cf-d06ca198d524\",\"type\":\"FixedTicker\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"Throughput\",\"@throughput\"],[\"Batch size\",\"@batch_size\"],[\"Device memory\",\"@device_mem MB\"]]},\"id\":\"42176ec0-2426-40dc-a7b0-58f8a0e5270e\",\"type\":\"HoverTool\"},{\"attributes\":{\"children\":[{\"id\":\"43fd4ee9-b762-41ef-9549-d5a730f887ad\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"id\":\"782060d5-6a60-41ba-b104-a9ba3dc1e45b\",\"subtype\":\"Figure\",\"type\":\"Plot\"}]},\"id\":\"6f4f3dc3-e10f-4c0f-b83b-de0885fbc395\",\"type\":\"Row\"},{\"attributes\":{\"plot\":null,\"text\":\"AlexNet @ Tesla K80\"},\"id\":\"d7e157a1-7b2c-4462-83bc-ec3dd54f9276\",\"type\":\"Title\"},{\"attributes\":{\"plot\":null,\"text\":\"ResNet-v2-50 @ Tesla K80\"},\"id\":\"8cdd5241-b80f-4bae-a7bc-a6c57a632bec\",\"type\":\"Title\"},{\"attributes\":{\"ticker\":null},\"id\":\"70b0df26-7521-4955-bd05-24a4d84cd5cf\",\"type\":\"LogTickFormatter\"},{\"attributes\":{},\"id\":\"a2bc0a1a-2166-42a1-a20b-eccbfd69252d\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"source\":{\"id\":\"eb97d47b-7bc2-4d9c-84cc-b2682e259774\",\"type\":\"ColumnDataSource\"}},\"id\":\"6207b4a4-ac5a-4136-9984-ff6ace51783a\",\"type\":\"CDSView\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"0c94c8a1-f0d6-4ccc-8b10-d3b5a1809b20\",\"type\":\"HoverTool\"}]},\"id\":\"81a2f403-f0f2-49d6-8be0-c53cfd27f163\",\"type\":\"Toolbar\"},{\"attributes\":{\"data_source\":{\"id\":\"eb97d47b-7bc2-4d9c-84cc-b2682e259774\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"7160fca1-5597-47de-86a7-53129f0b0aa9\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"61ef3ff3-336b-43b2-889e-a7f4ec2e8ede\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"6207b4a4-ac5a-4136-9984-ff6ace51783a\",\"type\":\"CDSView\"}},\"id\":\"14ef66fd-2839-4ed9-a4e2-6b7ca2ec7d46\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"ticker\":null},\"id\":\"c985dc6e-5a14-4bad-9445-f48584840d68\",\"type\":\"LogTickFormatter\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"Throughput\",\"@throughput\"],[\"Batch size\",\"@batch_size\"],[\"Device memory\",\"@device_mem MB\"]]},\"id\":\"9230f545-b5a1-47b0-b2db-1d2178e925f3\",\"type\":\"HoverTool\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"61ef3ff3-336b-43b2-889e-a7f4ec2e8ede\",\"type\":\"Circle\"},{\"attributes\":{\"fill_color\":{\"value\":\"steelblue\"},\"line_color\":{\"value\":\"steelblue\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"7160fca1-5597-47de-86a7-53129f0b0aa9\",\"type\":\"Circle\"},{\"attributes\":{\"ticks\":[1,2,4,8,16,32,64,128,256]},\"id\":\"96f2f05a-431b-4cc4-b038-83127a0c9887\",\"type\":\"FixedTicker\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"9230f545-b5a1-47b0-b2db-1d2178e925f3\",\"type\":\"HoverTool\"}]},\"id\":\"c05b728d-ec79-4a2e-9287-d260b2d9e08f\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"batch_size\",\"device\",\"device_mem\",\"model\",\"throughput\",\"workload\",\"size\",\"index\"],\"data\":{\"batch_size\":[1,2,4,8,16,32,64,128,256],\"device\":[\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\"],\"device_mem\":[48,74,125,655,1303,1719,2553,4235,7583],\"index\":[143,144,145,146,147,148,149,150,151],\"model\":[\"VGG-13\",\"VGG-13\",\"VGG-13\",\"VGG-13\",\"VGG-13\",\"VGG-13\",\"VGG-13\",\"VGG-13\",\"VGG-13\"],\"size\":{\"__ndarray__\":\"khD6pEVVIUAvY+OkbkUjQCOEEflWniVApZBL3dsILUCiHChKnA4wQABfwqhnrTBAY/X5FhaQMUAW9kyfJbIyQAAAAAAAADRA\",\"dtype\":\"float64\",\"shape\":[9]},\"throughput\":{\"__ndarray__\":\"+++NiPYOSkBzppaBJ6FNQHUYHY8wx1BAbhlsyLmLVUD5rjR/jhhgQCPY0nt6a2BAbUk0pMnlYUBFVv37Fq9hQLkc970jnFVA\",\"dtype\":\"float64\",\"shape\":[9]},\"workload\":[\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\"]}},\"id\":\"3d9dea4d-90fa-4160-9db7-1c810fb44d57\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"batch_size\",\"device\",\"device_mem\",\"model\",\"throughput\",\"workload\",\"size\",\"index\"],\"data\":{\"batch_size\":[1,2,4,8,16,32,64,128,256],\"device\":[\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\"],\"device_mem\":[13,18,117,125,142,113,221,392,827],\"index\":[0,1,2,3,4,5,6,7,8],\"model\":[\"AlexNet\",\"AlexNet\",\"AlexNet\",\"AlexNet\",\"AlexNet\",\"AlexNet\",\"AlexNet\",\"AlexNet\",\"AlexNet\"],\"size\":{\"__ndarray__\":\"0sFTOwb4FkA0L2sqCuIZQJXS6E+HUiVAI4QR+VaeJUDi9HW0fzAmQGJ52eKnKiVAS5xf+4MrKEBIBMSzarwqQOS+uLAfFC5A\",\"dtype\":\"float64\",\"shape\":[9]},\"throughput\":{\"__ndarray__\":\"OzaXhKf8b0DzT8eEoNh0QATIFdAFRIBAcb9W17lfiECH9FZVXuCRQIaPheKsI5dA/WdH2rpdnUB0k0GXJwyeQMuC7fKHjJ9A\",\"dtype\":\"float64\",\"shape\":[9]},\"workload\":[\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\"]}},\"id\":\"c683b4c1-dc82-4a44-b75a-92131e4cbfcc\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null},\"id\":\"c72014d3-5bc6-48aa-88b6-92430e872cfb\",\"type\":\"DataRange1d\"},{\"attributes\":{\"callback\":null},\"id\":\"b66dd53a-e007-4ae8-a733-3719998e422d\",\"type\":\"DataRange1d\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"below\":[{\"id\":\"930c73dc-ead8-45d7-9383-09abd041fd51\",\"type\":\"LogAxis\"}],\"border_fill_alpha\":{\"value\":0},\"left\":[{\"id\":\"41b3e842-7a3c-4ef5-b73f-eb401bdd8c3f\",\"type\":\"LinearAxis\"}],\"plot_height\":250,\"plot_width\":250,\"renderers\":[{\"id\":\"930c73dc-ead8-45d7-9383-09abd041fd51\",\"type\":\"LogAxis\"},{\"id\":\"77027c8f-2687-4841-b68b-03924f256f45\",\"type\":\"Grid\"},{\"id\":\"41b3e842-7a3c-4ef5-b73f-eb401bdd8c3f\",\"type\":\"LinearAxis\"},{\"id\":\"ace9eed1-0364-4b0a-ab3c-55979b380553\",\"type\":\"Grid\"},{\"id\":\"1365d769-de31-46c6-aab2-50aea88be3d2\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"45a4730c-c0ed-4965-ab50-20890d238a9b\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"399f2105-5f76-4775-a042-6953dae8ee4f\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"b8feff44-12a3-43f9-8544-8c78d9dbeabf\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"eb537404-9a32-4512-af02-a18c34ec1466\",\"type\":\"LogScale\"},\"y_range\":{\"id\":\"0b7c8fbd-3578-402c-8d37-a489dd0f9e4e\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"6b23e3ec-08ca-48a2-96e6-2a13cf393d44\",\"type\":\"LinearScale\"}},\"id\":\"43fd4ee9-b762-41ef-9549-d5a730f887ad\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"batch_size\",\"device\",\"device_mem\",\"model\",\"throughput\",\"workload\",\"size\",\"index\"],\"data\":{\"batch_size\":[1,2,4,8,16,32,64,128,256],\"device\":[\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\"],\"device_mem\":[15,25,47,125,215,431,774,1514,3026],\"index\":[108,109,110,111,112,113,114,115,116],\"model\":[\"ResNet-v2-50\",\"ResNet-v2-50\",\"ResNet-v2-50\",\"ResNet-v2-50\",\"ResNet-v2-50\",\"ResNet-v2-50\",\"ResNet-v2-50\",\"ResNet-v2-50\",\"ResNet-v2-50\"],\"size\":{\"__ndarray__\":\"wkn1ghNAGEAusGyhHtMcQPWxO9wjPSFAI4QR+VaeJUCsiTZP9wsoQDOZ5M4hKStAPPUQvDTILUAZW/NtoGQwQPvCFe9/8TFA\",\"dtype\":\"float64\",\"shape\":[9]},\"throughput\":{\"__ndarray__\":\"ZfrTKP5hTEAtkEBpbJ5UQHNdQADTilpAuwVyPba7XkAFIavtP5RiQDgGLw8GvGJA2AO5Q8L+Y0D2LPJ+1WZkQOiBcbWUjGRA\",\"dtype\":\"float64\",\"shape\":[9]},\"workload\":[\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\"]}},\"id\":\"eb97d47b-7bc2-4d9c-84cc-b2682e259774\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"num_minor_ticks\":10},\"id\":\"80430682-3f4f-4218-b6e9-93ade15fc682\",\"type\":\"LogTicker\"},{\"attributes\":{\"callback\":null},\"id\":\"ccbc2b7b-9bc8-4324-841c-1bd1edc96bab\",\"type\":\"DataRange1d\"},{\"attributes\":{\"num_minor_ticks\":10},\"id\":\"e23de69f-74e2-4c2e-b1d4-47d1b3d616d8\",\"type\":\"LogTicker\"},{\"attributes\":{\"children\":[{\"id\":\"3541a191-5b4e-4938-9edb-c0a2ed1e9844\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"id\":\"ecd083da-e2fe-42fc-a72b-a999f73826bf\",\"subtype\":\"Figure\",\"type\":\"Plot\"}]},\"id\":\"6d6605d0-6440-4f32-8001-3cc48e04be22\",\"type\":\"Row\"},{\"attributes\":{\"callback\":null},\"id\":\"87a053dc-ea23-4748-98a6-ff05e9fdc36f\",\"type\":\"DataRange1d\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"below\":[{\"id\":\"218b9685-2358-4e4d-87aa-abbf14dd6a27\",\"type\":\"LogAxis\"}],\"border_fill_alpha\":{\"value\":0},\"left\":[{\"id\":\"8860136c-8d25-4e3e-8e35-d4d128678314\",\"type\":\"LinearAxis\"}],\"plot_height\":250,\"plot_width\":250,\"renderers\":[{\"id\":\"218b9685-2358-4e4d-87aa-abbf14dd6a27\",\"type\":\"LogAxis\"},{\"id\":\"e26ab6ed-a44b-49e9-bf3c-2cfe5bac8b10\",\"type\":\"Grid\"},{\"id\":\"8860136c-8d25-4e3e-8e35-d4d128678314\",\"type\":\"LinearAxis\"},{\"id\":\"daeb93a2-6130-487d-b233-0fbfb2d49f50\",\"type\":\"Grid\"},{\"id\":\"44056001-8026-4d9b-9be9-1834ec7b5d34\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"260ce5f7-2f4d-40ed-a56f-bed931cbd55f\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"c05b728d-ec79-4a2e-9287-d260b2d9e08f\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"c72014d3-5bc6-48aa-88b6-92430e872cfb\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"49e3b276-53ef-4f2b-9661-c4392c2db86e\",\"type\":\"LogScale\"},\"y_range\":{\"id\":\"ccbc2b7b-9bc8-4324-841c-1bd1edc96bab\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"c4691f89-c7fa-4a99-85c4-b96a6579eae8\",\"type\":\"LinearScale\"}},\"id\":\"69bf5fe7-3d2a-469b-877a-92ecb631163b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"c4691f89-c7fa-4a99-85c4-b96a6579eae8\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"7ab3395a-e1bd-49d8-a041-869a756260a8\",\"type\":\"LinearScale\"},{\"attributes\":{\"grid_line_color\":{\"value\":null},\"plot\":{\"id\":\"69bf5fe7-3d2a-469b-877a-92ecb631163b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"80430682-3f4f-4218-b6e9-93ade15fc682\",\"type\":\"LogTicker\"}},\"id\":\"e26ab6ed-a44b-49e9-bf3c-2cfe5bac8b10\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"c683b4c1-dc82-4a44-b75a-92131e4cbfcc\",\"type\":\"ColumnDataSource\"}},\"id\":\"cab81aff-9cea-4770-9bfe-e703f501ad6d\",\"type\":\"CDSView\"},{\"attributes\":{\"grid_line_color\":{\"value\":null},\"plot\":{\"id\":\"ecd083da-e2fe-42fc-a72b-a999f73826bf\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"e23de69f-74e2-4c2e-b1d4-47d1b3d616d8\",\"type\":\"LogTicker\"}},\"id\":\"efd29f58-5738-4a4e-a011-b38ee20e6774\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"49e3b276-53ef-4f2b-9661-c4392c2db86e\",\"type\":\"LogScale\"},{\"attributes\":{},\"id\":\"0d20ea4d-4e2d-4b44-bbb9-5ccc92ec08b5\",\"type\":\"LogScale\"},{\"attributes\":{\"axis_label\":\"batch size\",\"formatter\":{\"id\":\"ce54dfab-6493-4500-afbd-3e5d04610194\",\"type\":\"LogTickFormatter\"},\"plot\":{\"id\":\"69bf5fe7-3d2a-469b-877a-92ecb631163b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"185e8efb-3f71-4b81-8065-e5eec1c508d9\",\"type\":\"FixedTicker\"}},\"id\":\"218b9685-2358-4e4d-87aa-abbf14dd6a27\",\"type\":\"LogAxis\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"Throughput\",\"@throughput\"],[\"Batch size\",\"@batch_size\"],[\"Device memory\",\"@device_mem MB\"]]},\"id\":\"448237ce-f3fa-4fbf-97b0-e0c22a9a9839\",\"type\":\"HoverTool\"},{\"attributes\":{\"axis_label\":\"batch size\",\"formatter\":{\"id\":\"24ae1261-d10a-4e5c-aee3-43f722a8c032\",\"type\":\"LogTickFormatter\"},\"plot\":{\"id\":\"ecd083da-e2fe-42fc-a72b-a999f73826bf\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"aee53fae-2085-4d81-967f-edc422d94ead\",\"type\":\"FixedTicker\"}},\"id\":\"e6aeff05-4e32-48d9-946f-31c956b28aa1\",\"type\":\"LogAxis\"},{\"attributes\":{\"axis_label\":\"#examples / sec\",\"formatter\":{\"id\":\"625a23c1-ef2c-43fd-a499-d33d1b9b4de5\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"69bf5fe7-3d2a-469b-877a-92ecb631163b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"cf8e27fd-41b4-4c94-8c44-393552bc44b9\",\"type\":\"BasicTicker\"}},\"id\":\"8860136c-8d25-4e3e-8e35-d4d128678314\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"7ff42c42-b897-43ed-b02b-408becaffe34\",\"type\":\"LinearScale\"},{\"attributes\":{\"axis_label\":\"#examples / sec\",\"formatter\":{\"id\":\"6b593e7f-1ecc-4056-9278-2002786f54e4\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"ecd083da-e2fe-42fc-a72b-a999f73826bf\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"7d9d8cec-a94a-4060-bd62-5078d8d5434a\",\"type\":\"BasicTicker\"}},\"id\":\"c9ca07a7-e90f-4474-be56-caff51d531da\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"cf8e27fd-41b4-4c94-8c44-393552bc44b9\",\"type\":\"BasicTicker\"},{\"attributes\":{\"axis_label\":\"batch size\",\"formatter\":{\"id\":\"c985dc6e-5a14-4bad-9445-f48584840d68\",\"type\":\"LogTickFormatter\"},\"plot\":{\"id\":\"782060d5-6a60-41ba-b104-a9ba3dc1e45b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"96f2f05a-431b-4cc4-b038-83127a0c9887\",\"type\":\"FixedTicker\"}},\"id\":\"418fe494-98fd-4bdc-bc6a-d4f8774dfc0c\",\"type\":\"LogAxis\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"69bf5fe7-3d2a-469b-877a-92ecb631163b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"cf8e27fd-41b4-4c94-8c44-393552bc44b9\",\"type\":\"BasicTicker\"}},\"id\":\"daeb93a2-6130-487d-b233-0fbfb2d49f50\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"7d9d8cec-a94a-4060-bd62-5078d8d5434a\",\"type\":\"BasicTicker\"},{\"attributes\":{\"children\":[{\"id\":\"69bf5fe7-3d2a-469b-877a-92ecb631163b\",\"subtype\":\"Figure\",\"type\":\"Plot\"}]},\"id\":\"339791a7-a460-4116-b5f0-fcf17b94866c\",\"type\":\"Row\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"ecd083da-e2fe-42fc-a72b-a999f73826bf\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"7d9d8cec-a94a-4060-bd62-5078d8d5434a\",\"type\":\"BasicTicker\"}},\"id\":\"f75df8b5-1db4-48c7-8d22-bd9821af6056\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null},\"id\":\"1fa1b96c-4b6f-4887-8d91-69d9581b2d16\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"625a23c1-ef2c-43fd-a499-d33d1b9b4de5\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{},\"id\":\"52fcf2fa-e4a8-417e-958d-4b1c3a17a506\",\"type\":\"LogScale\"},{\"attributes\":{},\"id\":\"6b593e7f-1ecc-4056-9278-2002786f54e4\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"batch_size\",\"device\",\"device_mem\",\"model\",\"throughput\",\"workload\",\"size\",\"index\"],\"data\":{\"batch_size\":[1,2,4,8,16,32,64,128,256],\"device\":[\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\"],\"device_mem\":[11,21,40,80,156,343,650,1280,2544],\"index\":[36,37,38,39,40,41,42,43,44],\"model\":[\"DensetNet-201\",\"DensetNet-201\",\"DensetNet-201\",\"DensetNet-201\",\"DensetNet-201\",\"DensetNet-201\",\"DensetNet-201\",\"DensetNet-201\",\"DensetNet-201\"],\"size\":{\"__ndarray__\":\"47bOkA95FUAmBB89bEMbQNkdP1FKhCBAyMM1NMueI0Cdpbb8RpwmQLgbUBBcIypA795WURMALUDDLQhgZwQwQFoZu/UPjjFA\",\"dtype\":\"float64\",\"shape\":[9]},\"throughput\":{\"__ndarray__\":\"3M5mLaO+MUCaECfzN4Y+QKuwxHuIp0dAwXlv6zZNUEDE/tWgv5tTQPnPhQTzVFVAmMX+E1rcVkCKKOgc6dlXQO0fq5TiQlhA\",\"dtype\":\"float64\",\"shape\":[9]},\"workload\":[\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\"]}},\"id\":\"f4fdc3dc-655b-409f-a8cd-de9437abb141\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"Throughput\",\"@throughput\"],[\"Batch size\",\"@batch_size\"],[\"Device memory\",\"@device_mem MB\"]]},\"id\":\"0c94c8a1-f0d6-4ccc-8b10-d3b5a1809b20\",\"type\":\"HoverTool\"},{\"attributes\":{\"children\":[{\"id\":\"6d6605d0-6440-4f32-8001-3cc48e04be22\",\"type\":\"Row\"},{\"id\":\"6f4f3dc3-e10f-4c0f-b83b-de0885fbc395\",\"type\":\"Row\"},{\"id\":\"339791a7-a460-4116-b5f0-fcf17b94866c\",\"type\":\"Row\"}]},\"id\":\"4c0606f8-3091-4859-a4d8-b8b04025576f\",\"type\":\"Column\"},{\"attributes\":{\"plot\":null,\"text\":\"VGG-13 @ Tesla K80\"},\"id\":\"260ce5f7-2f4d-40ed-a56f-bed931cbd55f\",\"type\":\"Title\"},{\"attributes\":{\"fill_color\":{\"value\":\"steelblue\"},\"line_color\":{\"value\":\"steelblue\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"480fe84e-15bf-4419-89a6-9eb60f8c033b\",\"type\":\"Circle\"},{\"attributes\":{\"ticks\":[1,2,4,8,16,32,64,128,256]},\"id\":\"56756522-b282-408c-aa7d-aef54f3165bb\",\"type\":\"FixedTicker\"},{\"attributes\":{\"plot\":null,\"text\":\"DensetNet-201 @ Tesla K80\"},\"id\":\"6d21750e-97f5-4fb1-bfeb-d620dc2a5337\",\"type\":\"Title\"},{\"attributes\":{\"source\":{\"id\":\"3d9dea4d-90fa-4160-9db7-1c810fb44d57\",\"type\":\"ColumnDataSource\"}},\"id\":\"7bbe8ada-5b21-4904-8e09-5c400d005697\",\"type\":\"CDSView\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"below\":[{\"id\":\"a2f1a0bd-a429-4972-8d0d-610052b048b6\",\"type\":\"LogAxis\"}],\"border_fill_alpha\":{\"value\":0},\"left\":[{\"id\":\"aa11d5c4-2304-495e-844a-620297377e0e\",\"type\":\"LinearAxis\"}],\"plot_height\":250,\"plot_width\":250,\"renderers\":[{\"id\":\"a2f1a0bd-a429-4972-8d0d-610052b048b6\",\"type\":\"LogAxis\"},{\"id\":\"e9fe79fd-5401-4cc0-9734-ca8113d88cfc\",\"type\":\"Grid\"},{\"id\":\"aa11d5c4-2304-495e-844a-620297377e0e\",\"type\":\"LinearAxis\"},{\"id\":\"05f56e47-127c-44e9-af52-22405e285aa1\",\"type\":\"Grid\"},{\"id\":\"ce37c20d-3582-44b4-88a2-3084385c677e\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"d7e157a1-7b2c-4462-83bc-ec3dd54f9276\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"5e67f59a-27e0-4b43-8b54-12d4a0c5df27\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"a436b7e0-2990-413e-a0e7-7563bbd7c9e3\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"52fcf2fa-e4a8-417e-958d-4b1c3a17a506\",\"type\":\"LogScale\"},\"y_range\":{\"id\":\"db64644b-3c47-479c-9a34-3f830e52ecc2\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"7ff42c42-b897-43ed-b02b-408becaffe34\",\"type\":\"LinearScale\"}},\"id\":\"3541a191-5b4e-4938-9edb-c0a2ed1e9844\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"data_source\":{\"id\":\"3d9dea4d-90fa-4160-9db7-1c810fb44d57\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"aecd208e-054f-4216-9b6a-3d62c5f8f115\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"25e4bc25-a893-4330-b568-bb32c8b1ec71\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"7bbe8ada-5b21-4904-8e09-5c400d005697\",\"type\":\"CDSView\"}},\"id\":\"44056001-8026-4d9b-9be9-1834ec7b5d34\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"f4fdc3dc-655b-409f-a8cd-de9437abb141\",\"type\":\"ColumnDataSource\"}},\"id\":\"792ddaef-a816-4a47-98e9-2d6f71a5a78b\",\"type\":\"CDSView\"},{\"attributes\":{\"data_source\":{\"id\":\"f4fdc3dc-655b-409f-a8cd-de9437abb141\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"0035df1d-ae45-4167-b6c5-36fbe07e6ad7\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"d3650592-ace2-4ba6-9f4b-f555105e9a20\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"792ddaef-a816-4a47-98e9-2d6f71a5a78b\",\"type\":\"CDSView\"}},\"id\":\"31cffc26-a3aa-409a-a6a7-9e3503b447a4\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"ticker\":null},\"id\":\"ce54dfab-6493-4500-afbd-3e5d04610194\",\"type\":\"LogTickFormatter\"},{\"attributes\":{\"ticker\":null},\"id\":\"24ae1261-d10a-4e5c-aee3-43f722a8c032\",\"type\":\"LogTickFormatter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"25e4bc25-a893-4330-b568-bb32c8b1ec71\",\"type\":\"Circle\"},{\"attributes\":{\"data_source\":{\"id\":\"c683b4c1-dc82-4a44-b75a-92131e4cbfcc\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"480fe84e-15bf-4419-89a6-9eb60f8c033b\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"d72d805b-f071-4ee9-8519-0026e9076d6f\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"cab81aff-9cea-4770-9bfe-e703f501ad6d\",\"type\":\"CDSView\"}},\"id\":\"ce37c20d-3582-44b4-88a2-3084385c677e\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"d3650592-ace2-4ba6-9f4b-f555105e9a20\",\"type\":\"Circle\"},{\"attributes\":{\"fill_color\":{\"value\":\"steelblue\"},\"line_color\":{\"value\":\"steelblue\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"aecd208e-054f-4216-9b6a-3d62c5f8f115\",\"type\":\"Circle\"},{\"attributes\":{\"fill_color\":{\"value\":\"steelblue\"},\"line_color\":{\"value\":\"steelblue\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"0035df1d-ae45-4167-b6c5-36fbe07e6ad7\",\"type\":\"Circle\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"448237ce-f3fa-4fbf-97b0-e0c22a9a9839\",\"type\":\"HoverTool\"}]},\"id\":\"5e67f59a-27e0-4b43-8b54-12d4a0c5df27\",\"type\":\"Toolbar\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"below\":[{\"id\":\"e6aeff05-4e32-48d9-946f-31c956b28aa1\",\"type\":\"LogAxis\"}],\"border_fill_alpha\":{\"value\":0},\"left\":[{\"id\":\"c9ca07a7-e90f-4474-be56-caff51d531da\",\"type\":\"LinearAxis\"}],\"plot_height\":250,\"plot_width\":250,\"renderers\":[{\"id\":\"e6aeff05-4e32-48d9-946f-31c956b28aa1\",\"type\":\"LogAxis\"},{\"id\":\"efd29f58-5738-4a4e-a011-b38ee20e6774\",\"type\":\"Grid\"},{\"id\":\"c9ca07a7-e90f-4474-be56-caff51d531da\",\"type\":\"LinearAxis\"},{\"id\":\"f75df8b5-1db4-48c7-8d22-bd9821af6056\",\"type\":\"Grid\"},{\"id\":\"31cffc26-a3aa-409a-a6a7-9e3503b447a4\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"6d21750e-97f5-4fb1-bfeb-d620dc2a5337\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"81a2f403-f0f2-49d6-8be0-c53cfd27f163\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"b66dd53a-e007-4ae8-a733-3719998e422d\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"0d20ea4d-4e2d-4b44-bbb9-5ccc92ec08b5\",\"type\":\"LogScale\"},\"y_range\":{\"id\":\"87a053dc-ea23-4748-98a6-ff05e9fdc36f\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"7ab3395a-e1bd-49d8-a041-869a756260a8\",\"type\":\"LinearScale\"}},\"id\":\"ecd083da-e2fe-42fc-a72b-a999f73826bf\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"ticks\":[1,2,4,8,16,32,64,128,256]},\"id\":\"aee53fae-2085-4d81-967f-edc422d94ead\",\"type\":\"FixedTicker\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"3e4b5fe9-d893-4ea7-b822-0ed6ae3d9688\",\"type\":\"HoverTool\"}]},\"id\":\"399f2105-5f76-4775-a042-6953dae8ee4f\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"batch_size\",\"device\",\"device_mem\",\"model\",\"throughput\",\"workload\",\"size\",\"index\"],\"data\":{\"batch_size\":[1,2,4,8,16,32,64,128,256],\"device\":[\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\",\"Tesla K80\"],\"device_mem\":[13,22,35,107,179,358,628,1185,2303],\"index\":[72,73,74,75,76,77,78,79,80],\"model\":[\"ResNet-v1-34\",\"ResNet-v1-34\",\"ResNet-v1-34\",\"ResNet-v1-34\",\"ResNet-v1-34\",\"ResNet-v1-34\",\"ResNet-v1-34\",\"ResNet-v1-34\",\"ResNet-v1-34\"],\"size\":{\"__ndarray__\":\"0sFTOwb4FkDCArxWEa4bQJFqllt31h9Az20cbh7sJEBoHb/k6jknQFjDtcdrVCpAsi9mvpvYLECnSSLrabAvQDjijfMFVTFA\",\"dtype\":\"float64\",\"shape\":[9]},\"throughput\":{\"__ndarray__\":\"1FDv585bU0C5Up02TutbQBvDt5fCimJAtdvtSftaZkC6FNt3xmhxQOrQVjJm3XJAo/fHTIbpdUBVcP/yZGV3QBco9qMgeXdA\",\"dtype\":\"float64\",\"shape\":[9]},\"workload\":[\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\"]}},\"id\":\"70dde195-c8d7-4806-aae7-9adde133bb40\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"callback\":null},\"id\":\"b8feff44-12a3-43f9-8544-8c78d9dbeabf\",\"type\":\"DataRange1d\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"below\":[{\"id\":\"418fe494-98fd-4bdc-bc6a-d4f8774dfc0c\",\"type\":\"LogAxis\"}],\"border_fill_alpha\":{\"value\":0},\"left\":[{\"id\":\"512d66a8-a25e-4634-bd6e-b1fa1820b1e2\",\"type\":\"LinearAxis\"}],\"plot_height\":250,\"plot_width\":250,\"renderers\":[{\"id\":\"418fe494-98fd-4bdc-bc6a-d4f8774dfc0c\",\"type\":\"LogAxis\"},{\"id\":\"5d6c1d0f-63d5-419f-b4e0-f129041c0bea\",\"type\":\"Grid\"},{\"id\":\"512d66a8-a25e-4634-bd6e-b1fa1820b1e2\",\"type\":\"LinearAxis\"},{\"id\":\"cd819b44-ee0e-422c-8a9a-854ba7cf06a9\",\"type\":\"Grid\"},{\"id\":\"14ef66fd-2839-4ed9-a4e2-6b7ca2ec7d46\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"8cdd5241-b80f-4bae-a7bc-a6c57a632bec\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"1a035f47-b1bb-4dd3-a2e2-6a8fb6281cd0\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1fa1b96c-4b6f-4887-8d91-69d9581b2d16\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"b7ac0403-df78-4d50-aa48-499e52726e77\",\"type\":\"LogScale\"},\"y_range\":{\"id\":\"da5d251f-5a7d-43ac-b436-5772036999a5\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"0b312b30-4043-4bfd-bf2b-0baebaa48fcf\",\"type\":\"LinearScale\"}},\"id\":\"782060d5-6a60-41ba-b104-a9ba3dc1e45b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"ticks\":[1,2,4,8,16,32,64,128,256]},\"id\":\"185e8efb-3f71-4b81-8065-e5eec1c508d9\",\"type\":\"FixedTicker\"},{\"attributes\":{\"num_minor_ticks\":10},\"id\":\"4aa15cec-0a12-418a-a8f0-5988c0d921fb\",\"type\":\"LogTicker\"},{\"attributes\":{\"callback\":null},\"id\":\"0b7c8fbd-3578-402c-8d37-a489dd0f9e4e\",\"type\":\"DataRange1d\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"42176ec0-2426-40dc-a7b0-58f8a0e5270e\",\"type\":\"HoverTool\"}]},\"id\":\"1a035f47-b1bb-4dd3-a2e2-6a8fb6281cd0\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"6b23e3ec-08ca-48a2-96e6-2a13cf393d44\",\"type\":\"LinearScale\"},{\"attributes\":{\"grid_line_color\":{\"value\":null},\"plot\":{\"id\":\"43fd4ee9-b762-41ef-9549-d5a730f887ad\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"4aa15cec-0a12-418a-a8f0-5988c0d921fb\",\"type\":\"LogTicker\"}},\"id\":\"77027c8f-2687-4841-b68b-03924f256f45\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"eb537404-9a32-4512-af02-a18c34ec1466\",\"type\":\"LogScale\"},{\"attributes\":{\"axis_label\":\"batch size\",\"formatter\":{\"id\":\"d3ef1175-3125-4c6a-9406-6461daea18c6\",\"type\":\"LogTickFormatter\"},\"plot\":{\"id\":\"43fd4ee9-b762-41ef-9549-d5a730f887ad\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"56756522-b282-408c-aa7d-aef54f3165bb\",\"type\":\"FixedTicker\"}},\"id\":\"930c73dc-ead8-45d7-9383-09abd041fd51\",\"type\":\"LogAxis\"},{\"attributes\":{\"axis_label\":\"#examples / sec\",\"formatter\":{\"id\":\"0e62db6d-e0e4-4096-881f-7c4ccfc0fbc5\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"43fd4ee9-b762-41ef-9549-d5a730f887ad\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"9db87257-dae3-4f4c-a839-2312d4d17d37\",\"type\":\"BasicTicker\"}},\"id\":\"41b3e842-7a3c-4ef5-b73f-eb401bdd8c3f\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"9db87257-dae3-4f4c-a839-2312d4d17d37\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"43fd4ee9-b762-41ef-9549-d5a730f887ad\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"9db87257-dae3-4f4c-a839-2312d4d17d37\",\"type\":\"BasicTicker\"}},\"id\":\"ace9eed1-0364-4b0a-ab3c-55979b380553\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"aca7420c-4880-426c-94da-060b0f5728eb\",\"type\":\"BasicTicker\"},{\"attributes\":{\"num_minor_ticks\":10},\"id\":\"33484252-8ab2-477c-bc1c-c14cf049070a\",\"type\":\"LogTicker\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"Throughput\",\"@throughput\"],[\"Batch size\",\"@batch_size\"],[\"Device memory\",\"@device_mem MB\"]]},\"id\":\"3e4b5fe9-d893-4ea7-b822-0ed6ae3d9688\",\"type\":\"HoverTool\"},{\"attributes\":{\"callback\":null},\"id\":\"a436b7e0-2990-413e-a0e7-7563bbd7c9e3\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"0e62db6d-e0e4-4096-881f-7c4ccfc0fbc5\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null},\"id\":\"da5d251f-5a7d-43ac-b436-5772036999a5\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"b7ac0403-df78-4d50-aa48-499e52726e77\",\"type\":\"LogScale\"},{\"attributes\":{\"plot\":null,\"text\":\"ResNet-v1-34 @ Tesla K80\"},\"id\":\"45a4730c-c0ed-4965-ab50-20890d238a9b\",\"type\":\"Title\"},{\"attributes\":{\"source\":{\"id\":\"70dde195-c8d7-4806-aae7-9adde133bb40\",\"type\":\"ColumnDataSource\"}},\"id\":\"c4037c24-3e17-4635-b542-52dee429c9fd\",\"type\":\"CDSView\"},{\"attributes\":{\"data_source\":{\"id\":\"70dde195-c8d7-4806-aae7-9adde133bb40\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"9f870a80-5361-4dbf-b902-46f98d1fff8f\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"88ce0bbe-084a-43db-9e86-2bcad5148a31\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"c4037c24-3e17-4635-b542-52dee429c9fd\",\"type\":\"CDSView\"}},\"id\":\"1365d769-de31-46c6-aab2-50aea88be3d2\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"ticker\":null},\"id\":\"d3ef1175-3125-4c6a-9406-6461daea18c6\",\"type\":\"LogTickFormatter\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"88ce0bbe-084a-43db-9e86-2bcad5148a31\",\"type\":\"Circle\"},{\"attributes\":{\"fill_color\":{\"value\":\"steelblue\"},\"line_color\":{\"value\":\"steelblue\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"9f870a80-5361-4dbf-b902-46f98d1fff8f\",\"type\":\"Circle\"},{\"attributes\":{\"axis_label\":\"batch size\",\"formatter\":{\"id\":\"70b0df26-7521-4955-bd05-24a4d84cd5cf\",\"type\":\"LogTickFormatter\"},\"plot\":{\"id\":\"3541a191-5b4e-4938-9edb-c0a2ed1e9844\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"2dfe26e2-340e-4d61-82cf-d06ca198d524\",\"type\":\"FixedTicker\"}},\"id\":\"a2f1a0bd-a429-4972-8d0d-610052b048b6\",\"type\":\"LogAxis\"},{\"attributes\":{\"grid_line_color\":{\"value\":null},\"plot\":{\"id\":\"782060d5-6a60-41ba-b104-a9ba3dc1e45b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"959cc564-b067-4456-b705-57474928e39a\",\"type\":\"LogTicker\"}},\"id\":\"5d6c1d0f-63d5-419f-b4e0-f129041c0bea\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null},\"id\":\"db64644b-3c47-479c-9a34-3f830e52ecc2\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"0b312b30-4043-4bfd-bf2b-0baebaa48fcf\",\"type\":\"LinearScale\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"batch_size\"},\"y\":{\"field\":\"throughput\"}},\"id\":\"d72d805b-f071-4ee9-8519-0026e9076d6f\",\"type\":\"Circle\"},{\"attributes\":{\"axis_label\":\"#examples / sec\",\"formatter\":{\"id\":\"d5e8bced-9419-4e0a-b920-cf8df70d743b\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"782060d5-6a60-41ba-b104-a9ba3dc1e45b\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"aca7420c-4880-426c-94da-060b0f5728eb\",\"type\":\"BasicTicker\"}},\"id\":\"512d66a8-a25e-4634-bd6e-b1fa1820b1e2\",\"type\":\"LinearAxis\"}],\"root_ids\":[\"4c0606f8-3091-4859-a4d8-b8b04025576f\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.13\"}};\n",
       "  var render_items = [{\"docid\":\"1b4d8739-f3aa-475d-b0b2-5b9ed7debcdb\",\"elementid\":\"af91d39a-b87c-4801-af15-a8c8842abd2a\",\"modelid\":\"4c0606f8-3091-4859-a4d8-b8b04025576f\"}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "4c0606f8-3091-4859-a4d8-b8b04025576f"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from dlmark import plot\n",
    "from bokeh.plotting import show, output_notebook\n",
    "output_notebook()\n",
    "\n",
    "data = thr[thr.device==devices[0]]\n",
    "show(plot.batch_size_vs_throughput_grid(data, models[::4]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The throughput increases with the batch size in log scale. The device memory, as exepcted, also increases linearly with the batch size. But note that, due to the pooled memory mechanism in MXNet, the measured device memory usage might be different to the actual memory usdage.\n",
    "\n",
    "One way to measure the actual device memory usage is finding the largest batch size we can run."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "3"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div class=\"bk-root\">\n",
       "    <div class=\"bk-plotdiv\" id=\"55a071ff-3613-4e3d-9a23-6b36196c2989\"></div>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"754e575c-7ff6-40fb-a93d-a3fedd42e869\":{\"roots\":{\"references\":[{\"attributes\":{},\"id\":\"31e97d06-5694-4bec-b9c8-399277f88cfa\",\"type\":\"ResetTool\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"46e999a8-d0c7-4210-ac08-dba6ab73dfe5\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"cf3c01f0-a724-4d22-9cb6-b32fe82464e0\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"166c6ce6-9013-4ee8-9629-3830443fb719\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"callback\":null,\"start\":0},\"id\":\"ccbbedf2-05ee-4e49-a998-09c5afcd4fa2\",\"type\":\"DataRange1d\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"below\":[{\"id\":\"92944587-180c-4088-a96e-9e482347714a\",\"type\":\"CategoricalAxis\"}],\"border_fill_alpha\":{\"value\":0},\"left\":[{\"id\":\"33992b2e-3426-4450-8564-a0ad6eda4783\",\"type\":\"LinearAxis\"}],\"plot_height\":250,\"renderers\":[{\"id\":\"92944587-180c-4088-a96e-9e482347714a\",\"type\":\"CategoricalAxis\"},{\"id\":\"142e6b3c-0493-4076-aae8-f587d3e7dd2f\",\"type\":\"Grid\"},{\"id\":\"33992b2e-3426-4450-8564-a0ad6eda4783\",\"type\":\"LinearAxis\"},{\"id\":\"34cf607d-1537-4b55-9e56-e307253ea198\",\"type\":\"Grid\"},{\"id\":\"46e999a8-d0c7-4210-ac08-dba6ab73dfe5\",\"type\":\"BoxAnnotation\"},{\"id\":\"dc87d7c2-c4f6-4ae9-97a7-0599807dc458\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"85c9f5ef-cbaa-4d1c-b5b8-e88376f6604e\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"ae2e2b5e-59b3-41a2-a1a0-4703e8f9a248\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"1d84a7ed-05c2-48ee-9a38-e52318527ee5\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"4c0bc1e3-1066-4b79-9222-5bea89ae9d36\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"ccbbedf2-05ee-4e49-a998-09c5afcd4fa2\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"460520b5-ee34-4ba7-a5e5-66508b9fea8a\",\"type\":\"LinearScale\"}},\"id\":\"877e61cf-2f22-4a6f-b5c6-d6416c7eea26\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"callback\":null,\"factors\":[\"AlexNet\",\"DensetNet-121\",\"DensetNet-161\",\"DensetNet-169\",\"DensetNet-201\",\"ResNet-v1-101\",\"ResNet-v1-152\",\"ResNet-v1-18\",\"ResNet-v1-34\",\"ResNet-v1-50\",\"ResNet-v2-18\",\"ResNet-v2-34\",\"ResNet-v2-50\",\"SqueezeNet-1.0\",\"SqueezeNet-1.1\",\"VGG-11\",\"VGG-13\",\"VGG-16\",\"VGG-19\"]},\"id\":\"1d84a7ed-05c2-48ee-9a38-e52318527ee5\",\"type\":\"FactorRange\"},{\"attributes\":{\"source\":{\"id\":\"ca302c31-d394-47ad-9acc-792a00f540aa\",\"type\":\"ColumnDataSource\"}},\"id\":\"8a2c5f1c-7463-46c2-aa1c-a3f516a18368\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"30b16ecf-61a6-4a2c-84a2-da75157b712a\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"formatter\":{\"id\":\"cf3c01f0-a724-4d22-9cb6-b32fe82464e0\",\"type\":\"CategoricalTickFormatter\"},\"major_label_orientation\":0.5,\"plot\":{\"id\":\"877e61cf-2f22-4a6f-b5c6-d6416c7eea26\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"166c6ce6-9013-4ee8-9629-3830443fb719\",\"type\":\"CategoricalTicker\"}},\"id\":\"92944587-180c-4088-a96e-9e482347714a\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"formatter\":{\"id\":\"30b16ecf-61a6-4a2c-84a2-da75157b712a\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"877e61cf-2f22-4a6f-b5c6-d6416c7eea26\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"44fe105a-17c1-4c2c-9fc5-32045f7c1148\",\"type\":\"BasicTicker\"}},\"id\":\"33992b2e-3426-4450-8564-a0ad6eda4783\",\"type\":\"LinearAxis\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"877e61cf-2f22-4a6f-b5c6-d6416c7eea26\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"44fe105a-17c1-4c2c-9fc5-32045f7c1148\",\"type\":\"BasicTicker\"}},\"id\":\"34cf607d-1537-4b55-9e56-e307253ea198\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"4c0bc1e3-1066-4b79-9222-5bea89ae9d36\",\"type\":\"CategoricalScale\"},{\"attributes\":{},\"id\":\"44fe105a-17c1-4c2c-9fc5-32045f7c1148\",\"type\":\"BasicTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"batch_size\"},\"width\":{\"value\":0.9},\"x\":{\"field\":\"model\"}},\"id\":\"4cbb9808-bc94-4a37-8b17-fc058e622dca\",\"type\":\"VBar\"},{\"attributes\":{\"plot\":null,\"text\":\"Maximal batch size @ Tesla M60\"},\"id\":\"85c9f5ef-cbaa-4d1c-b5b8-e88376f6604e\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"Model\",\"@model\"],[\"Max batch size\",\"@batch_size\"]]},\"id\":\"b1b63496-c121-4168-b78b-5e443bab6be8\",\"type\":\"HoverTool\"},{\"attributes\":{\"data_source\":{\"id\":\"ca302c31-d394-47ad-9acc-792a00f540aa\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"f8712841-5e49-462b-9026-6de8e75aa092\",\"type\":\"VBar\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"4cbb9808-bc94-4a37-8b17-fc058e622dca\",\"type\":\"VBar\"},\"selection_glyph\":null,\"view\":{\"id\":\"8a2c5f1c-7463-46c2-aa1c-a3f516a18368\",\"type\":\"CDSView\"}},\"id\":\"dc87d7c2-c4f6-4ae9-97a7-0599807dc458\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"39e97d3f-c3ff-4c22-b892-e139536b4dea\",\"type\":\"PanTool\"},{\"id\":\"9a2d0c57-4da2-4026-aac5-9d67704403b4\",\"type\":\"WheelZoomTool\"},{\"id\":\"422fd064-0626-4c51-9cc8-7d37a8bf93a2\",\"type\":\"BoxZoomTool\"},{\"id\":\"37911e4c-e253-46f6-adc9-cbb7d61dd06b\",\"type\":\"SaveTool\"},{\"id\":\"31e97d06-5694-4bec-b9c8-399277f88cfa\",\"type\":\"ResetTool\"},{\"id\":\"726e87cb-c120-40cf-87b8-dc6b92e4fa5e\",\"type\":\"HelpTool\"},{\"id\":\"b1b63496-c121-4168-b78b-5e443bab6be8\",\"type\":\"HoverTool\"}]},\"id\":\"ae2e2b5e-59b3-41a2-a1a0-4703e8f9a248\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"39e97d3f-c3ff-4c22-b892-e139536b4dea\",\"type\":\"PanTool\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"white\"},\"top\":{\"field\":\"batch_size\"},\"width\":{\"value\":0.9},\"x\":{\"field\":\"model\"}},\"id\":\"f8712841-5e49-462b-9026-6de8e75aa092\",\"type\":\"VBar\"},{\"attributes\":{\"grid_line_color\":{\"value\":null},\"plot\":{\"id\":\"877e61cf-2f22-4a6f-b5c6-d6416c7eea26\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"166c6ce6-9013-4ee8-9629-3830443fb719\",\"type\":\"CategoricalTicker\"}},\"id\":\"142e6b3c-0493-4076-aae8-f587d3e7dd2f\",\"type\":\"Grid\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"batch_size\",\"device\",\"model\",\"workload\",\"index\"],\"data\":{\"batch_size\":[2066,153,81,157,150,319,356,825,812,371,825,816,350,662,820,159,91,90,99],\"device\":[\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\"],\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18],\"model\":[\"AlexNet\",\"DensetNet-121\",\"DensetNet-161\",\"DensetNet-169\",\"DensetNet-201\",\"ResNet-v1-101\",\"ResNet-v1-152\",\"ResNet-v1-18\",\"ResNet-v1-34\",\"ResNet-v1-50\",\"ResNet-v2-18\",\"ResNet-v2-34\",\"ResNet-v2-50\",\"SqueezeNet-1.0\",\"SqueezeNet-1.1\",\"VGG-11\",\"VGG-13\",\"VGG-16\",\"VGG-19\"],\"workload\":[\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\"]}},\"id\":\"ca302c31-d394-47ad-9acc-792a00f540aa\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"overlay\":{\"id\":\"46e999a8-d0c7-4210-ac08-dba6ab73dfe5\",\"type\":\"BoxAnnotation\"}},\"id\":\"422fd064-0626-4c51-9cc8-7d37a8bf93a2\",\"type\":\"BoxZoomTool\"},{\"attributes\":{},\"id\":\"460520b5-ee34-4ba7-a5e5-66508b9fea8a\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"9a2d0c57-4da2-4026-aac5-9d67704403b4\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"37911e4c-e253-46f6-adc9-cbb7d61dd06b\",\"type\":\"SaveTool\"},{\"attributes\":{},\"id\":\"726e87cb-c120-40cf-87b8-dc6b92e4fa5e\",\"type\":\"HelpTool\"}],\"root_ids\":[\"877e61cf-2f22-4a6f-b5c6-d6416c7eea26\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.13\"}};\n",
       "  var render_items = [{\"docid\":\"754e575c-7ff6-40fb-a93d-a3fedd42e869\",\"elementid\":\"55a071ff-3613-4e3d-9a23-6b36196c2989\",\"modelid\":\"877e61cf-2f22-4a6f-b5c6-d6416c7eea26\"}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "877e61cf-2f22-4a6f-b5c6-d6416c7eea26"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "bs = dm.benchmark.load_results('cnn.py__benchmark_largest_batch_size.json')    \n",
    "show(plot.max_batch_size(bs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Throughput on various hardware"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "4"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div class=\"bk-root\">\n",
       "    <div class=\"bk-plotdiv\" id=\"e783a45f-b936-47a1-b90c-2b6f7752383f\"></div>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"e96af7e0-06b4-4ea7-903b-f511be3d3219\":{\"roots\":{\"references\":[{\"attributes\":{\"overlay\":{\"id\":\"e5c9a43f-14f8-4b99-af1d-65848ef172fb\",\"type\":\"BoxAnnotation\"}},\"id\":\"f23c56b2-fd21-4c48-8959-471848270b76\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"formatter\":{\"id\":\"a581c9b3-fcb4-4fbc-8ff5-db02884248aa\",\"type\":\"CategoricalTickFormatter\"},\"plot\":{\"id\":\"b2ac2337-56b4-4c53-b15a-db87ad4246f0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"f4597aa3-047a-4d1d-a172-2ab97a66c3de\",\"type\":\"CategoricalTicker\"}},\"id\":\"2e8566ad-b507-4207-8283-05d32ee88182\",\"type\":\"CategoricalAxis\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"b2ac2337-56b4-4c53-b15a-db87ad4246f0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"e4c173e9-e3db-4845-8b1f-ae5b01420f65\",\"type\":\"BasicTicker\"}},\"id\":\"fcf05cee-527e-4816-908c-a2b1c2298135\",\"type\":\"Grid\"},{\"attributes\":{\"grid_line_color\":{\"value\":null},\"plot\":{\"id\":\"b2ac2337-56b4-4c53-b15a-db87ad4246f0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"f4597aa3-047a-4d1d-a172-2ab97a66c3de\",\"type\":\"CategoricalTicker\"}},\"id\":\"6c25b73a-855b-4a84-8fd3-e3831a56f31e\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"89f6e080-81ec-4d62-89db-6d710d29cdfc\",\"type\":\"ResetTool\"},{\"attributes\":{\"axis_label\":\"#examples/sec\",\"formatter\":{\"id\":\"dbf96a83-e4cc-4e0f-9ded-699025c8c900\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"b2ac2337-56b4-4c53-b15a-db87ad4246f0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"e4c173e9-e3db-4845-8b1f-ae5b01420f65\",\"type\":\"BasicTicker\"}},\"id\":\"6ff34b40-5556-41e5-8a93-d854c8746371\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"8f7e7600-9d3e-4c06-8a19-2ad887f9d19d\",\"type\":\"WheelZoomTool\"},{\"attributes\":{},\"id\":\"9e4e182b-9f92-40b2-a1c6-7fae6f49e337\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"95e800de-79d5-495c-a8a4-fa265ac94e43\",\"type\":\"LinearScale\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"9e4e182b-9f92-40b2-a1c6-7fae6f49e337\",\"type\":\"PanTool\"},{\"id\":\"8f7e7600-9d3e-4c06-8a19-2ad887f9d19d\",\"type\":\"WheelZoomTool\"},{\"id\":\"f23c56b2-fd21-4c48-8959-471848270b76\",\"type\":\"BoxZoomTool\"},{\"id\":\"9105d33a-c6a6-4def-9f48-4381cd9f8308\",\"type\":\"SaveTool\"},{\"id\":\"89f6e080-81ec-4d62-89db-6d710d29cdfc\",\"type\":\"ResetTool\"},{\"id\":\"97fb9145-f9b8-482e-9077-7a0f45935fc8\",\"type\":\"HelpTool\"},{\"id\":\"6e628807-c0d5-4cc3-9c03-3afa5516b1dd\",\"type\":\"HoverTool\"}]},\"id\":\"ea0e7d40-20f9-40c7-8391-c051669b4cc0\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"e4c173e9-e3db-4845-8b1f-ae5b01420f65\",\"type\":\"BasicTicker\"},{\"attributes\":{},\"id\":\"71048dab-a3b8-41c6-8085-022517a39164\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"y\"},\"width\":{\"value\":0.8},\"x\":{\"field\":\"x\"}},\"id\":\"ba4965e3-cf7c-4e9e-97ad-49d361144e54\",\"type\":\"VBar\"},{\"attributes\":{\"plot\":null,\"text\":\"Throughput vs devices @ AlexNet\"},\"id\":\"a6bacca2-c8b0-46d3-a7f1-5d5197a96487\",\"type\":\"Title\"},{\"attributes\":{\"data_source\":{\"id\":\"0b842309-8cb0-4e84-b6f4-4933f5675186\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"eeac3de6-2b9f-4731-880c-9afe16872593\",\"type\":\"VBar\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"ba4965e3-cf7c-4e9e-97ad-49d361144e54\",\"type\":\"VBar\"},\"selection_glyph\":null,\"view\":{\"id\":\"cc35d962-0240-40af-869f-d48e4d80f74c\",\"type\":\"CDSView\"}},\"id\":\"9fc71eee-c26d-4815-9684-556e5a992ba4\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"x\",\"y\"],\"data\":{\"x\":[[\"Tesla K80\",\"1\"],[\"Tesla K80\",\"2\"],[\"Tesla K80\",\"4\"],[\"Tesla K80\",\"8\"],[\"Tesla K80\",\"16\"],[\"Tesla K80\",\"32\"],[\"Tesla K80\",\"64\"],[\"Tesla K80\",\"128\"],[\"Tesla K80\",\"256\"],[\"Tesla M60\",\"1\"],[\"Tesla M60\",\"2\"],[\"Tesla M60\",\"4\"],[\"Tesla M60\",\"8\"],[\"Tesla M60\",\"16\"],[\"Tesla M60\",\"32\"],[\"Tesla M60\",\"64\"],[\"Tesla M60\",\"128\"],[\"Tesla M60\",\"256\"],[\"Tesla V100-SXM2-16GB\",\"1\"],[\"Tesla V100-SXM2-16GB\",\"2\"],[\"Tesla V100-SXM2-16GB\",\"4\"],[\"Tesla V100-SXM2-16GB\",\"8\"],[\"Tesla V100-SXM2-16GB\",\"16\"],[\"Tesla V100-SXM2-16GB\",\"32\"],[\"Tesla V100-SXM2-16GB\",\"64\"],[\"Tesla V100-SXM2-16GB\",\"128\"],[\"Tesla V100-SXM2-16GB\",\"256\"]],\"y\":{\"__ndarray__\":\"OzaXhKf8b0DzT8eEoNh0QATIFdAFRIBAcb9W17lfiECH9FZVXuCRQIaPheKsI5dA/WdH2rpdnUB0k0GXJwyeQMuC7fKHjJ9A/KkoyX8XcUDozvT3YGR5QDY6QPP1E4lAFlgSi/qPlEBNmn75JvqXQN9ROpdBMaFAGXuQZLTEpEAWZyUWl6CmQP88Nktw5KdAbdkLATTOg0A0XfcSM9qSQLKpB32qE6BAE2mQKkNgsEA6K9dGTsu5QJMqEj216sFAKfGK4OtRxkCViIrOF23IQKYv6acQpMxA\",\"dtype\":\"float64\",\"shape\":[27]}}},\"id\":\"0b842309-8cb0-4e84-b6f4-4933f5675186\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"dbf96a83-e4cc-4e0f-9ded-699025c8c900\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"y\"},\"width\":{\"value\":0.8},\"x\":{\"field\":\"x\"}},\"id\":\"eeac3de6-2b9f-4731-880c-9afe16872593\",\"type\":\"VBar\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"e5c9a43f-14f8-4b99-af1d-65848ef172fb\",\"type\":\"BoxAnnotation\"},{\"attributes\":{},\"id\":\"9105d33a-c6a6-4def-9f48-4381cd9f8308\",\"type\":\"SaveTool\"},{\"attributes\":{\"source\":{\"id\":\"0b842309-8cb0-4e84-b6f4-4933f5675186\",\"type\":\"ColumnDataSource\"}},\"id\":\"cc35d962-0240-40af-869f-d48e4d80f74c\",\"type\":\"CDSView\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"below\":[{\"id\":\"2e8566ad-b507-4207-8283-05d32ee88182\",\"type\":\"CategoricalAxis\"}],\"border_fill_alpha\":{\"value\":0},\"left\":[{\"id\":\"6ff34b40-5556-41e5-8a93-d854c8746371\",\"type\":\"LinearAxis\"}],\"plot_height\":250,\"renderers\":[{\"id\":\"2e8566ad-b507-4207-8283-05d32ee88182\",\"type\":\"CategoricalAxis\"},{\"id\":\"6c25b73a-855b-4a84-8fd3-e3831a56f31e\",\"type\":\"Grid\"},{\"id\":\"6ff34b40-5556-41e5-8a93-d854c8746371\",\"type\":\"LinearAxis\"},{\"id\":\"fcf05cee-527e-4816-908c-a2b1c2298135\",\"type\":\"Grid\"},{\"id\":\"e5c9a43f-14f8-4b99-af1d-65848ef172fb\",\"type\":\"BoxAnnotation\"},{\"id\":\"9fc71eee-c26d-4815-9684-556e5a992ba4\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"a6bacca2-c8b0-46d3-a7f1-5d5197a96487\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"ea0e7d40-20f9-40c7-8391-c051669b4cc0\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"d313ba78-3920-4539-b42c-8a60dccfe09e\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"71048dab-a3b8-41c6-8085-022517a39164\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"8b4e217f-a433-4312-b8b0-1ba3a707e91f\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"95e800de-79d5-495c-a8a4-fa265ac94e43\",\"type\":\"LinearScale\"}},\"id\":\"b2ac2337-56b4-4c53-b15a-db87ad4246f0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"97fb9145-f9b8-482e-9077-7a0f45935fc8\",\"type\":\"HelpTool\"},{\"attributes\":{\"callback\":null,\"factors\":[[\"Tesla K80\",\"1\"],[\"Tesla K80\",\"2\"],[\"Tesla K80\",\"4\"],[\"Tesla K80\",\"8\"],[\"Tesla K80\",\"16\"],[\"Tesla K80\",\"32\"],[\"Tesla K80\",\"64\"],[\"Tesla K80\",\"128\"],[\"Tesla K80\",\"256\"],[\"Tesla M60\",\"1\"],[\"Tesla M60\",\"2\"],[\"Tesla M60\",\"4\"],[\"Tesla M60\",\"8\"],[\"Tesla M60\",\"16\"],[\"Tesla M60\",\"32\"],[\"Tesla M60\",\"64\"],[\"Tesla M60\",\"128\"],[\"Tesla M60\",\"256\"],[\"Tesla V100-SXM2-16GB\",\"1\"],[\"Tesla V100-SXM2-16GB\",\"2\"],[\"Tesla V100-SXM2-16GB\",\"4\"],[\"Tesla V100-SXM2-16GB\",\"8\"],[\"Tesla V100-SXM2-16GB\",\"16\"],[\"Tesla V100-SXM2-16GB\",\"32\"],[\"Tesla V100-SXM2-16GB\",\"64\"],[\"Tesla V100-SXM2-16GB\",\"128\"],[\"Tesla V100-SXM2-16GB\",\"256\"]]},\"id\":\"d313ba78-3920-4539-b42c-8a60dccfe09e\",\"type\":\"FactorRange\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"Throughput\",\"@y\"]]},\"id\":\"6e628807-c0d5-4cc3-9c03-3afa5516b1dd\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"a581c9b3-fcb4-4fbc-8ff5-db02884248aa\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{\"callback\":null,\"start\":0},\"id\":\"8b4e217f-a433-4312-b8b0-1ba3a707e91f\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"f4597aa3-047a-4d1d-a172-2ab97a66c3de\",\"type\":\"CategoricalTicker\"}],\"root_ids\":[\"b2ac2337-56b4-4c53-b15a-db87ad4246f0\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.13\"}};\n",
       "  var render_items = [{\"docid\":\"e96af7e0-06b4-4ea7-903b-f511be3d3219\",\"elementid\":\"e783a45f-b936-47a1-b90c-2b6f7752383f\",\"modelid\":\"b2ac2337-56b4-4c53-b15a-db87ad4246f0\"}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "b2ac2337-56b4-4c53-b15a-db87ad4246f0"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show(plot.throughput_vs_device(thr[(thr.model=='AlexNet')]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "5"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div class=\"bk-root\">\n",
       "    <div class=\"bk-plotdiv\" id=\"44177ebe-9ece-4c08-bc23-d4bbb268be0b\"></div>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"3b5b8087-5334-4201-bafa-7f5f1cab8534\":{\"roots\":{\"references\":[{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"c92473d8-fc9a-4650-b53c-a92f1a68900e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"038ad4dc-b935-4336-aaca-4140a1163db3\",\"type\":\"BasicTicker\"}},\"id\":\"530fe59a-617d-4902-895c-520ae3cd6d6e\",\"type\":\"Grid\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"y\"},\"width\":{\"value\":0.8},\"x\":{\"field\":\"x\"}},\"id\":\"7ced37ee-be2e-48fa-9c19-a1014d9d9472\",\"type\":\"VBar\"},{\"attributes\":{},\"id\":\"760914ad-1704-4e34-a95c-5e3252883c5b\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"x\",\"y\"],\"data\":{\"x\":[[\"Tesla K80\",\"1\"],[\"Tesla K80\",\"2\"],[\"Tesla K80\",\"4\"],[\"Tesla K80\",\"8\"],[\"Tesla K80\",\"16\"],[\"Tesla K80\",\"32\"],[\"Tesla K80\",\"64\"],[\"Tesla K80\",\"128\"],[\"Tesla K80\",\"256\"],[\"Tesla M60\",\"1\"],[\"Tesla M60\",\"2\"],[\"Tesla M60\",\"4\"],[\"Tesla M60\",\"8\"],[\"Tesla M60\",\"16\"],[\"Tesla M60\",\"32\"],[\"Tesla M60\",\"64\"],[\"Tesla M60\",\"128\"],[\"Tesla M60\",\"256\"],[\"Tesla V100-SXM2-16GB\",\"1\"],[\"Tesla V100-SXM2-16GB\",\"2\"],[\"Tesla V100-SXM2-16GB\",\"4\"],[\"Tesla V100-SXM2-16GB\",\"8\"],[\"Tesla V100-SXM2-16GB\",\"16\"],[\"Tesla V100-SXM2-16GB\",\"32\"],[\"Tesla V100-SXM2-16GB\",\"64\"],[\"Tesla V100-SXM2-16GB\",\"128\"],[\"Tesla V100-SXM2-16GB\",\"256\"]],\"y\":{\"__ndarray__\":\"ZfrTKP5hTEAtkEBpbJ5UQHNdQADTilpAuwVyPba7XkAFIavtP5RiQDgGLw8GvGJA2AO5Q8L+Y0D2LPJ+1WZkQOiBcbWUjGRAy7rMzj21U0DFwMkE+SFfQLW3Xq4jo2RA5TAyAYKOaEC8FMqrSmtqQELrlu88Q2xAQYGFEmfTbED+NYFUCLxtQKdmEiWQBG5Atnh0WmQrXkDpTepfGi1sQJx9FTsfSnhA6DnEXVKygkCLGcU12FGKQGDnoBuqtY5A+YmZvwwykUDR+bxcb4aSQMUL4lsaQZNA\",\"dtype\":\"float64\",\"shape\":[27]}}},\"id\":\"ffc38f54-c189-4d9b-b886-dfc8e49ebac3\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"74156f77-cba0-4809-86d0-1f5b0e14a1bb\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"Throughput\",\"@y\"]]},\"id\":\"8a454943-a0c0-4bf4-95b9-afae71597a82\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"5d088dae-38cc-43cb-841b-9a7263e88d42\",\"type\":\"PanTool\"},{\"attributes\":{\"callback\":null,\"factors\":[[\"Tesla K80\",\"1\"],[\"Tesla K80\",\"2\"],[\"Tesla K80\",\"4\"],[\"Tesla K80\",\"8\"],[\"Tesla K80\",\"16\"],[\"Tesla K80\",\"32\"],[\"Tesla K80\",\"64\"],[\"Tesla K80\",\"128\"],[\"Tesla K80\",\"256\"],[\"Tesla M60\",\"1\"],[\"Tesla M60\",\"2\"],[\"Tesla M60\",\"4\"],[\"Tesla M60\",\"8\"],[\"Tesla M60\",\"16\"],[\"Tesla M60\",\"32\"],[\"Tesla M60\",\"64\"],[\"Tesla M60\",\"128\"],[\"Tesla M60\",\"256\"],[\"Tesla V100-SXM2-16GB\",\"1\"],[\"Tesla V100-SXM2-16GB\",\"2\"],[\"Tesla V100-SXM2-16GB\",\"4\"],[\"Tesla V100-SXM2-16GB\",\"8\"],[\"Tesla V100-SXM2-16GB\",\"16\"],[\"Tesla V100-SXM2-16GB\",\"32\"],[\"Tesla V100-SXM2-16GB\",\"64\"],[\"Tesla V100-SXM2-16GB\",\"128\"],[\"Tesla V100-SXM2-16GB\",\"256\"]]},\"id\":\"78c9eaa6-0f93-47c6-bea6-07761759e366\",\"type\":\"FactorRange\"},{\"attributes\":{\"data_source\":{\"id\":\"ffc38f54-c189-4d9b-b886-dfc8e49ebac3\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"683c91dc-c979-4a82-9d5f-28a41e2ae378\",\"type\":\"VBar\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"7ced37ee-be2e-48fa-9c19-a1014d9d9472\",\"type\":\"VBar\"},\"selection_glyph\":null,\"view\":{\"id\":\"0ad201ed-2010-42c0-aad9-5a6819fd53db\",\"type\":\"CDSView\"}},\"id\":\"ed5c72f4-e3d3-4b98-828a-8ec6a5920f6a\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"below\":[{\"id\":\"415cc53e-e4f3-49fa-9eda-a1d9c9a95cc2\",\"type\":\"CategoricalAxis\"}],\"border_fill_alpha\":{\"value\":0},\"left\":[{\"id\":\"eff78427-e71a-465e-94b9-204543824dc7\",\"type\":\"LinearAxis\"}],\"plot_height\":250,\"renderers\":[{\"id\":\"415cc53e-e4f3-49fa-9eda-a1d9c9a95cc2\",\"type\":\"CategoricalAxis\"},{\"id\":\"b9f16055-a3f1-4869-bb74-27c0b3b8367f\",\"type\":\"Grid\"},{\"id\":\"eff78427-e71a-465e-94b9-204543824dc7\",\"type\":\"LinearAxis\"},{\"id\":\"530fe59a-617d-4902-895c-520ae3cd6d6e\",\"type\":\"Grid\"},{\"id\":\"74156f77-cba0-4809-86d0-1f5b0e14a1bb\",\"type\":\"BoxAnnotation\"},{\"id\":\"ed5c72f4-e3d3-4b98-828a-8ec6a5920f6a\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"dfcaaa71-19ce-44af-90ab-857cd0cd1f7f\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"fb5ffd5b-9abb-4923-af8f-3a651a60d26e\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"78c9eaa6-0f93-47c6-bea6-07761759e366\",\"type\":\"FactorRange\"},\"x_scale\":{\"id\":\"a196b985-11fa-4efc-90d5-1aeab1089dac\",\"type\":\"CategoricalScale\"},\"y_range\":{\"id\":\"339075bc-b142-4022-a465-9599a3c09b2d\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"b8731076-7e66-45fd-b7ed-19832a437bc2\",\"type\":\"LinearScale\"}},\"id\":\"c92473d8-fc9a-4650-b53c-a92f1a68900e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{},\"id\":\"94e73268-9848-4e2b-8e1e-e063b208616b\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"plot\":null,\"text\":\"Throughput vs devices @ ResNet-v2-50\"},\"id\":\"dfcaaa71-19ce-44af-90ab-857cd0cd1f7f\",\"type\":\"Title\"},{\"attributes\":{\"overlay\":{\"id\":\"74156f77-cba0-4809-86d0-1f5b0e14a1bb\",\"type\":\"BoxAnnotation\"}},\"id\":\"07c6ed97-1197-45e7-a7ae-e8309f1ca8e8\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"callback\":null,\"start\":0},\"id\":\"339075bc-b142-4022-a465-9599a3c09b2d\",\"type\":\"DataRange1d\"},{\"attributes\":{},\"id\":\"c723ead7-583c-4148-916f-919459d208b6\",\"type\":\"SaveTool\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"5d088dae-38cc-43cb-841b-9a7263e88d42\",\"type\":\"PanTool\"},{\"id\":\"94e73268-9848-4e2b-8e1e-e063b208616b\",\"type\":\"WheelZoomTool\"},{\"id\":\"07c6ed97-1197-45e7-a7ae-e8309f1ca8e8\",\"type\":\"BoxZoomTool\"},{\"id\":\"c723ead7-583c-4148-916f-919459d208b6\",\"type\":\"SaveTool\"},{\"id\":\"f87a7711-b77d-47eb-8e0f-ec382f1f8518\",\"type\":\"ResetTool\"},{\"id\":\"b4532355-798e-4f88-988d-ec7a7587055a\",\"type\":\"HelpTool\"},{\"id\":\"8a454943-a0c0-4bf4-95b9-afae71597a82\",\"type\":\"HoverTool\"}]},\"id\":\"fb5ffd5b-9abb-4923-af8f-3a651a60d26e\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"f87a7711-b77d-47eb-8e0f-ec382f1f8518\",\"type\":\"ResetTool\"},{\"attributes\":{\"axis_label\":\"#examples/sec\",\"formatter\":{\"id\":\"760914ad-1704-4e34-a95c-5e3252883c5b\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"c92473d8-fc9a-4650-b53c-a92f1a68900e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"038ad4dc-b935-4336-aaca-4140a1163db3\",\"type\":\"BasicTicker\"}},\"id\":\"eff78427-e71a-465e-94b9-204543824dc7\",\"type\":\"LinearAxis\"},{\"attributes\":{},\"id\":\"b4532355-798e-4f88-988d-ec7a7587055a\",\"type\":\"HelpTool\"},{\"attributes\":{},\"id\":\"b8731076-7e66-45fd-b7ed-19832a437bc2\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"a196b985-11fa-4efc-90d5-1aeab1089dac\",\"type\":\"CategoricalScale\"},{\"attributes\":{\"grid_line_color\":{\"value\":null},\"plot\":{\"id\":\"c92473d8-fc9a-4650-b53c-a92f1a68900e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"db313ec5-337e-4da2-bba9-03141b7c0bad\",\"type\":\"CategoricalTicker\"}},\"id\":\"b9f16055-a3f1-4869-bb74-27c0b3b8367f\",\"type\":\"Grid\"},{\"attributes\":{\"source\":{\"id\":\"ffc38f54-c189-4d9b-b886-dfc8e49ebac3\",\"type\":\"ColumnDataSource\"}},\"id\":\"0ad201ed-2010-42c0-aad9-5a6819fd53db\",\"type\":\"CDSView\"},{\"attributes\":{\"formatter\":{\"id\":\"c468ae4d-e1a2-4cb5-b6db-873aed0bbe20\",\"type\":\"CategoricalTickFormatter\"},\"plot\":{\"id\":\"c92473d8-fc9a-4650-b53c-a92f1a68900e\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"db313ec5-337e-4da2-bba9-03141b7c0bad\",\"type\":\"CategoricalTicker\"}},\"id\":\"415cc53e-e4f3-49fa-9eda-a1d9c9a95cc2\",\"type\":\"CategoricalAxis\"},{\"attributes\":{},\"id\":\"c468ae4d-e1a2-4cb5-b6db-873aed0bbe20\",\"type\":\"CategoricalTickFormatter\"},{\"attributes\":{},\"id\":\"db313ec5-337e-4da2-bba9-03141b7c0bad\",\"type\":\"CategoricalTicker\"},{\"attributes\":{\"fill_color\":{\"value\":\"#1f77b4\"},\"line_color\":{\"value\":\"#1f77b4\"},\"top\":{\"field\":\"y\"},\"width\":{\"value\":0.8},\"x\":{\"field\":\"x\"}},\"id\":\"683c91dc-c979-4a82-9d5f-28a41e2ae378\",\"type\":\"VBar\"},{\"attributes\":{},\"id\":\"038ad4dc-b935-4336-aaca-4140a1163db3\",\"type\":\"BasicTicker\"}],\"root_ids\":[\"c92473d8-fc9a-4650-b53c-a92f1a68900e\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.13\"}};\n",
       "  var render_items = [{\"docid\":\"3b5b8087-5334-4201-bafa-7f5f1cab8534\",\"elementid\":\"44177ebe-9ece-4c08-bc23-d4bbb268be0b\",\"modelid\":\"c92473d8-fc9a-4650-b53c-a92f1a68900e\"}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "c92473d8-fc9a-4650-b53c-a92f1a68900e"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "show(plot.throughput_vs_device(thr[(thr.model=='ResNet-v2-50')]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction accuracy versus throughput\n",
    "\n",
    "We measture the prediction accuracy of each model using the ILSVRC 2012 validation dataset. Then plot the results together with the throughput with fixed batch size 64. We colorize models from the same family with the same color."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "attributes": {
     "classes": [],
     "id": "",
     "n": "6"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<div class=\"bk-root\">\n",
       "    <div class=\"bk-plotdiv\" id=\"6d1bb65d-28ac-4792-ae71-a6c848d63d00\"></div>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "(function(root) {\n",
       "  function embed_document(root) {\n",
       "    \n",
       "  var docs_json = {\"2d64b474-ea4b-4a39-aaa5-60af8eca0639\":{\"roots\":{\"references\":[{\"attributes\":{\"end\":1,\"factors\":[\"AlexNet\",\"DensetNet\",\"ResNet-v1\",\"ResNet-v2\",\"SqueezeNet\",\"VGG\"],\"palette\":[\"#1f77b4\",\"#ff7f0e\",\"#2ca02c\",\"#d62728\",\"#9467bd\",\"#8c564b\"]},\"id\":\"c38977a8-fd41-4823-aae6-ffcf4183e12f\",\"type\":\"CategoricalColorMapper\"},{\"attributes\":{},\"id\":\"79e8c27d-6a8e-437f-bcd2-60aa0a7af8dc\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"50040b25-93c5-4cd5-8618-eded244daaba\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"79e8c27d-6a8e-437f-bcd2-60aa0a7af8dc\",\"type\":\"BasicTicker\"}},\"id\":\"040d7940-eedc-418a-843f-c9bb0316a288\",\"type\":\"Grid\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"b6b9b742-e5a1-4d52-944e-e25bb9771ce6\",\"type\":\"HoverTool\"}]},\"id\":\"c7a9aadf-8bc4-4710-907d-7b1b895ba7db\",\"type\":\"Toolbar\"},{\"attributes\":{\"callback\":null,\"tooltips\":[[\"Model\",\"@model\"],[\"Throughput\",\"@throughput\"],[\"Accuracy\",\"@accuracy\"],[\"Device memory\",\"@device_mem MB\"]]},\"id\":\"b6b9b742-e5a1-4d52-944e-e25bb9771ce6\",\"type\":\"HoverTool\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"below\":[{\"id\":\"9219aab5-723d-4b54-8561-d068e6c1dc8c\",\"type\":\"LogAxis\"}],\"border_fill_alpha\":{\"value\":0},\"left\":[{\"id\":\"81412e1d-0a2f-4c2b-8e1d-7454ea9bb825\",\"type\":\"LinearAxis\"}],\"plot_height\":500,\"renderers\":[{\"id\":\"9219aab5-723d-4b54-8561-d068e6c1dc8c\",\"type\":\"LogAxis\"},{\"id\":\"5c52a511-2067-4328-978d-3526119474f5\",\"type\":\"Grid\"},{\"id\":\"81412e1d-0a2f-4c2b-8e1d-7454ea9bb825\",\"type\":\"LinearAxis\"},{\"id\":\"040d7940-eedc-418a-843f-c9bb0316a288\",\"type\":\"Grid\"},{\"id\":\"265abb90-d914-4704-9281-e1f5742a58bd\",\"type\":\"Legend\"},{\"id\":\"987815dc-9361-4fba-8de4-30128a02f47d\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"c2f9a166-860d-48c7-ac4f-5aeec4367744\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"c7a9aadf-8bc4-4710-907d-7b1b895ba7db\",\"type\":\"Toolbar\"},\"toolbar_location\":null,\"x_range\":{\"id\":\"ee22c3e1-4e4f-4df4-8df5-54ffd161b77e\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"6613ea1f-83bd-4ff4-9081-b8fedd04c43a\",\"type\":\"LogScale\"},\"y_range\":{\"id\":\"9b7948ef-02ac-498d-9007-1b14d86d9c0c\",\"type\":\"DataRange1d\"},\"y_scale\":{\"id\":\"5cb50020-ec9c-46d9-bc02-118ccc1d1e3d\",\"type\":\"LinearScale\"}},\"id\":\"50040b25-93c5-4cd5-8618-eded244daaba\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"background_fill_alpha\":{\"value\":0},\"items\":[{\"id\":\"de3fc364-f3df-430e-8b73-df2e98eabb14\",\"type\":\"LegendItem\"}],\"plot\":{\"id\":\"50040b25-93c5-4cd5-8618-eded244daaba\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"265abb90-d914-4704-9281-e1f5742a58bd\",\"type\":\"Legend\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"batch_size\",\"device\",\"device_mem\",\"throughput\",\"workload\",\"accuracy\",\"model_prefix\",\"size\",\"model\"],\"data\":{\"accuracy\":{\"__ndarray__\":\"fleLmzC/4T8d2TMQlQ7oP4ubML+rxeg/7RmISkd26D83YX5Xi5voP5+BqHRkT+g/a3ET5ne16D97BqLSkb3lP89AVDqy5+Y/NBCVjuyZ5z97BqLSkT3mP5jf1eImTOc/4NXiJszv5z9H9gxEpSPiP/pdLW7C/OE/bsL8rhY35T86smcgKp3lP06Y39XiJuY/GohKR/aM5j8=\",\"dtype\":\"float64\",\"shape\":[19]},\"batch_size\":[64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64],\"device\":[\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\",\"Tesla M60\"],\"device_mem\":[222,548,821,547,548,758,758,562,573,758,571,573,758,767,431,1941,2554,2554,2553],\"model\":[\"AlexNet\",\"DensetNet-121\",\"DensetNet-161\",\"DensetNet-169\",\"DensetNet-201\",\"ResNet-v1-101\",\"ResNet-v1-152\",\"ResNet-v1-18\",\"ResNet-v1-34\",\"ResNet-v1-50\",\"ResNet-v2-18\",\"ResNet-v2-34\",\"ResNet-v2-50\",\"SqueezeNet-1.0\",\"SqueezeNet-1.1\",\"VGG-11\",\"VGG-13\",\"VGG-16\",\"VGG-19\"],\"model_prefix\":[\"AlexNet\",\"DensetNet\",\"DensetNet\",\"DensetNet\",\"DensetNet\",\"ResNet-v1\",\"ResNet-v1\",\"ResNet-v1\",\"ResNet-v1\",\"ResNet-v1\",\"ResNet-v2\",\"ResNet-v2\",\"ResNet-v2\",\"SqueezeNet\",\"SqueezeNet\",\"VGG\",\"VGG\",\"VGG\",\"VGG\"],\"size\":{\"__ndarray__\":\"jsoTs4ewIUBS3gHU78orQLZRJRRXAjFA2u6tL3HEK0BS3gHU78orQF/IsnbwVzBAX8iydvBXMED8pdR1PyUsQCrFxJBrayxAX8iydvBXMEDqZVJGtl4sQCrFxJBrayxAX8iydvBXMECQJZtntHAwQCacF/napShAT6R4JTMnOkAAAAAAAAA+QAAAAAAAAD5AH2RpD3/+PUA=\",\"dtype\":\"float64\",\"shape\":[19]},\"throughput\":{\"__ndarray__\":\"GXuQZLTEpEA/Qlkvm4JqQIVYBs1FQFdAQqO8M4uMZUA7dOM48FFgQB+iLVaex15A+GfBUR19VUDjxUxJTzWKQBnDchQZKX5AEvJblIMCaUCYAEjvC0uIQNKM2Z8xS31AQYGFEmfTbEBrn+WEeZ2CQDTE7teccpFAfGPCkxurcUAMeJuv3zJmQLjggAOYamJAiBUxwO7NX0A=\",\"dtype\":\"float64\",\"shape\":[19]},\"workload\":[\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\",\"Inference\"]}},\"id\":\"55002f6a-5023-4d81-b268-8b461ce9161e\",\"type\":\"ColumnDataSource\"},{\"attributes\":{},\"id\":\"870b3b08-3c1c-43cf-85ce-faffe66275d1\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"c2f9a166-860d-48c7-ac4f-5aeec4367744\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null},\"id\":\"ee22c3e1-4e4f-4df4-8df5-54ffd161b77e\",\"type\":\"DataRange1d\"},{\"attributes\":{\"data_source\":{\"id\":\"55002f6a-5023-4d81-b268-8b461ce9161e\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"53fde05b-ac08-4cad-8329-96872c21d279\",\"type\":\"Circle\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"38ae419b-e1ff-4cf8-9975-9786464122ec\",\"type\":\"Circle\"},\"selection_glyph\":null,\"view\":{\"id\":\"5c3b8ca2-1f81-4f60-bf28-13daf5ac6709\",\"type\":\"CDSView\"}},\"id\":\"987815dc-9361-4fba-8de4-30128a02f47d\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"num_minor_ticks\":10},\"id\":\"472c232b-eafa-4f62-aa11-9ccc6ce06a36\",\"type\":\"LogTicker\"},{\"attributes\":{\"ticker\":null},\"id\":\"6070c4e0-6c1f-4213-897e-6ca9f98f5a2d\",\"type\":\"LogTickFormatter\"},{\"attributes\":{\"callback\":null},\"id\":\"9b7948ef-02ac-498d-9007-1b14d86d9c0c\",\"type\":\"DataRange1d\"},{\"attributes\":{\"source\":{\"id\":\"55002f6a-5023-4d81-b268-8b461ce9161e\",\"type\":\"ColumnDataSource\"}},\"id\":\"5c3b8ca2-1f81-4f60-bf28-13daf5ac6709\",\"type\":\"CDSView\"},{\"attributes\":{\"axis_label\":\"Accuracy\",\"formatter\":{\"id\":\"870b3b08-3c1c-43cf-85ce-faffe66275d1\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"50040b25-93c5-4cd5-8618-eded244daaba\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"79e8c27d-6a8e-437f-bcd2-60aa0a7af8dc\",\"type\":\"BasicTicker\"}},\"id\":\"81412e1d-0a2f-4c2b-8e1d-7454ea9bb825\",\"type\":\"LinearAxis\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"#1f77b4\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"#1f77b4\"},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"throughput\"},\"y\":{\"field\":\"accuracy\"}},\"id\":\"38ae419b-e1ff-4cf8-9975-9786464122ec\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"5cb50020-ec9c-46d9-bc02-118ccc1d1e3d\",\"type\":\"LinearScale\"},{\"attributes\":{\"fill_color\":{\"field\":\"model_prefix\",\"transform\":{\"id\":\"c38977a8-fd41-4823-aae6-ffcf4183e12f\",\"type\":\"CategoricalColorMapper\"}},\"line_color\":{\"field\":\"model_prefix\",\"transform\":{\"id\":\"c38977a8-fd41-4823-aae6-ffcf4183e12f\",\"type\":\"CategoricalColorMapper\"}},\"size\":{\"field\":\"size\",\"units\":\"screen\"},\"x\":{\"field\":\"throughput\"},\"y\":{\"field\":\"accuracy\"}},\"id\":\"53fde05b-ac08-4cad-8329-96872c21d279\",\"type\":\"Circle\"},{\"attributes\":{\"grid_line_color\":{\"value\":null},\"plot\":{\"id\":\"50040b25-93c5-4cd5-8618-eded244daaba\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"472c232b-eafa-4f62-aa11-9ccc6ce06a36\",\"type\":\"LogTicker\"}},\"id\":\"5c52a511-2067-4328-978d-3526119474f5\",\"type\":\"Grid\"},{\"attributes\":{\"label\":{\"field\":\"model_prefix\"},\"renderers\":[{\"id\":\"987815dc-9361-4fba-8de4-30128a02f47d\",\"type\":\"GlyphRenderer\"}]},\"id\":\"de3fc364-f3df-430e-8b73-df2e98eabb14\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"6613ea1f-83bd-4ff4-9081-b8fedd04c43a\",\"type\":\"LogScale\"},{\"attributes\":{\"axis_label\":\"#examples/sec\",\"formatter\":{\"id\":\"6070c4e0-6c1f-4213-897e-6ca9f98f5a2d\",\"type\":\"LogTickFormatter\"},\"plot\":{\"id\":\"50040b25-93c5-4cd5-8618-eded244daaba\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"472c232b-eafa-4f62-aa11-9ccc6ce06a36\",\"type\":\"LogTicker\"}},\"id\":\"9219aab5-723d-4b54-8561-d068e6c1dc8c\",\"type\":\"LogAxis\"}],\"root_ids\":[\"50040b25-93c5-4cd5-8618-eded244daaba\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.13\"}};\n",
       "  var render_items = [{\"docid\":\"2d64b474-ea4b-4a39-aaa5-60af8eca0639\",\"elementid\":\"6d1bb65d-28ac-4792-ae71-a6c848d63d00\",\"modelid\":\"50040b25-93c5-4cd5-8618-eded244daaba\"}];\n",
       "  root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n",
       "\n",
       "  }\n",
       "  if (root.Bokeh !== undefined) {\n",
       "    embed_document(root);\n",
       "  } else {\n",
       "    var attempts = 0;\n",
       "    var timer = setInterval(function(root) {\n",
       "      if (root.Bokeh !== undefined) {\n",
       "        embed_document(root);\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "      attempts++;\n",
       "      if (attempts > 100) {\n",
       "        console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n",
       "        clearInterval(timer);\n",
       "      }\n",
       "    }, 10, root)\n",
       "  }\n",
       "})(window);"
      ],
      "application/vnd.bokehjs_exec.v0+json": ""
     },
     "metadata": {
      "application/vnd.bokehjs_exec.v0+json": {
       "id": "50040b25-93c5-4cd5-8618-eded244daaba"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc = dm.benchmark.load_results('cnn*accuracy.json')\n",
    "\n",
    "data = thr[(thr.model.isin(acc.model)) &\n",
    "           (thr.batch_size.isin(acc.batch_size)) &\n",
    "           (thr.device.isin(acc.device))]\n",
    "data = data.set_index('model').join(acc[['model','accuracy']].set_index('model'))\n",
    "data['model_prefix'] = [i[:i.rfind('-')] if i.rfind('-') > 0 else i for i in data.index]\n",
    "\n",
    "show(plot.throughput_vs_accuracy(data))"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}