Merge pull request #77 from VirusTotal/jupyter-examples

Adds Jupyter examples.

.github/workflows/testing.yaml

@@ -27,7 +27,7 @@ jobs:
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-          codespell --ignore-words-list="nd" --skip="*.css,*.js,*.svg"
+          codespell --ignore-words-list="nd" --skip="*.css,*.js,*.svg,*ipynb"
       - name: Test with pytest
         run: |
           pytest

examples/jupyter/ransomware_report_usecases2.ipynb

@@ -0,0 +1,304 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "2de615a6",
+   "metadata": {},
+   "source": [
+    "# Jupyter Notebook - Ransomware report use cases 2\n",
+    "\n",
+    "Copyright © 2021 Google"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "f2c002ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import vt\n",
+    "import nest_asyncio\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a0e8fb38",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#@markdown Please, insert your VT API Key*:\n",
+    "\n",
+    "API_KEY = ''  #@param  {type: \"string\"}\n",
+    "\n",
+    "#@markdown **The API key should have Premium permissions, otherwise some of the use cases might not provide the expected results.*\n",
+    "\n",
+    "#@markdown "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "6a43b2f8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "81e7e852",
+   "metadata": {},
+   "source": [
+    "# 1. Collecting hashes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "dd8258f0",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "9426 hashes have been written to the file hashes.log\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "nh = 0\n",
+    "with vt.Client(API_KEY) as client:\n",
+    "    it = client.iterator('/intelligence/search',\n",
+    "        params={'query': 'engines:ryuk  fs:2021-01-01+ (type:peexe or type:pedll) p:10+'})\n",
+    "    with open('hashes.log','w') as f:\n",
+    "        for obj in it:\n",
+    "            if obj.id:\n",
+    "                f.write(f'{obj.id}\\n')\n",
+    "                nh += 1\n",
+    "    f.close()\n",
+    "print(f'{nh} hashes have been written to the file hashes.log\\n')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "af491db1",
+   "metadata": {},
+   "source": [
+    "# 2. Malicious contacted IOCs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "91f52171",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_malicious_IPs(file_hash):\n",
+    "    contacted_IPs = set()\n",
+    "    for ip in client.iterator('/files/{}/contacted_ips', file_hash, limit=20):\n",
+    "        stats = ip.get('last_analysis_stats')\n",
+    "        if stats and stats['malicious'] >= min_positives:\n",
+    "            contacted_IPs.add(ip.id)\n",
+    "    return contacted_IPs\n",
+    "\n",
+    "def get_malicious_Domains(file_hash):\n",
+    "    contacted_domains = set()\n",
+    "    for domain in client.iterator('/files/{}/contacted_domains', file_hash, limit=20):\n",
+    "        stats = domain.get('last_analysis_stats')\n",
+    "        if stats and stats['malicious'] >= min_positives:\n",
+    "            contacted_domains.add(domain.id)\n",
+    "    return contacted_domains"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "3fe5a2cb",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "29 IOCs have been written to the file iocs.log\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "malIOCs = set()\n",
+    "min_positives = 5 # Minimun number of positives for every IOC\n",
+    "nIOCs = 0\n",
+    "\n",
+    "with vt.Client(API_KEY) as client:\n",
+    "    it = client.iterator('/intelligence/search',\n",
+    "        params={'query': 'engines:babuk fs:2021-07-01+ (have:contacted_domains or have:contacted_ips)'})\n",
+    "    for obj in it:\n",
+    "        ips = get_malicious_IPs(obj.id)\n",
+    "        domains = get_malicious_Domains(obj.id)\n",
+    "        malIOCs = malIOCs | ips | domains\n",
+    "\n",
+    "with open('iocs.log','w') as f:\n",
+    "    for ioc in malIOCs:\n",
+    "        f.write(f'{ioc}\\n')\n",
+    "        nIOCs += 1\n",
+    "        \n",
+    "print(f'{nIOCs} IOCs have been written to the file iocs.log\\n')        \n",
+    "f.close()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "75eb0fbb",
+   "metadata": {},
+   "source": [
+    "# 3. Searching for a text in domains and URLs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a621ab7d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def search_domains(file_hash, text):\n",
+    "    domains = set()\n",
+    "    for domain in client.iterator('/files/{}/embedded_domains', file_hash, limit=20):\n",
+    "        if text in domain.id:\n",
+    "            domains.add(domain.id)\n",
+    "    for domain in client.iterator('/files/{}/itw_domains', file_hash, limit=20):\n",
+    "        if text in domain.id:\n",
+    "            domains.add(domain.id)            \n",
+    "    return domains\n",
+    "\n",
+    "def search_urls(file_hash, text):\n",
+    "    urls = set()\n",
+    "    for url in client.iterator('/files/{}/embedded_urls', file_hash, limit=20):\n",
+    "        if text in url.id:\n",
+    "            urls.add(url.id)\n",
+    "    for url in client.iterator('/files/{}/itw_urls', file_hash, limit=20):\n",
+    "        if text in url.id:\n",
+    "            urls.add(url.id)\n",
+    "    return urls"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "627139ad",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "** Results found for file hash: 1a5f649438646bfe909ac1e08bfe37f969e21b7900933d51ad2fe0c5c549a79b (pedll)\n",
+      "\n",
+      " - www.google.fr\n",
+      "\n",
+      "** Results found for file hash: af48fb4200cff418d513fe0ca841a8adb8699b266266fcf46ef44e460361cff0 (peexe)\n",
+      "\n",
+      " - www.google-analytics.com\n",
+      "\n",
+      "** Results found for file hash: 062227eaca57abaf97dd2c6d5f198fdd0d11b15cfd45eef0f0f0217d5c70aa06 (android)\n",
+      "\n",
+      " - play.google.com\n",
+      "\n",
+      " - googleads.g.doubleclick.net\n",
+      "\n",
+      " - pagead2.googlesyndication.com\n",
+      "\n",
+      " - plus.google.com\n",
+      "\n",
+      " - www.google.com\n",
+      "\n",
+      " - support.google.com\n",
+      "\n",
+      "** Results found for file hash: 569a267a4282bc694a8b39502b4e51bd3b78d449e075f4480c51eba71433b5dd (peexe)\n",
+      "\n",
+      " - www.google.de\n",
+      "\n",
+      "** Results found for file hash: 87ceec489fbfc70c2d935dc0717dba6f7d7147d06be08c661e05f7c83c3b67f0 (android)\n",
+      "\n",
+      " - play.google.com\n",
+      "\n",
+      " - googleads.g.doubleclick.net\n",
+      "\n",
+      " - pagead2.googlesyndication.com\n",
+      "\n",
+      " - plus.google.com\n",
+      "\n",
+      " - www.google.com\n",
+      "\n",
+      " - support.google.com\n",
+      "\n",
+      "** Results found for file hash: d419b2e7a6e9252b1cb3b86a6f8f107206c31cc666a74af97738f46b650ed4c5 (peexe)\n",
+      "\n",
+      " - www.googleadservices.com\n",
+      "\n",
+      " - googleads.g.doubleclick.net\n",
+      "\n",
+      " - www.googletagmanager.com\n",
+      "\n",
+      "** Results found for file hash: bcd862cc4d790ff73de8f573b3719f2c857bf0d03efa311e1f9cbf5cfa05d9ed (peexe)\n",
+      "\n",
+      " - partner.googleadservices.com\n",
+      "\n",
+      "** Results found for file hash: 147e54a51effe8a0cb42691e0e967752698b4db5883532f88daed9ba4f8b69a7 (peexe)\n",
+      "\n",
+      " - ssl.google-analytics.com\n",
+      "\n",
+      "** Results found for file hash: ac8eac1c5644ea1563783e7bdccaddbfaa6146fdcd610a7acc852fe76420352a (peexe)\n",
+      "\n",
+      " - partner.googleadservices.com\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "stxt=\"google\" # Write here any text you want to search for\n",
+    "query_string = 'engines:cerber fs:2021-06-01+ (embedded_domains: %s OR embedded_urls:%s OR itw: %s)' % (stxt, stxt, stxt)\n",
+    "\n",
+    "with vt.Client(API_KEY) as client:\n",
+    "    it = client.iterator('/intelligence/search', params={'query': query_string})\n",
+    "\n",
+    "    for obj in it:\n",
+    "        results = search_domains(obj.id, search_text) | search_urls(obj.id, search_text)\n",
+    "        \n",
+    "        if results:\n",
+    "            print(f'** Results found for file hash: {obj.id} (%s)\\n' %(obj.type_tag) )\n",
+    "            for result in results:\n",
+    "                print(f' - {result}\\n')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}