Init commit

Author: glebignatieff

.gitignore

@@ -0,0 +1,3 @@
+__pycache__/
+apks/
+tmp/

androguard_test.py

@@ -0,0 +1,2 @@
+from androguard.misc import AnalyzeAPK
+

apkcrawler.py

@@ -0,0 +1,91 @@
+##
+## Crawls apks from https://www.androiddrawer.com/
+##
+
+import scrapy
+import os
+import logging
+from datetime import datetime
+
+
+class ApkSpider(scrapy.Spider):
+    name = 'apk_spider'
+    start_urls = ['https://www.androiddrawer.com/']
+
+    # Setup logger
+    logger = logging.getLogger('crawl-logger')
+    formatter = logging.Formatter('%(levelname)-5s  [%(asctime)s]  %(message)s')
+    fileHandler = logging.FileHandler('apkcrawler_' + datetime.now().strftime("%Y_%m_%d__%H_%M_%S") + '.log', mode='w')
+    fileHandler.setFormatter(formatter)
+    streamHandler = logging.StreamHandler()
+    streamHandler.setFormatter(formatter)
+
+    logger.setLevel(logging.DEBUG)
+    logger.addHandler(fileHandler)
+    logger.addHandler(streamHandler)
+
+    logger.info('Starting to crawl...')
+
+    if os.path.exists('apks') is False:
+        os.mkdir('apks')
+    os.chdir('apks')
+
+    # Parses the whole category list
+    def parse(self, response):
+        for category in response.css('#categoriesContainer li'):
+            category_href = category.css('a ::attr(href)').extract_first()
+            if category_href:
+                request = scrapy.Request(
+                    url=response.urljoin(category_href),
+                    callback=self.parse_category
+                )
+                foldername = category.css('a ::text').extract_first()
+                request.meta['foldername'] = foldername
+                yield request
+
+    # Parses one category
+    def parse_category(self, response):
+        for app in response.css('a.box-click-target.animate'):
+            app_href = app.css('a ::attr(href)').extract_first()
+            if app_href:
+                yield scrapy.Request(
+                    url=response.urljoin(app_href),
+                    callback=self.parse_app,
+                    meta=response.meta
+                )
+
+    # Parses app page
+    def parse_app(self, response):
+        download_btn = response.css('a.download-btn.animate')
+        size = download_btn.css('.download-size ::text').extract_first()
+        size = int(float(size[:-3]))
+        if size <= 50:
+            href = download_btn.css('a ::attr(href)').extract_first()
+            request = scrapy.Request(
+                url=response.urljoin(href),
+                callback=self.save_apk,
+                meta=response.meta
+            )
+            filename = response.css('h1.entry-title.single-title ::text').extract_first()
+            request.meta['filename'] = filename
+            yield request
+
+    # Saves .apk to the category folder
+    def save_apk(self, response):
+        logger = logging.getLogger('crawl-logger')
+
+        foldername = response.meta['foldername']
+        filename = response.meta['filename'].replace(':', ' ') + '.apk'
+        path = os.path.join(foldername, filename)
+
+        if len(response.body) // (1 << 20) == 0 or len(response.body) == 0:
+            logger.debug("Couldn't download {} correctly :( Length: {}".format(path, len(response.body)))
+            return
+
+        if os.path.exists(foldername) is False:
+            os.mkdir(foldername)
+
+        with open(path, 'wb') as f:
+            f.write(response.body)
+
+        logger.info(path + ' Length: ' + str(len(response.body)))

common.py

@@ -0,0 +1,40 @@
+# from __future__ import division
+
+import sys
+import os
+import argparse
+
+
+# Prints text progress bar
+def update_progress(current, total):
+    amtDone = (current + 1) / total
+    sys.stdout.write("\rProgress: [{0:50s}] {1:.1f}%".format('#' * int(amtDone * 50), amtDone * 100))
+
+
+# Gets all the files from a given path
+def get_files_paths(path):
+    files_paths = []
+    for dirname, dirnames, filenames in os.walk(path):
+        for filename in filenames:
+            files_paths.append(os.path.join(os.path.abspath(dirname), filename))
+    return files_paths
+
+
+# Checks if a path is an actual file
+def is_file(filename):
+    filename = os.path.abspath(filename)
+    if not os.path.isfile(filename):
+        msg = "{0} is not a file".format(filename)
+        raise argparse.ArgumentTypeError(msg)
+    else:
+        return filename
+
+
+# Checks if a path is an actual directory
+def is_dir(dirname):
+    dirname = os.path.abspath(dirname)
+    if not os.path.isdir(dirname):
+        msg = "{0} is not a directory".format(dirname)
+        raise argparse.ArgumentTypeError(msg)
+    else:
+        return dirname

samples_picker.py

@@ -0,0 +1,42 @@
+import os
+import random
+import argparse
+from shutil import copyfile
+from common import *
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Let\'s pick some samples for you.')
+    parser.add_argument('<src path>', type=is_dir, help='Directory to pick samples from')
+    parser.add_argument('<dst path>', type=is_dir, help='Directory to put samples to')
+    parser.add_argument('<nsamples>', type=int, help='Number of samples to pick')
+    args = vars(parser.parse_args())
+
+    src_path = args['<src path>']
+    dst_path = args['<dst path>']
+    samples_num = args['<nsamples>']
+
+    print(src_path, dst_path)
+
+    files = get_files_paths(src_path)
+    if len(files) < samples_num:
+        print("Too many samples you want to pick! In total there are {} samples.".format(len(files)))
+        return
+    elif len(files) == samples_num:
+        samples = files
+    else:
+        samples = random.sample(files, samples_num)
+
+    print('Picking samples...')
+
+    for sample in samples:
+        src_sample_path = sample
+        dst_sample_path = os.path.join(dst_path, sample.split('\\')[-1])
+        copyfile(src_sample_path, dst_sample_path)
+        update_progress(samples.index(sample), len(samples))
+
+    print('\nDone!')
+
+
+if __name__ == '__main__':
+    main()