<file_sep>'''
Created on Mar 18, 2015

@author: Wschive
'''
import MySQLdb
import ast
from data.App import App
from _ast import TryExcept
import json

def connect():
    connect.db = MySQLdb.connect(host="", # your host, usually localhost
                     user="", # your username
                      passwd="", # your password
                      db="wilhelmw_thesis") # name of the data base

# you must create a Cursor object. It will let
#  you execute all the queries you need
    connect.cursor = connect.db.cursor()
    connect.db.set_character_set('utf8')
    connect.cursor.execute('SET NAMES utf8;')
    connect.cursor.execute('SET CHARACTER SET utf8;')
    connect.cursor.execute('SET character_set_connection=utf8;')
    print "DB connected and cursor created"

def saveApp(app):
    if not connect.db.open:
        connect()
    sql = """INSERT INTO `App`(`name`, `permissions`, `similar`, `logoUrl`, `packageName`) VALUES (%s,%s,%s,%s,%s)"""
    if(app):
        connect.cursor.execute(sql, (app.name,json.dumps(app.permissions),json.dumps(app.similar),app.logo, app.packageName))
        connect.db.commit()
    else:
        print "element was missing from full App"

    print "saved ", app.packageName
def getAll():
    if not connect.db.open:
        connect()
    connect.cursor.execute("SELECT * FROM App")


# print all the first cell of all the rows
    for row in connect.cursor.fetchall() :
        print row[0]

def getApp(packageName):
    if not connect.db.open:
        connect()

    sql = """SELECT * FROM App WHERE packageName =%s"""

    connect.cursor.execute(sql,[packageName])
    row = connect.cursor.fetchone()
    realName = row[0]
    permissions = ast.literal_eval(row[1])
    similar = ast.literal_eval(row[2])
    logo= row[3]
    packageName = row[4]
    app = App(realName, packageName,logo, similar, permissions, )

    return app

def getJSonApp(packageName):
    app = getApp(packageName)
    data = {}
    data["permissions"] = app.permissions
    data["logo"] = app.logo
    data["name"] = app.name
    data["packageName"] = app.packageName
    return data

def doesExist(packageName):
    if not connect.db.open:
        connect()
    #returns 1 if found, 0 if not
    connect.cursor.execute("SELECT EXISTS(SELECT 1 FROM App WHERE packageName = '%s')" % (packageName))
    result = connect.cursor.fetchone()
    return result[0] == 1



<file_sep>'''
Created on Mar 4, 2015

@author: Wschive
'''
import BaseHTTPServer, SimpleHTTPServer, ssl, requests
import json
from BaseHTTPServer import BaseHTTPRequestHandler
from urlparse import urlparse
from data.App import App
from database import db
from getApps import getApps
from risk import risk
import io
import os.path




def getIP():
    print "getting ip"
    r = requests.get("http://jsonip.com")
    tempJson = r.json()
    ip = tempJson["ip"]
    return ip.encode("utf-8")

def run(server_class=BaseHTTPServer.HTTPServer):

    ip = getIP()
    port = 8888
    handler = MyHandler
    server_address = (ip, port)
    httpd = server_class(server_address, handler)
    print "starting server at", str(ip)+":"+str(port)
    httpd.serve_forever()

class MyHandler(BaseHTTPRequestHandler):
    def do_GET(self):
        self.send_response(200, "ok")
        self.send_header('Access-Control-Allow-Origin', '*')
        self.send_header('Content-type',    'text/html')
        self.end_headers()
        print self.path
        print "hi"
        hi = urlparse(self.path)
        if(hi.path == "/replace"):
            print "replace app called"
            rawResult = replaceRequest(hi.query)#list of appnames
            self.wfile.write(createReplaceResponse(rawResult))

    def do_POST(self):
        self.send_error(502, "not implemented")

def createReplaceResponse(raw):
    list = []
    for app in raw:
        unicodeString = db.getJSonApp(app.packageName)
        list.append(unicodeString)
    return json.dumps(list)

def replaceRequest(packageName):
    app = getOrDL(packageName)
    resultList = []
    if(app.similar):
        for similar in app.similar:
            temp = getOrDL(similar)
            if(temp is not None):
                resultList.append(temp)
#         app.similar = sorted(resultList, key=getValue)#sorted list of similar apps
        app.similar = resultList
    else:
        print "similar apps list does not exist"
        return ["no similar apps"]


    return app.similar#list of suggested apps

def getOrDL(packageName):
#     app =db.getApp(name)
    print "getOrDL", packageName
    try:
        app = db.getApp(packageName)#add if published is old, redownload
    except:
        app = requestApp(packageName)
#         app = downloadApp(packageName)
        if(app is not None):
            try:
                db.saveApp(app)
            except:
                print "app exists in db", app.packageName
    return app

def getValue(app):
    fuzzy = app.analysis["FuzzyRisk"]
    return fuzzy["VALUE"]

def requestApp(packageName):

    permissionDict = getApps.getPermissions(packageName)#Dict (name: realName, permissions: permissions)
    appname = permissionDict["name"]
    logo = permissionDict["logo"]
    similar = getApps.getOnlySimilar(packageName)
    permissions = permissionDict["permissions"]

    app = App(appname, packageName, logo, similar, permissions)

    return app



def downloadApp(packageName):
#     print "downloadApp", packageName
#     appInfo = getAppInfo(packageName)
#     if(appInfo["price"] == "free"):
#         try:
#             permissions = risk.permissions(packageName)
# #             analysis = risk.run(name)
#         except:
#             getApps.run(packageName)
#             permissions = risk.permissions(packageName)
# #             analysis = risk.run(name)
#         s = appInfo["description"]
#         infoline = s.split(".",1)[0]#not used
#         similar = appInfo["recommendedApps"]#is a list
#         logo = appInfo["logo"]
#         appname = appInfo["appName"]
#         app = App(appname, packageName, logo, infoline, similar, permissions)
#         return app
#     print "app " + packageName + " is not free, will not be downloaded"
    return None


# returns json with all the info
def getAppInfo(packageName):
#
#     if(os.path.isfile("data/"+packageName+".json") ):
#         file= open("data/"+packageName+".json")
#         data = json.load(file)
#         return data
#
# #    works, but limited amount of calls
#     print "getting Appinfo for ", packageName
#     apiKey = {"key" :"9494f057c1a1a67ab30e5e7afdc6afe2"}
#     r = requests.get("http://api.playstoreapi.com/v1.1/apps/"+packageName, params = apiKey)
#     data = json.loads(r.content)
#     with open("data/" + packageName + ".json", 'w') as f:
#        f.write(json.dumps(data))
#     return data
#     print data["recommendedApps"]
    return None

<file_sep>from risk import risk
from recommend import recommender
from server import server
from database import db
import socket
import sys
from thread import *
import traceback
import json
import select
import BaseHTTPServer, SimpleHTTPServer, ssl
from data import App


def main() :

#    name = "com.hdezninirola.frequency"
    name = "no.nrk.yr"
#     name = "com.rovio.angrybirds"
#     dir = "/Users/Wschive/Desktop/"
#     name = "com.kabam.underworldandroid"
#     getApps.run("com.bitdefender.clueful")
#     risk.run("com.bitdefender.clueful")
#     recommender.recommend()

#     s = requests.Session()
# #     s.auth=('<EMAIL>', '1.9.Alpha')
#     s.post("https://accounts.google.com/ServiceLogin", {"Email":'<EMAIL>', "Passwd":'<PASSWORD>'})
#     print s
#     value = {"id":"com.squareenix.smoothieswipe"}
# #     r = s.get("https://play.google.com/store/apps/details", params = value)
# #     print r
#
#     g = s.get("http://play.google.com/store/apps/similar", params = value)
#     print g
#     print(g.url)
#     print g.content

#     works, but limited amount of calls
#     apiKey = {"key" :"9494f057c1a1a67ab30e5e7afdc6afe2"}
#     r = requests.get("http://api.playstoreapi.com/v1.1/apps/"+name, params = apiKey)
#     data = json.loads(r.content)
#     print data
#     print data["recommendedApps"]

    db.connect()

    server.run()
    print "it ran!"

if __name__ == "__main__" :
    main()<file_sep># Recommender

in the database folder:
use the SQL in dbInit to create a database with some entries
open db.py and enter the information for connecting to the database

in the getApps folder:
open config.py
enter email and password of a valid google account.
enter an androidID of a device where the above google account is connected.

Install the python dependencies:
requests
beautifulsoup4
mysqldb - http://mysql-python.blogspot.no/2012/11/is-mysqldb-hard-to-install.html

pip install is a good way of installing the first two - https://pip.pypa.io/en/latest/installing.html<file_sep>
from androguard.core.bytecodes import apk
from androguard.core import androconf
from androguard.core.analysis import risk
from androguard.core.bytecodes.apk import APK



def display_result(res) :

    for i in res :
        print "\t", i
        for j in res[i] :
            print "\t\t", j, res[i][j]

def analyze_app(filename, ri, a) :
    print filename
    return ri.with_apk(a)


def run(name,dir = ""):

    ri = risk.RiskIndicator()
    ri.add_risk_analysis( risk.RedFlags() )
    ri.add_risk_analysis( risk.FuzzyRisk() )
    ret_type = androconf.is_android(dir + name + ".apk")
    if ret_type == "APK" :
        a = apk.APK(dir + name + ".apk")
        return analyze_app( name, ri, a )
    else:
        print "ret_type was not APK"

def permissions(name, dir=""):

    a = APK(dir+name+".apk")

    #TODO: append list with score from watchdog

    return a.permissions



<file_sep># from scikits.crab import datasets
# from scikits.crab.models import MatrixPreferenceDataModel
# from scikits.crab.metrics import pearson_correlation
# from scikits.crab.similarities import UserSimilarity
# from scikits.crab.recommenders.knn import UserBasedRecommender
# from scikits.crab.recommenders.knn import ItemBasedRecommender
# from scikits.crab.similarities.basic_similarities import ItemSimilarity
# import json
# import pandas as panda
# from pandas.io.json import json_normalize
# from sklearn.metrics.pairwise import euclidean_distances
# from scikits.crab.recommenders.knn.item_strategies import ItemsNeighborhoodStrategy
# 
# 
# 
# global data
# 
# def recommend():
# 
#     movies = datasets.load_sample_movies()
#     #Build the model
#     model = MatrixPreferenceDataModel(movies.data)
#     #Build the similarity
#     similarity = UserSimilarity(model, pearson_correlation)
#     #Build the User based recommender
#     recommender = UserBasedRecommender(model, similarity, with_preference=True)
#     #Recommend items for the user 5 (Toby)
#     recommender.recommend(5)
# 
#     items_strategy = ItemsNeighborhoodStrategy()
#     similarity = ItemSimilarity(model, euclidean_distances)
#     recsys = ItemBasedRecommender(model, similarity, items_strategy)
# #     recsys.recommend('Leopoldo Pires')
# 
# #     >>> recsys.recommend('Leopoldo Pires')
# #     ['Just My Luck', 'You, Me and Dupree']
# #     >>> #Return the 2 explanations for the given recommendation.
# #     >>> recsys.recommended_because('Leopoldo Pires', 'Just My Luck',2)
# #     ['The Night Listener', 'Superman Returns']
# 
#     try:
#         with open('data/data.json') as file:
#             print "opening json"
#             list = json.load(file)
#     except:
#         print "no data available"
# 
#     global data
#     data = json_normalize(list)
#     model = MatrixPreferenceDataModel(data)
# 
# # beers = df.beer_name.unique()
# #     apps = data.Name.unique()
# #     simple_distances = []
# #     for app1 in apps:
# #         print "starting", app1
# #         for app2 in apps:
# #             if app1 != app2:
# #                 row = [app1, app2] + calculateBasicSimilarity(app1, app2)
# #                 simple_distances.append(row)
# 
#     print data.head(1)
# # ALL_FEATURES = ['RedFlags', 'FuzzyValue']
# 
# def getRedFlags(app):
#     x = 1
# 
# def getFuzzyValue(app):
# #     mask = data.Name.isin(str(app))
#     value = data.sort('FuzzyValue')
# 
# # def get_beer_reviews(beer, common_users):
# #     mask = (data.review_profilename.isin(common_users)) & (data.beer_name==beer)
# #     reviews = data[mask].sort('review_profilename')
# #     reviews = reviews[reviews.review_profilename.duplicated()==False]
# #     return reviews
# # beer_1_reviews = get_beer_reviews(beer_1, common_reviewers)
# # beer_2_reviews = get_beer_reviews(beer_2, common_reviewers)
# #
# # cols = ['beer_name', 'review_profilename', 'review_overall', 'review_aroma', 'review_palate', 'review_taste']
# # beer_2_reviews[cols].head()
# #
# def calculateBasicSimilarity(app1, app2):
#     # find common reviewers
# #     beer_1_reviewers = data[data.beer_name==beer1].review_profilename.unique()
# #     beer_2_reviewers = data[data.beer_name==beer2].review_profilename.unique()
# #     common_reviewers = set(beer_1_reviewers).intersection(beer_2_reviewers)
#     app1Value = getFuzzyValue(app1)
#     app2Value = getFuzzyValue(app2)
#     return euclidean_distances(app1Value, app2Value)

<file_sep>#!/usr/bin/python

# Do not remove

import sys
from pprint import pprint
from googleplay import GooglePlayAPI
from helpers import sizeof_fmt
import config
from bs4 import BeautifulSoup
import requests


def run(packagename):

    if (len(sys.argv) == 3):
        filename = sys.argv[2]
    else:
        filename = packagename + ".apk"

# Connect
    api = GooglePlayAPI(config.ANDROID_ID)

    api.login(config.GOOGLE_LOGIN, config.GOOGLE_PASSWORD, config.AUTH_TOKEN)

# Get the version code and the offer type from the app details
    m = api.details(packagename)
    doc = m.docV2
    vc = doc.details.appDetails.versionCode
    ot = doc.offer[0].offerType

# Download
    print "Downloading %s..." % sizeof_fmt(doc.details.appDetails.installationSize),
    data = api.download(packagename, vc, ot)
    open(filename, "wb").write(data)
    print "Done"

def getSimilarWithLogo(packageName):
    print "getting apps similar to ", packageName
    s = requests.Session()
    s.post("https://accounts.google.com/ServiceLogin", {"Email":config.GOOGLE_LOGIN, "Passwd":config.GOOGLE_PASSWORD})
    value = {"id":packageName}
# r = s.get("https://play.google.com/store/apps/details", params = value)
# print r
    g = s.get("http://play.google.com/store/apps/similar", params = value)
    soup = BeautifulSoup( g.content)
    apps = soup.find_all("a", class_ = "title")
    logos = soup.find_all("img", class_ = "cover-image")

    result = {}
    for link, logo in zip(apps, logos):
        line = link.get("href")
        image = logo.get("data-cover-small")
        result[(line.split("=",1)[1])] = image
        return result

def getOnlySimilar(packageName):
    print "getting apps similar to ", packageName

    s = requests.Session()
    s.post("https://accounts.google.com/ServiceLogin", {"Email":config.GOOGLE_LOGIN, "Passwd":config.GOOGLE_PASSWORD})
    value = {"id":packageName}
# r = s.get("https://play.google.com/store/apps/details", params = value)
# print r
    g = s.get("http://play.google.com/store/apps/similar", params = value)
    soup = BeautifulSoup( g.content)
    apps = soup.find_all("a", class_ = "title")

    result = []
    for link in apps:
        line = link.get("href")
        result.append(line.split("=",1)[1])

    return result


def getPermissions(packageName, multiple = False):

    api = GooglePlayAPI(config.ANDROID_ID)
    api.login(config.GOOGLE_LOGIN, config.GOOGLE_PASSWORD, config.AUTH_TOKEN)
    result = {}
    # Only one app
    if (not multiple):
        response = api.details(packageName)
        permissions =  []
        values = response.docV2.details.appDetails.permission._values
        permissions = [x.encode('utf-8') for x in values]
        name = response.docV2.title
        result["name"] = name.encode("utf8")
        result["permissions"] = permissions
        logo = response.docV2.image._values[0]
        image = logo.imageUrl.encode("utf8")
        result["logo"] = image
        return result

    else: # More than one app, not tested or used
        response = api.bulkDetails(packageName)
        for entry in response.entry:
            if (not not entry.ListFields()): # if the entry is not empty
                result[entry.doc.docid] = entry.doc.details.appDetails.permission
        return result
<file_sep>'''
Created on Mar 18, 2015

@author: Wschive
'''
class App:

    def __init__(self, name, packageName, logo, similar, permissions):
        self.name = name
        self.logo = logo
        self.similar = similar
        self.permissions = permissions
        self.packageName = packageName
