<file_sep>from google.appengine.ext import ndb

class ChasmConfig(ndb.Model):
    current = ndb.BooleanProperty()
    project_id = ndb.StringProperty()
    bucket_id = ndb.StringProperty()


<file_sep>import logging
import os
import json
import sys
import traceback
import base64
import operator
import logging
import time
import cloudstorage as gcs
import re
import uuid
from google.cloud import bigquery
from chasm_common.simplegrok import SimpleGrok
from chasm_common.chasm_gcs import chasm_gcs
from chasm_common.chasm_ds_config import ChasmConfig
from google.cloud.bigquery import QueryJobConfig
from google.appengine.api import app_identity
from google.appengine.ext import ndb
from google.appengine.api import taskqueue

class BQAlerts(ndb.Model):
    alert_name = ndb.StringProperty()
    alert_description = ndb.StringProperty()
    alert_query = ndb.TextProperty()
    alert_schedule_minutes = ndb.IntegerProperty()
   
class BQQuerySync(ndb.Model):
    locked = ndb.BooleanProperty()
    

class BQAlertData(ndb.Model):
    alert_id = ndb.StringProperty()
    alert_name = ndb.StringProperty()
    alert_result = ndb.StringProperty()
    email_sent = ndb.BooleanProperty(default=False)
    alert_status = ndb.IntegerProperty(default=0)



class chasm_bq(object):
    def __init__(self):
        logging.info("Initialize chasm bq")
        self.sync, self.conf = self.load_defaults()
        self.error = ""
        self.bytes_processed = 0
        self.bytes_billed = 0
        #if 'DEV' in project_id:
        self.client = bigquery.Client(project=self.conf.project_id)
        #    else:
        #    self.client = bigquery.Client(project=project_id)

        #self.bucket_name = os.environ.get('BUCKET_NAME',
        #                    app_identity.get_default_gcs_bucket_name())

        self.load_job = None #main load job
        self.rt_load_job = None #realtime batches
        self.chsm_gcs = chasm_gcs()


    def table_exists(self,client, table_reference):
        """Return if a table exists.

        Args:
            client (google.cloud.bigquery.client.Client):
                A client to connect to the BigQuery API.
            table_reference (google.cloud.bigquery.table.TableReference):
                A reference to the table to look for.

        Returns:
            bool: ``True`` if the table exists, ``False`` otherwise.
        """
        from google.cloud.exceptions import NotFound

        try:
            self.client.get_table(table_reference)
            return True
        except NotFound:
            return False


    # def insert_rows(self, rows_to_insert, event_type):        
    #     dataset_ref = self.client.dataset("chasm")
    #     dataset = bigquery.Dataset(dataset_ref)

    #     tables = list(self.client.list_dataset_tables(dataset_ref))
    #     source_table_ref = dataset_ref.table("events_all")
    #     source_table = self.client.get_table(source_table_ref)
    #     SOURCE_SCHEMA = list(source_table.schema)
   
    #     table_ref = dataset.table("EVENTS_{}".format(event_type))
    #     errors = []
    #     if self.table_exists(self.client,table_ref):
    #         table = bigquery.Table(table_ref, schema=SOURCE_SCHEMA)
    #         errors = self.client.create_rows(table, rows_to_insert)   
    #     else:
    #         table = bigquery.Table(table_ref, schema=SOURCE_SCHEMA)
    #         table.partitioning_type = 'DAY'
    #         table = self.client.create_table(table)
    #         errors = self.client.create_rows(table, rows_to_insert)

    #     if not errors:
    #         logging.info('Loaded {} row into {}:{}'.format(len(rows_to_insert),
    #                     "chasm", event_type))
    #     else:
    #         logging.error('Errors:{}'.format(errors))


    # def insert_batch(self,rows):
    #     #sort list by event type so we can do some batching on insert
    #     rows.sort(key=operator.itemgetter('EVENTTYPE'))
        
    #     event_type = ""
    #     temp_rows = []
    #     row_len = len(rows)
    #     i = 1

    #     for row in rows:
    #         if event_type == "":
    #             temp_rows.append(row)
    #             event_type = row["EVENTTYPE"]
    #             if i == row_len:
    #                 self.insert_rows(temp_rows, event_type)
    #                 del temp_rows[:]

    #             i = i + 1
    #         elif event_type != row["EVENTTYPE"]:
    #             self.insert_rows(temp_rows, event_type)
                
    #             del temp_rows[:]
                
    #             temp_rows.append(row)
                
    #             if i == row_len:
    #                 self.insert_rows(temp_rows, event_type)
    #                 del temp_rows[:]
                
    #             event_type = row["EVENTTYPE"]
               
    #             i = i + 1
    #         else:
    #             temp_rows.append(row)
    #             if i == row_len:
    #                 self.insert_rows(temp_rows, event_type)
    #                 del temp_rows[:]
    #             i = i + 1


    def run_query(self, query,query_type='Temp'):
        query_uuid = uuid.uuid1()
        alert_table_name =  'alert_' + str(query_uuid)
        alert_table_name = alert_table_name.replace('-','_') 
        table_ref = self.client.dataset('chasm').table(alert_table_name)
        job_config = bigquery.QueryJobConfig()
        if query_type == 'Alert':
            job_config.destination = table_ref 
        job_config.maximumBytesBilled = 1588743680
        job_config.useLegacySql = False
        self.error = ""
        
        
     

        query_job = self.client.query(query,job_config=job_config)
                                      
        results = None
        try:
            results = query_job.result(timeout=30)
            if query_job.cache_hit:
                bytes_processed = 0
                bytes_billed = 0
            else:
                bytes_processed = query_job.total_bytes_processed or 0
                bytes_billed = query_job.total_bytes_billed or 0

        except:
            ex = sys.exc_info()
            self.error = ex
            logging.error(self.error)
            return None, None, None, None, self.error, none

        result_rows = list(results)
        total_rows = results.total_rows
        schema = {
            'fields': [
                field.to_api_repr()
                for field in results.schema],
        }

        return schema, result_rows, bytes_processed, bytes_billed, self.error , alert_table_name



    def load_job_from_gcs(self):
        logging.info("Loading job from GCS")
        files = self.chsm_gcs.get_batch()
        if not files:
            return "No Logs to Process"
        
        lock = BQQuerySync.get_by_id('Default')
        if lock.locked:
            return "Query Lock Running"

        
        self.load_job = None
        initial_error = ''
        dataset_ref = self.client.dataset("chasm")
        job_config = bigquery.LoadJobConfig()
        job_config.source_format = 'NEWLINE_DELIMITED_JSON'
        job_config._properties['schemaUpdateOptions'] = ['ALLOW_FIELD_ADDITION']
        job_config.write_disposition = 'WRITE_APPEND'
        dataset = bigquery.Dataset(dataset_ref)
        
        #Main batch job
        self.load_job = self.client.load_table_from_uri(
            files,
            dataset.table('logs'),
            job_config=job_config)
        
        #Realtime batch job
        batch_uuid = uuid.uuid1()
        rt_table_name =  'realtime_' + str(batch_uuid)
        rt_table_name = rt_table_name.replace('-','_')    
        self.rt_load_job = self.client.load_table_from_uri(
            files,
            dataset.table(rt_table_name),
            job_config=job_config)
        
        try:
            logging.info("Trying initial load")
            self.load_job.result()
            self.rt_load_job.result()

            for del_file in files:
                print "Removing file {}".format(del_file)
                self.chsm_gcs.delete_log(del_file)

            return "success on initial load"

        except Exception, e:
            logging.info("Trying initial load Exception Caught : " + str(e))
            logging.info("Trying initial load Exception Caught Load Job : " + str(self.load_job.errors))
            error_type = ''
            location = ''
            initial_error = str(self.load_job.errors)
            print str(self.load_job.errors)
            if 'No such field'  in str(self.load_job.errors):
                print "No such field in except"
                error_type = 'No such feield +++ '
                error_list = self.load_job.errors
                error_type = error_list[0]
                location = error_type['location']
                error_job_config = bigquery.LoadJobConfig()
                error_job_config.autodetect = True
                error_job_config.source_format = 'NEWLINE_DELIMITED_JSON'
                error_job_config._properties['schemaUpdateOptions'] = ['ALLOW_FIELD_ADDITION']
                error_job_config.write_disposition = 'WRITE_APPEND'
                dataset = bigquery.Dataset(dataset_ref)
                
                #Main batch job
                self.load_job = self.client.load_table_from_uri(
                    location,
                    dataset.table('logs'),
                    job_config=error_job_config)
                                
                #Realtime batch job
                batch_uuid = uuid.uuid1()
                rt_table_name =  'realtime_' + str(batch_uuid)
                rt_table_name = rt_table_name.replace('-','_')   
                self.rt_load_job = self.client.load_table_from_uri(
                    location,
                    dataset.table(rt_table_name),
                    job_config=error_job_config)
        
                try:
                    self.load_job.result()
                    self.rt_load_job.result()

                    self.chsm_gcs.delete_log(location)
                    return "New field found in log {}".format(location)
                except:
                    return str(self.load_job.errors) + initial_error + location            

            elif 'No schema specified on job or table.'  in str(self.load_job.errors):
                error_job_config = bigquery.LoadJobConfig()
                error_job_config.autodetect = True
                error_job_config.source_format = 'NEWLINE_DELIMITED_JSON'
                error_job_config._properties['schemaUpdateOptions'] = ['ALLOW_FIELD_ADDITION']
                error_job_config.write_disposition = 'WRITE_APPEND'
                dataset = bigquery.Dataset(dataset_ref)

                #Main batch job
                self.load_job = self.client.load_table_from_uri(
                    files,
                    dataset.table('logs'),
                    job_config=error_job_config)
                                
                #Realtime batch job
                batch_uuid = uuid.uuid1()
                rt_table_name =  'realtime_' + str(batch_uuid)
                rt_table_name = rt_table_name.replace('-','_')    
                self.rt_load_job = self.client.load_table_from_uri(
                    files,
                    dataset.table(rt_table_name),
                    job_config=error_job_config)
        
                try:
                    self.load_job.result()
                    self.rt_load_job.result()
                    
                    #self.load_job.result()

                    for del_file in files:
                        self.chsm_gcs.delete_log(del_file)

                    return "No schema specified for job of table"

                except:
                    if 'No such field'  in str(self.load_job.errors):
                        print "No Such Field Type Found"
                        error_type = 'No such feield +++ '
                        error_list = self.load_job.errors
                        error_type = error_list[0]
                        location = error_type['location']
                        error_job_config = bigquery.LoadJobConfig()
                        error_job_config.autodetect = True
                        error_job_config.source_format = 'NEWLINE_DELIMITED_JSON'
                        error_job_config._properties['schemaUpdateOptions'] = ['ALLOW_FIELD_ADDITION']
                        error_job_config.write_disposition = 'WRITE_APPEND'
                        dataset = bigquery.Dataset(dataset_ref)
                        
                        #Main batch job
                        self.load_job = self.client.load_table_from_uri(
                            location,
                            dataset.table('logs'),
                            job_config=error_job_config)
                                
                        #Realtime batch job
                        batch_uuid = uuid.uuid1()
                        rt_table_name =  'realtime_' + str(batch_uuid)
                        rt_table_name = rt_table_name.replace('-','_')    
                        self.rt_load_job = self.client.load_table_from_uri(
                            location,
                            dataset.table(rt_table_name),
                            job_config=error_job_config)
        
                        try:
                            self.load_job.result()
                            self.rt_load_job.result()

                            self.chsm_gcs.delete_log(location)

                            return "Field added successfully"
                        except:
                                  
                            return str(self.load_job.errors) + initial_error + location
            else:
                    # return str(self.load_job.errors) + location       
                    return str(self.load_job.errors) + location
       
       

  
    
    def run_scheduled_alerts(self,scheduled_time):
        logging.info("Run scheduled alerts")
        lock = self.sync #BQQuerySync.get_by_id('Default')
        logging.info(lock.locked)
        if lock.locked:
            logging.info("Query Locked - Scheduled alerts will not run")
            return "Query Lock Running"

        #lock = BQQuerySync.get_by_id('Default')
        lock.locked = True
        lock.put()
        logging.info(str(lock))

        q = BQAlerts.query().fetch()
        #table = self.list_tables()
        try:
            for item in q:
                logging.info(item)

                schema, result_rows, bytes_processed, bytes_billed, error,  alert_table = self.run_query(item.alert_query,query_type='Alert')
                
                #logging.info(schema)
                logging.info(error)
                #logging.info(result_rows)
                
                #Save alert
                
                if result_rows:
                    logging.info("Result rows found")
                    #logging.info(result_rows)
                    alert = BQAlertData()
                    alert.alert_id=item.key.urlsafe()
                    alert.alert_name=item.alert_name
                    alert.alert_result =alert_table
                    alert.email_sent=False
                    alert.put() 
                else:
                    logging.info("No result rows found")
                    logging.info(alert_table)
                    table_ref = self.client.dataset('chasm').table(alert_table)
                    self.client.delete_table(table_ref)


            #tables = self.get_tables()
            logging.info("Display tables")
            table_query = "SELECT table_id FROM `" + self.conf.project_id +".chasm.__TABLES_SUMMARY__`"
            schema, result_rows, bytes_processed, bytes_billed, error,  alert_table = self.run_query(table_query)
               
            
            for table in result_rows:
                if 'realtime_' in table['table_id']:
                    logging.info(table['table_id'])
                    table_ref = self.client.dataset('chasm').table(table['table_id'])
                    self.client.delete_table(table_ref)


            lock = BQQuerySync.get_by_id('Default')
            lock.locked = False
            lock.put()

        except Exception, e:
            
            logging.error("Error occurred during query execution" + str(e))
            lock = BQQuerySync.get_by_id('Default')
            lock.locked = False
            lock.put()

   
    
    def get_alerts(self):
        query = BQAlertData.query(BQAlertData.alert_status == 0).fetch()
        return query

    #def get_table_data_by_name(self,table_name):
    def update_generated_alert_by_key(self, alert_status, alert_key):
        alert_key = ndb.Key(urlsafe=alert_key)
        alert = alert_key.get()
        logging.info(alert)
        logging.info("alert status")
        logging.info(alert_status)
        alert.alert_status = alert_status 
        
        key = alert.put()
        
        return key.urlsafe()

    def get_generated_alert_by_key(self,alert_key):
        alert_key = ndb.Key(urlsafe=alert_key)
        alert = alert_key.get()

        destination_table_ref = alert.alert_result
        table_ref = self.client.dataset('chasm').table(destination_table_ref)
        

        table = self.client.get_table(table_ref)
        rows  = self.client.list_rows(table)
        schema = {
            'fields': [
                field.to_api_repr()
                for field in rows.schema],
        }
        return alert,rows,schema

    def add_scheduled_alert(self,alert_name,alert_description,
                                         alert_query,alert_minutes):
        
        alert = BQAlerts(alert_name =alert_name ,
                         alert_description = alert_description,
                         alert_query = alert_query,
                         alert_schedule_minutes= alert_minutes)
        alert_key = alert.put()
        
        return alert_key.urlsafe()
    
    def update_scheduled_alert(self,alert_name,alert_description,
                                         alert_query,alert_minutes,alert_key):
        alert_key = ndb.Key(urlsafe=alert_key)
        alert = alert_key.get()
        alert.alert_name =alert_name 
        alert.alert_description = alert_description
        alert.alert_query = alert_query
        alert.alert_schedule_minutes= alert_minutes
        alert_key = alert.put()
        
        return alert_key.urlsafe()

    def delete_scheduled_alert(self,alert_key):
        
        alert_key = ndb.Key(urlsafe=alert_key)
        alert = alert_key.get()
        alert.key.delete()
    
    def get_scheduled_alert(self,alert_key):
        alert_key = ndb.Key(urlsafe=alert_key)
        alert = alert_key.get()
        return alert.alert_name, alert.alert_description,alert.alert_query
    
    def get_scheduled_alerts(self):
        query = BQAlerts.query().fetch()
        return query

    def load_defaults(self):
        """load patterns"""
        keyname = "Default"
        sync = BQQuerySync.get_or_insert(keyname, locked=False)
        conf = ChasmConfig.get_or_insert(keyname, project_id="TEMP_PROJECT", bucket_id="TEMP_BUCKET")
        return sync, conf<file_sep>import logging
import os
import json
import sys
import traceback
import base64
import operator
import logging
import uuid
from google.appengine.api import app_identity
from chasm_common.chasm_ds_config import ChasmConfig
from google.cloud import storage
from google.appengine.api import app_identity



class chasm_gcs(object):
    def __init__(self):
        logging.info("Initialize chasm bq")
        self.conf = self.load_defaults()
        self.error = ""
        self.bytes_processed = 0 
        self.bytes_billed = 0  
        
        self.client = storage.Client(project=self.conf.project_id)   
        print "bucket id"
        print self.conf.bucket_id
        self.bucket = self.client.bucket(self.conf.bucket_id)
        print "bucket"
        print self.bucket
        
  

    def delete_log(self,log_uri):
        
        try:
            print "deleting location {}".format(log_uri)
            blob_name = log_uri.rsplit('/', 1)[-1]
            print "deleting blob {}".format(blob_name)
            print "Current blob {}".format(self.conf.bucket_id)
            
            blob = self.bucket.blob(blob_name)

            blob.delete()
        except:
            print "error occured deleting {}".format(sys.exc_info()[0])   

    def get_batch(self):
        bucket = '/' + self.conf.bucket_id
        page_size = 1
        print bucket
        
        
        blobs = self.bucket.list_blobs(max_results=50)
        
        stats = ['gs://' + self.conf.bucket_id + '/' + stat.name for stat in blobs]
        
        return stats


    def write_logs_file(self,log_data):
        try:
            file_uuid = uuid.uuid1()
            filename =  'events_' + str(file_uuid)  
            json_string = ''
            blob = self.bucket.blob(filename)

            for line in log_data:
                json_string = json_string + json.dumps(line)
                json_string = json_string + '\n'
                
                
            blob.upload_from_string(
                    json_string,
                    content_type='application/json; charset=UTF-8')
               
            
            return filename
        except:
            exc_tuple = sys.exc_info()
            print str(exc_tuple)
        
    def load_defaults(self):
        """load patterns"""
        keyname = "Default"
        s = ChasmConfig.get_or_insert(keyname, project_id="TEMP_PROJECT")
        return s<file_sep>import re
import os
import requests
import time
from collections import defaultdict
from google.appengine.api import app_identity
from google.appengine.ext import ndb

from requests_toolbelt.adapters import appengine
appengine.monkeypatch()

class Pattern(ndb.Model):
    pattern_name = ndb.StringProperty()
    pattern_string = ndb.StringProperty()


class SimpleGrok(object):
    """Grok for this app return Dict of field names and data"""

    types = {
        'WORD': r'\w+',
        'NUMBER': r'\d+',
        'GREEDYDATA': r'.*',
        'WINPATH': r'(?:[A-Za-z]+:|\\)(?:\\[^\\?*]*)+',
        'POSINT': r'\b(?:[1-9][0-9]*)\b',
        'NONNEGINT': r'\b(?:[0-9]+)\b',
        'WORD': r'\b\w+\b',
        'NOTSPACE': r'\S+',
        'SPACE': r'\s*',
        'DATA': r'.*?',
        'INT': r'(?:[+-]?(?:[0-9]+))',

        # todo: extend me
        }

    def __init__(self):
        print "loading grok patterns"
        #print "grok project id {}".format(project_id)
        self.grok_patterns = defaultdict(list)
        self.temp_patterns = defaultdict(list)
        self.load_patterns()

    def grokker(self, m):
        return "(?P<" + m.group(2) + ">" + self.types[m.group(1)] + ")"

    def clean_empty(self, m):
        return self.types[m.group(1)]

    def compile(self, pat):
        temp_reg = re.sub(r'%{(\w+):EMPTY}', self.clean_empty, pat)
        reg = re.sub(r'%{(\w+):(\w+)}', self.grokker, temp_reg)
        return reg

    def get_field_dict(self, element):
        """Run through all parser and return a dict of values"""
        event_fields = {}
        event_fields["EVENTTYPE"] = "NONE"
        event_fields["FULLEVENT"] = element
        for key, list_item in self.grok_patterns.iteritems():

            for reg in list_item:
                compiled_reg = self.compile(reg)
                match_obj = re.search(compiled_reg, element)

                if match_obj is not None:
                    my_dict = match_obj.groupdict()

                    for k, v in my_dict.items():
                        if 'EMPTY' not in k:
                            event_fields[k] = v

                    event_fields["EVENTTYPE"] = key
                    break

        return event_fields



    def add_pattern(self, pattern_name, pattern):
        self.grok_patterns[pattern_name].append(pattern)

    def remove_pattern(self, pattern_name):
        if pattern_name in self.grok_patterns.keys():
            del self.grok_patterns[pattern_name]

    def get_patterns(self):
        return self.grok_patterns

    def load_patterns(self):
        """load patterns"""
        print "Loading patterns"
        fetched_entities = Pattern.query()
        i = 0
        for pat in fetched_entities:
            print "Pattern found"
            self.grok_patterns[pat.pattern_name].append(pat.pattern_string)
            i = i + 1

        if i < 1:
            print "Putting new pattern"
            self.temp_patterns["SYSMON_PROCESS_TERMINATED"].append("EventType: WCCLogSource: %{GREEDYDATA:EVENTSOURCE} Source: %{GREEDYDATA:SOURCE} EventID: 5  P%{GREEDYDATA:EMPTY} UtcTime: %{GREEDYDATA:UTCTIME} ProcessGuid: %{GREEDYDATA:PROCESSGUID} Proc%{GREEDYDATA:EMPTY} Image: %{WINPATH:IMAGE}")
            self.temp_patterns["SYSMON_NETWORK_CONNECTION"].append("EventType: WCCLogSource: %{GREEDYDATA:EVENTSOURCE} Source: %{GREEDYDATA:SOURCE} EventID: 3  Net%{GREEDYDATA:EMPTY} UtcTime: %{GREEDYDATA:UTCTIME} ProcessGuid: %{GREEDYDATA:PROCESSGUID} P%{GREEDYDATA:EMPTY} Image: %{GREEDYDATA:IMAGE} User: %{GREEDYDATA:USER} Protocol: %{GREEDYDATA:PROTOCOL} Initiated: %{GREEDYDATA:INITIATED} SourceIsIpv6: %{GREEDYDATA:SRCISIPV6} SourceIp: %{GREEDYDATA:SRCIP} SourceHostname: %{GREEDYDATA:SRCHOSTNAME}SourcePort: %{INT:SRCPORT} SourcePortName: %{GREEDYDATA:SRCPORTNAME} DestinationIsIpv6: %{GREEDYDATA:DSTISIPV6} DestinationIp: %{GREEDYDATA:DSTIP} DestinationHostname: %{GREEDYDATA:DSTHOSTNAME} DestinationPort: %{GREEDYDATA:DSTPORT} DestinationPortName: %{GREEDYDATA:DSTPORTNAME}")
            self.temp_patterns["SYSMON_DRIVER_LOADED"].append("EventType: WCCLogSource: %{GREEDYDATA:EVENTSOURCE} Source: %{GREEDYDATA:SOURCE} EventID: 6  D%{GREEDYDATA:EMPTY} UtcTime: %{GREEDYDATA:UTCTIME} ImageLoaded: %{WINPATH:IMAGELOADED} Hashes: %{GREEDYDATA:HASHES} Signed: %{WORD:SIGNED} Signature: %{GREEDYDATA:SIGNATURE}")
            self.temp_patterns["SYSMON_FILE_CREATION_TIME"].append("EventType: WCCLogSource: %{GREEDYDATA:EVENTSOURCE} Source: %{GREEDYDATA:SOURCE} EventID: 2  F%{GREEDYDATA:EMPTY} UtcTime: %{GREEDYDATA:UTCTIME} ProcessGuid: %{GREEDYDATA:PROCESSGUID} P%{GREEDYDATA:EMPTY} Image: %{WINPATH:IMAGE} TargetFilename: %{WINPATH:TARGETFILENAME} CreationUtcTime: %{GREEDYDATA:CREATIONUTCTIME} PreviousCreationUtcTime: %{GREEDYDATA:PREVCREATIONTIME}")
            self.temp_patterns["SYSMON_PROCESS_CREATE"].append("EventType: WCCLogSource: %{GREEDYDATA:EVENTSOURCE} Source: %{GREEDYDATA:SOURCE} EventID: 1  P%{GREEDYDATA:EMPTY} UtcTime: %{GREEDYDATA:UTCTIME} ProcessGuid: %{GREEDYDATA:PROCESSGUID} P%{GREEDYDATA:EMPTY} Image: %{WINPATH:IMAGE} CommandLine: %{GREEDYDATA:COMMANDLINE} CurrentDirectory: %{WINPATH:CURRENTDIRECTORY} User: %{GREEDYDATA:USER} LogonGuid: %{GREEDYDATA:LOGONGUID} Log%{GREEDYDATA:EMPTY} Hashes: %{GREEDYDATA:HASHES} ParentProcessGuid: %{GREEDYDATA:PARENTPROCESSGUID} Pare%{GREEDYDATA:EMPTY} ParentCommandLine: %{GREEDYDATA:PARENTCOMMANDLINE}")
            self.temp_patterns["SYSMON_PROCESS_CREATE"].append("EventType: WCCLogSource: %{GREEDYDATA:EVENTSOURCE} Source: %{GREEDYDATA:SOURCE} EventID: 1  P%{GREEDYDATA:EMPTY} UtcTime: %{GREEDYDATA:UTCTIME} ProcessGuid: %{GREEDYDATA:PROCESSGUID} P%{GREEDYDATA:EMPTY} Image: %{WINPATH:IMAGE} CommandLine: %{GREEDYDATA:COMMANDLINE} CurrentDirectory: %{WINPATH:CURRENTDIRECTORY} User: %{GREEDYDATA:USER} LogonGuid: %{GREEDYDATA:LOGONGUID} Log%{GREEDYDATA:EMPTY} Hashes: %{GREEDYDATA:HASHES} ParentProcessGuid: %{GREEDYDATA:PARENTPROCESSGUID} Pare%{GREEDYDATA:EMPTY} ParentCommandLine: %{GREEDYDATA:PARENTCOMMANDLINE}")
            self.temp_patterns["SYSMON_PROCESS_CREATE"].append("EventType: WCCLogSource: %{GREEDYDATA:EVENTSOURCE} Source: %{GREEDYDATA:SOURCE} EventID: 1 Process Create: UtcTime: %{GREEDYDATA:UTCTIME} ProcessGuid: %{GREEDYDATA:PROCESSGUID} ProcessId:%{GREEDYDATA:EMPTY} Image: %{GREEDYDATA:IMAGE} CommandLine: %{GREEDYDATA:COMMANDLINE} CurrentDirectory: %{GREEDYDATA:EMPTY} User: %{GREEDYDATA:USER} LogonGuid:%{GREEDYDATA:EMPTY} Hashes: %{GREEDYDATA:HASHES} ParentProcessGuid: %{GREEDYDATA:PARENTPROCESSGUID} ParentProcessId:%{GREEDYDATA:EMPTY} ParentImage: %{GREEDYDATA:PARENTIMAGE} ParentCommandLine: %{GREEDYDATA:PARENTCOMMANDLINE}")
            for key, list_item in self.temp_patterns.iteritems():
                for reg in list_item:
                    new_pat = Pattern()
                    new_pat.pattern_name = key
                    new_pat.pattern_string = reg
                    new_pat.put()
            time.sleep(3)
            fetched_entities = Pattern.query()
            for pat in fetched_entities:
                self.grok_patterns[pat.pattern_name].append(pat.pattern_string)
                <file_sep>GoogleAppEngineCloudStorageClient==1.9.22.1
google-cloud-bigquery==0.30.0
requests-toolbelt
flask-bootstrap==3.3.7.1
flask-wtf==0.14.2
<file_sep># CHASM


# Learning Project
Creating an effective Endpoint Detection Platform using Google Cloud IOT, Appengine Standard and Big Query to analyze security events at scale.  Dealing with the necessary events to even handle my home lab had become an annoyance so I began looking for a "cloud" based solution to free me from managing hardware while still allowing me to run queries over billions of events and log entries on a budget.

# Goals and Constraints
  * Learn to program (Python/C#/GO).
  * Run on GCP free tier.
  * Fully managed solution so analyst can focus on detection.
  * 5 Minute basic setup.
 

# Update  07/26/2018
Current version works to parse 5 basic Sysmon events from a C# windows client.  Events are sent securely over Cloud IOT, pushed to Appengine Standard instance where they are written to Cloud Datastore and batched into Big Query. Basic event alerting scheduled on Appengine standard cron.

# Next Steps
  * Review Python code and understand possible insecure coding practices.
  * Automate CHASM client install as service and certificate creation for Cloud IOT.
  * Work on a better solution so alert queries are isolated to new events only to minimize Big Query billing. 
  * Build basic documentation if other users can benefit from the solution.
<file_sep>from google.appengine.ext import vendor
import os
# Add any libraries installed in the "lib" folder.
vendor.add(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib'))

import sys

on_appengine = os.environ.get('SERVER_SOFTWARE','').startswith('Development')

if on_appengine and os.name == 'nt':
    sys.platform = "Not Windows"<file_sep>from google.appengine.ext import testbed
import os
import main
import unittest
import tempfile
import json
import base64
from chasm_common.simplegrok import SimpleGrok
from chasm_common.chasm_ds_config import ChasmConfig

from google.appengine.ext import ndb
class MainTestCase(unittest.TestCase):

    def setUp(self):
        #self.db_fd, flaskr.app.config['DATABASE'] = tempfile.mkstemp()
        # First, create an instance of the Testbed class.
        
        # Then activate the testbed, which prepares the service stubs for use.
        self.testbed = testbed.Testbed()
       
        self.testbed.activate()
        self.testbed.init_all_stubs()
      
        
        main.app.config['WTF_CSRF_ENABLED'] = False
        main.app.testing = True
        
        self.app = main.app.test_client()
        ndb.get_context().clear_cache()

        s = ChasmConfig.get_or_insert("Default", 
                                    project_id=os.environ.get('APPLICATION_ID'),
                                    bucket_id=os.environ.get('BUCKET_NAME'))

        #with main.app.app_context():
        #    flaskr.init_db()

    def tearDown(self):
        # Don't forget to deactivate the testbed after the tests are
        # completed. If the testbed is not deactivated, the original
        # stubs will not be restored.
        self.testbed.deactivate()
    
    def test_hello(self):
        """Basic home page"""
        rv = self.app.get('/')
        #print rv.data
        assert b'stuff' in rv.get_data()


    def test_pubsubmessages(self):
        """Testing pubsub message"""
        rv = self.app.post('/_ah/push-handlers/pushsub', 
                      data=json.dumps({"message":{"data":"<KEY>
                                                 "attributes":{"projectId":"project","deviceId":"LAPTOP1","deviceNumId":"11111111111111","deviceRegistryId":"events","deviceRegistryLocation":"us-central1","subFolder":""},"message_id":"22520556751004","messageId":"22520556751004","publish_time":"2018-01-12T22:54:16.500Z","publishTime":"2018-01-12T22:54:16.500Z"},"subscription":"projects/sub/subscriptions/push"}
                     ),follow_redirects=True)
        print rv.data
        assert b'Success 4' in rv.data


    def test_grok_pattern(self):
        """Testing grok pattern testing form"""
        rv = self.app.post('/grok_test', data=dict(
              event_text='EventType: WCCLogSource: Microsoft-Windows-Sysmon/Operational Source: DESKTOP-3L8ABIH EventID: 1 Process Create: UtcTime: 2018-02-08 23:46:10.129 ProcessGuid: {EF342AE3-E142-5A7C-0000-001082982B33} ProcessId: 18648 Image: C:\Program Files\Docker\Docker\resources\com.docker.9pdb.exe CommandLine: "C:\Program Files\Docker\Docker\Resources\com.docker.9pdb.exe" set com.docker.driver.amd64-linux/slirp/dns="nameserver 192.168.6.1 nameserver 172.16.58.3 nameserver 172.16.31.10 search hil-grvwyhx.slc.wayport.net" CurrentDirectory: C:\WINDOWS\system32\ User: NT AUTHORITY\SYSTEM LogonGuid: {EF342AE3-AF49-5A73-0000-0020E7030000} LogonId: 0x3E7 TerminalSessionId: 0 IntegrityLevel: System Hashes: SHA1=E5A302480361BCF66F34EB933BD599715A7A50D9 ParentProcessGuid: {EF342AE3-AF4C-5A73-0000-001041870200} ParentProcessId: 3404 ParentImage: C:\Program Files\Docker\Docker\com.docker.service ParentCommandLine: "C:\Program Files\Docker\Docker\com.docker.service"',
              pattern_text='EventType: WCCLogSource: %{GREEDYDATA:EVENTSOURCE} Source: %{GREEDYDATA:SOURCE} EventID: 1 Process Create: UtcTime: %{GREEDYDATA:UTCTIME} ProcessGuid: %{GREEDYDATA:PROCESSGUID} ProcessId:%{GREEDYDATA:EMPTY} Image: %{GREEDYDATA:IMAGE} CommandLine: %{GREEDYDATA:COMMANDLINE} CurrentDirectory: %{GREEDYDATA:EMPTY} User: %{GREEDYDATA:USER} LogonGuid:%{GREEDYDATA:EMPTY} Hashes: %{GREEDYDATA:HASHES} ParentProcessGuid: %{GREEDYDATA:PARENTPROCESSGUID} ParentProcessId:%{GREEDYDATA:EMPTY} ParentImage: %{GREEDYDATA:PARENTIMAGE} ParentCommandLine: %{GREEDYDATA:PARENTCOMMANDLINE}'
            ), follow_redirects=True)
        assert b'<h4> EVENTTYPE </h4> TEST_PARSER' in rv.data
        assert b'<h4> IMAGE </h4> C:\Program Files\Docker\Docker\resources\com.docker.9pdb.exe' in rv.data
        assert b'<h4> PARENTPROCESSGUID </h4> {EF342AE3-AF4C-5A73-0000-001041870200}' in rv.data
        assert b'<h4> UTCTIME </h4> 2018-02-08 23:46:10.129' in rv.data
        assert b'<h4> USER </h4> NT AUTHORITY\SYSTEM' in rv.data
        assert b'<h4> PROCESSGUID </h4> {EF342AE3-E142-5A7C-0000-001082982B33}' in rv.data
        assert b'<h4> SOURCE </h4> DESKTOP-3L8ABIH' in rv.data
        assert b'<h4> EVENTSOURCE </h4> Microsoft-Windows-Sysmon/Operational' in rv.data
        assert b'<h4> EVENTSOURCE </h4> Microsoft-Windows-Sysmon/Operational' in rv.data
    
    def test_grok_default_patterns(self):
        """Testing parsers"""
        rv = self.app.get('/grok_parsers' , follow_redirects=True)
        assert b'<div>Current Patterns<br>' in rv.data
        assert b'<h4> SYSMON_NETWORK_CONNECTION [u&#39;EventType: WCCLogSource: %{GREEDYDATA:EVENTSOURCE} Source: %{GREEDYDATA:SOURCE} EventID: 3  Net%{GREEDYDATA:EMPTY} UtcTime: %{GREEDYDATA:UTCTIME} ProcessGuid: %{GREEDYDATA:PROCESSGUID} P%{GREEDYDATA:EMPTY} Image: %{GREEDYDATA:IMAGE} User: %{GREEDYDATA:USER} Protocol: %{GREEDYDATA:PROTOCOL} Initiated: %{GREEDYDATA:INITIATED} SourceIsIpv6: %{GREEDYDATA:SRCISIPV6} SourceIp: %{GREEDYDATA:SRCIP} SourceHostname: %{GREEDYDATA:SRCHOSTNAME}SourcePort: %{INT:SRCPORT} SourcePortName: %{GREEDYDATA:SRCPORTNAME} DestinationIsIpv6: %{GREEDYDATA:DSTISIPV6} DestinationIp: %{GREEDYDATA:DSTIP} DestinationHostname: %{GREEDYDATA:DSTHOSTNAME} DestinationPort: %{GREEDYDATA:DSTPORT} DestinationPortName: %{GREEDYDATA:DSTPORTNAME}&#39;]</h4>' in rv.data
       
    def test_big_query(self):
        """Testing query from bigdata"""
        rv = self.app.post('/query_main', data=dict(
                query_text="""SELECT * FROM `chasm.logs*` where 
                            REGEXP_CONTAINS(fullevent,r"EF342AE3-060D-5A34-0000-0010328BDE01") 
                            limit 10""",
                ), follow_redirects=True)
        assert b'<td>{EF342AE3-060D-5A34-0000-0010328BDE01}' in rv.data

    def test_add_alert_big_query(self):
        """Testing add a big query alert."""
       
        rv1 = self.app.post('/add_alert_big_query', data=dict(
              alert_name="TESTERNAME",
              alert_description="TESTERDESCRIPTION",
              alert_query="SELECT",
              submit="submit"
             ), follow_redirects=True)
        
        rv = self.app.get('/add_alert_big_query', follow_redirects=True)
        assert b'">TESTERNAME</a>' in rv.data
    
    def test_alert_query(self):
        """Testing running bigquery alert test."""
        
        rv1 = self.app.post('/add_alert_big_query', data=dict(
              alert_name="TESTER",
              alert_description="TESTERDESCRIPTION",
              alert_query="SELECT FULLEVENT FROM `chasm.logs`  limit 10",
              submit="submit"
             ), follow_redirects=True)
        
        rv = self.app.get('/add_alert_big_query', follow_redirects=True)
        assert b'">TESTER</a>' in rv.data
        
        rv = self.app.get('/fifteen_minute_cron', follow_redirects=True)

        rv = self.app.get('/view_alerts', follow_redirects=True)
        assert b'<td>TESTER</td>' in rv.data
        




if __name__ == '__main__':
    unittest.main()<file_sep>class Config:
    SECRET_KEY = os.environ.get('SECRET_KEY') or 'hard to guess string'

class TestingConfig(Config):
    TESTING = True
    WTF_CSRF_ENABLED = False<file_sep># [START app]

import logging
import os
import json
import sys

from flask import Flask,flash, request,redirect, render_template
import base64

from chasm_common.simplegrok import SimpleGrok
from chasm_common.chasm_bq import chasm_bq
from chasm_common.chasm_gcs import chasm_gcs
from google.appengine.api import app_identity
from flask_bootstrap import Bootstrap
from google.appengine.ext import ndb
import cloudstorage as gcs

from google.appengine.api import app_identity
from flask_wtf import FlaskForm
from wtforms import StringField, PasswordField, BooleanField, SubmitField,TextAreaField,SelectField
from wtforms.validators import DataRequired




app = Flask(__name__)
Bootstrap(app)
app.config['SECRET_KEY'] = os.environ.get('SECRET_KEY') or 'you-will-never-guess'

s_grok = None
bq_chasm_client = None 
chasm_gcs_client = None
PROJECT_ID = None



class AlertsDisplayForm(FlaskForm):
    alert_status = SelectField(choices=[('0', 'New'),
                                        ('1', 'False Positive'), 
                                        ('3', 'Attack')])
    submit = SubmitField('Resolve')
    cancel = SubmitField('Cancel')

class AlertsForm(FlaskForm):
    alert_name = StringField('Alert Name', validators=[DataRequired()])
    alert_description = StringField('Alert Description', validators=[DataRequired()])
    alert_query = TextAreaField('Alert Query', validators=[DataRequired()])
    #remember_me = BooleanField('Remember Me')
    test_alert = SubmitField('Test Alert Query')
    submit = SubmitField('Add Alert')
    update = SubmitField('Update Alert')
    delete = SubmitField('Delete Alert')



def setup_app(app):
    """ Initialize Parser """
    #global s_grok
    #global bq_chasm_client
   
    
    global PROJECT_ID 
    PROJECT_ID = "TEMP"

    on_appengine = os.environ.get('SERVER_SOFTWARE', '').startswith('Development')
    print "Initialize Flask"
    if not on_appengine:
        print "not on appengine"
        PROJECT_ID = os.getenv('APPLICATION_ID')
    else:
        print "Looking for google cloud project"
        PROJECT_ID = os.environ.get('GOOGLE_CLOUD_PROJECT')
        if PROJECT_ID is None:
            PROJECT_ID = "DEV"
    
    if 's~' in PROJECT_ID:
        PROJECT_ID = PROJECT_ID.replace('s~','')

 
    


setup_app(app)

@app.route('/_ah/push-handlers/pushsub',methods=['GET','POST'])
def pubsub_push():
    global s_grok
    global bq_chasm_client
    global chasm_gcs_client

    rows = []
    if s_grok is None:
        s_grok = SimpleGrok()
    if bq_chasm_client is None:
        bq_chasm_client = chasm_bq()
    if chasm_gcs_client is None:
        chasm_gcs_client = chasm_gcs()

    page_data = request.data.decode('utf-8')
    
    try:
        
        envelope = json.loads(request.data.decode('utf-8'))
        
        base64_payload = base64.b64decode(envelope['message']['data'])
        
        data = json.loads(base64_payload)
               
        for log in data:
            event_fields = s_grok.get_field_dict(log)
            
            rows.append(event_fields)
          
        
        chasm_gcs_client.write_logs_file(rows)
        
    except:

        exc_tuple = sys.exc_info()
        print str(exc_tuple)

    return "Success {}".format(len(rows))   

@app.route('/manage_alert',methods=['GET','POST'])
def manage_alert():
    global s_grok
    global bq_chasm_client
    global chasm_gcs_client

    
    if s_grok is None:
        s_grok = SimpleGrok()
    if bq_chasm_client is None:
        bq_chasm_client = chasm_bq()
    if chasm_gcs_client is None:
        chasm_gcs_client = chasm_gcs()

    alert_key = request.args.get('id')
    alert, rows, schema = bq_chasm_client.get_generated_alert_by_key(alert_key)
    
   

    form = AlertsDisplayForm()
   
    if form.validate_on_submit():
       
        if form.submit.data:
            value = form.alert_status.data
            bq_chasm_client.update_generated_alert_by_key(int(value),alert_key)
            
            return redirect('/view_alerts',code=302)
    else:
        form.alert_status.data = str(alert.alert_status)

    return render_template('bq_manage_alert.html',alert=alert,form=form,rows=rows,schema=schema)




@app.route('/view_alerts',methods=['GET','POST'])
def view_alerts():
    global s_grok
    global bq_chasm_client
    global chasm_gcs_client

    
    if s_grok is None:
        s_grok = SimpleGrok()
    if bq_chasm_client is None:
        bq_chasm_client = chasm_bq()
    if chasm_gcs_client is None:
        chasm_gcs_client = chasm_gcs()

    alerts = bq_chasm_client.get_alerts()
     
    return render_template('bq_display_alerts.html',alerts=alerts)

@app.route('/cron_batch_logs',methods=['GET','POST'])
def cron_batch_logs():
    global s_grok
    global bq_chasm_client
    global chasm_gcs_client

    rows = []
    if s_grok is None:
        s_grok = SimpleGrok()
    if bq_chasm_client is None:
        bq_chasm_client = chasm_bq()
    if chasm_gcs_client is None:
        chasm_gcs_client = chasm_gcs()

    result = bq_chasm_client.load_job_from_gcs()
   
    return result
   
@app.route('/cron_real_time_alert',methods=['GET','POST'])
def cron_real_time_alert():
    global s_grok
    global bq_chasm_client
    global chasm_gcs_client

    rows = []
    if s_grok is None:
        s_grok = SimpleGrok()
    if bq_chasm_client is None:
        bq_chasm_client = chasm_bq()
    if chasm_gcs_client is None:
        chasm_gcs_client = chasm_gcs()
             
    result = bq_chasm_client.load_job_from_gcs()
   
    return result
    

@app.route('/')
def hello():
    return render_template('index.html')

@app.route('/fifteen_minute_cron', methods=['GET'])
def fifteen_minute_cron():
    global bq_chasm_client
    if bq_chasm_client is None:
        bq_chasm_client = chasm_bq()
    
    results = bq_chasm_client.run_scheduled_alerts(15)
    
    print results
    
    return render_template('index.html')

@app.route('/add_alert_big_query',methods=['GET','POST'])
def add_alert_big_query():
    global bq_chasm_client
    if bq_chasm_client is None:
        bq_chasm_client = chasm_bq()
   
    form = AlertsForm()
    
    if form.validate_on_submit():
        flash('Login requested for user {}, remember_me={}'.format(
            form.alert_name.data, form.alert_description.data))
        
        if form.submit.data:
            bq_chasm_client.add_scheduled_alert(form.alert_name.data,
                                            form.alert_description.data,
                                            form.alert_query.data,15)
            return redirect('/add_alert_big_query',code=302)
        elif form.test_alert.data:
            pattern_text = form.alert_query.data
            print pattern_text
            schema,rows,bytes_billed,byte_processed,errors,alert_table = bq_chasm_client.run_query(pattern_text)
        
         
            
            return render_template('bq_alert.html', title='Sign In', form=form,
                                                alert_data=bq_chasm_client.get_scheduled_alerts(),
                                                bytes_billed=bytes_billed,
                                                byte_processed=byte_processed,
                                                schema=schema,query_result=rows,
                                                search_text=pattern_text,
                                                error=errors)

        
    return render_template('bq_alert.html', title='Sign In', form=form,
                                                alert_data=bq_chasm_client.get_scheduled_alerts())
    

@app.route('/edit_alert',methods=['GET','POST'])
def edit_alert(): 
    global bq_chasm_client
    if bq_chasm_client is None:
        bq_chasm_client = chasm_bq()
   
    
    form = AlertsForm()
    
    if form.validate_on_submit():
        flash('Login requested for user {}, remember_me={}'.format(
            form.alert_name.data, form.alert_description.data))
        
        if form.update.data:
            alert_key = request.args.get('id')
            bq_chasm_client.update_scheduled_alert(form.alert_name.data,
                                            form.alert_description.data,
                                            form.alert_query.data,15,alert_key)
            return render_template('bq_edit_alert.html', title='Sign In', form=form,
                                                alert_data=bq_chasm_client.get_scheduled_alerts())
        elif form.test_alert.data:
            pattern_text = form.alert_query.data
            print pattern_text
            schema,rows,bytes_billed,byte_processed,errors,alert_table = bq_chasm_client.run_query(pattern_text)
        
         
            
            return render_template('bq_edit_alert.html', title='Sign In', form=form,
                                                alert_data=bq_chasm_client.get_scheduled_alerts(),
                                                bytes_billed=bytes_billed,
                                                byte_processed=byte_processed,
                                                schema=schema,query_result=rows,
                                                search_text=pattern_text,
                                                error=errors)
        elif form.delete.data:
            alert_key = request.args.get('id')
            bq_chasm_client.delete_scheduled_alert(alert_key)
            
            return redirect('/add_alert_big_query',code=302)

    alert_key = request.args.get('id')
    alert_name, alert_description, alert_query = bq_chasm_client.get_scheduled_alert(alert_key)    
    form.alert_name.data = alert_name
    form.alert_description.data = alert_description
    form.alert_query.data = alert_query
    return render_template('bq_edit_alert.html', title='Sign In', form=form,
                                                alert_data=bq_chasm_client.get_scheduled_alerts())
    


@app.route('/grok_test',methods=['GET','POST'])
def grok_test(): 
    global s_grok
    if s_grok is None:
       s_grok = SimpleGrok()
    
    if request.method == 'POST':
        s_grok.remove_pattern("TEST_PARSER")
        event_text = request.form['event_text']
        pattern_text = request.form['pattern_text']
        s_grok.add_pattern("TEST_PARSER",pattern_text)
        event_fields = s_grok.get_field_dict(event_text)
        return render_template('grok_test.html',event=event_text,
                                               pattern=pattern_text,
                                               parse_results=event_fields,
                                               current_reg=s_grok.types)
    else:
        return render_template('grok_test.html')

@app.route('/grok_parsers',methods=['GET','POST'])
def grok_current_parsers(): 
    global s_grok
    if s_grok is None:
        s_grok = SimpleGrok()
    
    if request.method == 'GET':

        grok_patterns = s_grok.get_patterns()
        return render_template('grok_current_parsers.html',patterns=grok_patterns)

@app.route('/query_main',methods=['GET','POST'])
def query_main(): 
    global s_grok
    global bq_chasm_client
    #rows = []
    print PROJECT_ID
    if bq_chasm_client is None:
        bq_chasm_client = chasm_bq()
    
    if request.method == 'POST':
        pattern_text = request.form['query_text']
        print pattern_text
        schema,rows,bytes_billed,byte_processed,errors,alert_table = bq_chasm_client.run_query(pattern_text)
        #print rows
        return render_template('query_main.html',bytes_billed=bytes_billed,
                                                 byte_processed=byte_processed,
                                                 schema=schema,query_result=rows,
                                                 search_text=pattern_text,
                                                 error=errors)
    else:
        return render_template('query_main.html')


@app.errorhandler(500)
def server_error(e):
    # Log the error and stacktrace.
    logging.exception('An error occurred during a request.')
    return 'An internal error occurred.', 500
# [END app]