<repo_name>VestedSkeptic/datagrab<file_sep>/reddit/config.py
import cLogger

clog = None

# --------------------------------------------------------------------------
def initializeCLogger():
    global clog
    clog = cLogger.cLogger(__name__)
    clog.addTraceLoggingLevel()
    clog.generateFuncDict()
    clog.addConsoleLogger(cLogger.cLoggerLevel_INFO, "hConsole",  cLogger.cLoggerFilter_None)

    clog.addFileLogger("/home/delta/work/logs/out.txt",   cLogger.cLoggerLevel_DEBUG,   "hFile out",    cLogger.cLoggerFile_archiveOlder,     cLogger.cLoggerFilter_None)
    clog.addFileLogger("/home/delta/work/logs/trace.txt", cLogger.cLoggerLevel_TRACE,   "hFile trace",  cLogger.cLoggerFile_overwriteOlder,   cLogger.cLoggerFilter_SpecificLevelOnly)
    clog.addFileLogger("/home/delta/work/logs/alert.txt", cLogger.cLoggerLevel_WARNING, "hFile alert",  cLogger.cLoggerFile_appendOlder,      cLogger.cLoggerFilter_GreaterThanOrEqualToLevel)

    clog.setLoggerInfoLevel(cLogger.cLoggerLevel_TRACE)
    clog.setMethodInfoLevel(cLogger.cLoggerLevel_DEBUG)
    clog.dumpLoggerInfo()












<file_sep>/reddit/views/vthread.py
from django.http import HttpResponse
from django.shortcuts import redirect, render
from django.core.exceptions import ObjectDoesNotExist
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger   # for pagination
from ..config import clog
from ..models import mthread

# *****************************************************************************
def list(request, subreddit):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    qs = mthread.objects.filter(subreddit__name=subreddit).order_by("-rcreated")
    paginator = Paginator(qs, 20) # Show 20 per page

    page = request.GET.get('page')
    try:
        threads = paginator.page(page)
    except PageNotAnInteger:
        threads = paginator.page(1) # If page is not an integer, deliver first page.
    except EmptyPage:
        threads = paginator.page(paginator.num_pages) # If page is out of range (e.g. 9999), deliver last page of results.

    return render(request, 'vthread_list.html', {'threads': threads, 'subreddit':subreddit})

# *****************************************************************************
def delAll(request):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    vs = ''
    qs = mthread.objects.all()
    vs += str(qs.count()) + " mSubmissions deleted"
    qs.delete()

    clog.logger.info(vs)
    sessionKey = 'blue'
    request.session[sessionKey] = vs
    return redirect('vbase.main', xData=sessionKey)

























<file_sep>/reddit/models/msubreddit.py
from __future__ import unicode_literals
from django.db import models
from django.core.exceptions import ObjectDoesNotExist
from .mbase import mbase
from ..config import clog
from datetime import datetime
from django.utils import timezone
import pprint

# *****************************************************************************
class msubredditManager(models.Manager):
    def addOrUpdate(self, name, prawSubreddit):
        mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        try:
            i_msubreddit = self.get(name=name)
            redditFieldDict = i_msubreddit.getRedditFieldDict()
            changedCount = i_msubreddit.updateRedditFields(prawSubreddit, redditFieldDict)
            if changedCount == 0: i_msubreddit.addOrUpdateTempField = "oldUnchanged"
            else:                 i_msubreddit.addOrUpdateTempField = "oldChanged"
        except ObjectDoesNotExist:
            i_msubreddit = self.create(name=name)
            redditFieldDict = i_msubreddit.getRedditFieldDict()
            i_msubreddit.addRedditFields(prawSubreddit, redditFieldDict)
            i_msubreddit.addOrUpdateTempField = "new"

        clog.logger.info("i_msubreddit = %s" % (pprint.pformat(vars(i_msubreddit))))
        i_msubreddit.save()
        return i_msubreddit

# *****************************************************************************
# class msubreddit(mbase, models.Model):
class msubreddit(mbase):
    name                            = models.CharField(max_length=30)
    # properties
    ppoi                            = models.BooleanField(default=False)
    # precentlyupdated                = models.BooleanField(default=False)
    pprioritylevel                  = models.IntegerField(default=0)
    # pthreadupdatetimestamp          = models.DateTimeField(default=datetime(2000, 1, 1, 1, 0, 0))
    # pthreadupdatetimestamp          = models.DateTimeField(default=timezone.now())
    # pthreadupdatetimestamp          = models.DateTimeField(default=timezone.now)
    pthreadupdatetimestamp          = models.DateTimeField(default=timezone.make_aware(datetime(2000, 1, 1, 1, 0, 0)))
    pupdateswithnochanges           = models.IntegerField(default=0)
    pcountnew                       = models.IntegerField(default=0)
    pcountOldUnchanged              = models.IntegerField(default=0)
    pcountOldChanged                = models.IntegerField(default=0)
    # Redditor fields
    raccounts_active_is_fuzzed      = models.BooleanField(default=False)
    rcollapse_deleted_comments      = models.BooleanField(default=False)
    rpublic_traffic                 = models.BooleanField(default=False)
    rquarantine                     = models.BooleanField(default=False)
    rshow_media_preview             = models.BooleanField(default=False)
    rshow_media                     = models.BooleanField(default=False)
    rspoilers_enabled               = models.BooleanField(default=False)
    rhide_ads                       = models.BooleanField(default=False)
    rover18                         = models.BooleanField(default=False)

    raccounts_active                = models.IntegerField(default=0)
    rcomment_score_hide_mins        = models.IntegerField(default=0)
    rsubscribers                    = models.IntegerField(default=0)

    rdescription                    = models.TextField(default='')
    rpublic_description             = models.TextField(default='')

    rdisplay_name                   = models.CharField(max_length=100, default='', blank=True)
    rlang                           = models.CharField(max_length=20, default='', blank=True)
    rid                             = models.CharField(max_length=12, default='', blank=True)
    rsubmission_type                = models.CharField(max_length=12, default='', blank=True)
    rsubreddit_type                 = models.CharField(max_length=12, default='', blank=True)
    rtitle                          = models.CharField(max_length=101, default='', blank=True)
    rurl                            = models.CharField(max_length=40, default='', blank=True)
    rwhitelist_status               = models.CharField(max_length=24, default='', blank=True)

    rcreated_utc                    = models.DecimalField(default=0, max_digits=12,decimal_places=1)
    rcreated                        = models.DecimalField(default=0, max_digits=12,decimal_places=1)

    radvertiser_category            = models.TextField(default='')
    rwiki_enabled                   = models.TextField(default='')

    objects = msubredditManager()

    # -------------------------------------------------------------------------
    def getRedditFieldDict(self):
        mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        redditFieldDict = {
            # mThreadFieldName              redditFieldName                 convertMethodPtr
            'raccounts_active_is_fuzzed':   ("accounts_active_is_fuzzed",   None),      # bool
            'rcollapse_deleted_comments':   ("collapse_deleted_comments",   None),      # bool
            'rpublic_traffic':              ("public_traffic",              None),      # bool
            'rquarantine':                  ("quarantine",                  None),      # bool
            'rshow_media_preview':          ("show_media_preview",          None),      # bool
            'rshow_media':                  ("show_media",                  None),      # bool
            'rspoilers_enabled':            ("spoilers_enabled",            None),      # bool
            'rhide_ads':                    ("hide_ads",                    None),      # bool
            'rover18':                      ("over18",                      None),      # bool

            'raccounts_active':             ("accounts_active",             int),       # int
            'rcomment_score_hide_mins':     ("comment_score_hide_mins",     int),       # int
            'rsubscribers':                 ("subscribers",                 int),       # int

            'rdescription':                 ("description",                 None),      # string text area
            'rpublic_description':          ("public_description",          None),      # string text area

            'rdisplay_name':                ("display_name",                None),      # string
            'rlang':                        ("lang",                        None),      # string
            'rid':                          ("id",                          None),      # string
            'rsubmission_type':             ("submission_type",             None),      # string
            'rsubreddit_type':              ("subreddit_type",              None),      # string
            'rtitle':                       ("title",                       None),      # string
            'rurl':                         ("url",                         None),      # string
            'rwhitelist_status':            ("whitelist_status",            self.getStringOrNoneAsEmptyString),      # string

            'rcreated_utc':                 ("created_utc",                 None),      # 1493534605.0,
            'rcreated':                     ("created",                     None),      # 1493534605.0,

            'radvertiser_category':         ("advertiser_category",         self.getStringOrNoneAsEmptyString),      # string text area
            'rwiki_enabled':                ("wiki_enabled",                self.getStringOrNoneAsEmptyString),      # string text are
        }
        return redditFieldDict

    # -------------------------------------------------------------------------
    def __str__(self):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        s = self.name
        # if self.ppoi: s += " (ppoi)"
        return format(s)

    # --------------------------------------------------------------------------
    def getThreadsBestBeforeValue(self, prawReddit):
        mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        # clog.logger.info("METHOD NOT COMPLETED")
        return ''

    # --------------------------------------------------------------------------
    def threadsUpdated(self, countNew, countOldUnchanged, countOldChanged):
        mi = clog.dumpMethodInfo()
        # self.precentlyupdated =True

        if countNew > 0:
            self.pprioritylevel -= 1
            self.pupdateswithnochanges = 0
            if self.pprioritylevel < 0:
                self.pprioritylevel = 0;
        else:
            self.pprioritylevel += 1
            self.pupdateswithnochanges += 1
            if self.pprioritylevel > 3:
                self.pprioritylevel = 3;

        self.pcountnew          = countNew
        self.pcountOldUnchanged = countOldUnchanged
        self.pcountOldChanged   = countOldChanged

        # self.pthreadupdatetimestamp = datetime.now()
        self.pthreadupdatetimestamp = timezone.now()
        self.save()
        return

# # ----------------------------------------------------------------------------
# # SUBREDDIT attributes
# {
#   '_banned': None,
#   '_comments': None,
#   '_contributor': None,
#   '_fetched': True,
#   '_filters': None,
#   '_flair': None,
#   '_info_params': {},
#   '_mod': None,
#   '_moderator': None,
#   '_modmail': None,
#   '_muted': None,
#   '_path': 'r/redditdev/',
#   '_quarantine': None,
#   '_reddit': <praw.reddit.Reddit object at 0x7f1474006e48>,
#   '_stream': None,
#   '_stylesheet': None,
#   '_wiki': None,
#   'accounts_active_is_fuzzed': True,
#   'accounts_active': 9,
#   'advertiser_category': None,
#   'allow_images': False,
#   'banner_img': '',
#   'banner_size': None,
#   'collapse_deleted_comments': False,
#   'comment_score_hide_mins': 0,
#   'created_utc': 1213802177.0,
#   'created': 1213830977.0,
#   'description_html': '<!-- SC_OFF --><div class="md"><p>A subreddit for '
#   'description': 'A subreddit for discussion of reddit API clients and the ',
#   'display_name_prefixed': 'r/redditdev',
#   'display_name': 'redditdev',
#   'header_img': 'https://e.thumbs.redditmedia.com/bOToSJt13ylszjE4.png',
#   'header_size': [120, 40],
#   'header_title': None,
#   'hide_ads': False,
#   'icon_img': '',
#   'icon_size': None,
#   'id': '2qizd',
#   'key_color': '#ff4500',
#   'lang': 'en',
#   'name': 't5_2qizd',
#   'over18': False,
#   'public_description_html': '<!-- SC_OFF --><div class="md"><p>A subreddit for ',
#   'public_description': 'A subreddit for discussion of reddit API clients and ',
#   'public_traffic': True,
#   'quarantine': False,
#   'show_media_preview': False,
#   'show_media': False,
#   'spoilers_enabled': False,
#   'submission_type': 'self',
#   'submit_link_label': None,
#   'submit_text_html': '<!-- SC_OFF --><div class="md"><p>Get faster, better ',
#   'submit_text_label': None,
#   'submit_text': 'Get faster, better responses by including more information, ',
#   'subreddit_type': 'public',
#   'subscribers': 10887,
#   'suggested_comment_sort': None,
#   'title': 'reddit Development',
#   'url': '/r/redditdev/',
#   'user_is_banned': False,
#   'user_is_contributor': False,
#   'user_is_moderator': False,
#   'user_is_muted': False,
#   'user_is_subscriber': False,
#   'user_sr_theme_enabled': False,
#   'whitelist_status': 'all_ads',
#   'wiki_enabled': None
#  }

<file_sep>/reddit/forms/fsubreddit.py
from django import forms

# *****************************************************************************
class fNewPoiSubreddit(forms.Form):
    poiSubreddit = forms.CharField(label='Subreddit Name', max_length=30)

<file_sep>/reddit/views/vbase.py
from django.http import HttpResponse
from django.shortcuts import redirect
from ..config import clog
from ..models import mcomment, msubreddit, mthread, muser
from ..tasks.tmisc import TASK_generateModelCountData
# import pprint

# *****************************************************************************
def main(request, xData=None):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    vs = '<b>vuser</b>:'
    vs += ' <a href="http://localhost:8000/reddit/vuser/list">list</a>'
    vs += ', <b>add</b> <a href="http://localhost:8000/reddit/vuser/formNewPoiUser">newPoiUser</a>'

    vs += '<br><b>vsubreddit</b>:'
    vs += ' <a href="http://localhost:8000/reddit/vsubreddit/list">list</a>'
    vs += ', <b>add</b> <a href="http://localhost:8000/reddit/vsubreddit/formNewPoiSubreddit">newPoiSubreddit</a>'

    vs += '<br><a href="http://localhost:8000/reddit/vbase/map">map</a>'

    # vs += ', <a href="http://localhost:8000/reddit/vuser/delAll">delAll</a>'
    # vs += ', <a href="http://localhost:8000/reddit/vsubreddit/delAll">delAll</a>'
    # vs += '  <a href="http://localhost:8000/reddit/vthread/delAll">delAll</a>'

    vs += displayDatabaseModelCounts()
    vs += '<br><a href="http://localhost:8000/reddit/vbase/test">vbase.test</a>'

    vs += '<br>' + request.session.get(xData, '')
    return HttpResponse(vs)

# *****************************************************************************
def displayDatabaseModelCounts():
    mi = clog.dumpMethodInfo()
    # clog.logger.info(mi)

    listOfModelCountStrings = TASK_generateModelCountData()

    s = '<BR>============================'
    s += '<font face="Courier New" color="green">'
    for line in listOfModelCountStrings:
        s += '<BR>'
        s += line.replace(" ", "&nbsp;")
    s += '</font>'
    s += '<BR>============================'
    return s

# # *****************************************************************************
# def test(request):
#     mi = clog.dumpMethodInfo()
#     clog.logger.info(mi)
#
#     vs = "vbase.test: EMPTY TEST"
#
#     sessionKey = 'blue'
#     request.session[sessionKey] = vs
#     return redirect('vbase.main', xData=sessionKey)

# *****************************************************************************
def test(request):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    vs = ''

    prawReddit = mcomment.getPrawRedditInstance()
    prawSubmissionObject = prawReddit.submission(id='6fvbyb')

    for duplicate in prawSubmissionObject.duplicates():
        vs += duplicate.subreddit_name_prefixed
        vs += ', '


    sessionKey = 'blue'
    request.session[sessionKey] = vs
    return redirect('vbase.main', xData=sessionKey)

# *****************************************************************************
def generateMapLink():
    vs = '<img src="'
    vs += "https://maps.googleapis.com/maps/api/staticmap?"

    # vs += "center=Quyon,Quebec"
    # vs += "center=45.529997,-76.12571597"       # compound
    vs += "center=45.53716674,-76.14626169"     # NW of compound, center of walking area

    vs += "&zoom=14"
    vs += "&size=1000x800"
    vs += "&maptype=roadmap"

    # vs += "&markers=color:blue%7Clabel:S%7C40.702147,-74.015794"
    # vs += "&markers=color:green%7Clabel:G%7C40.711614,-74.012318"
    # vs += "&markers=color:red%7Clabel:C%7C40.718217,-73.998284"

    vs += "&style=feature:road%7Celement:geometry%7Ccolor:0x000000&size=480x360"

    # my google api key
    # <KEY>
    vs += "&key=<KEY>"
    vs += '">'

    return vs

# *****************************************************************************
def map(request):
    vs = generateMapLink()
    return HttpResponse(vs)

<file_sep>/reddit/forms/fuser.py
from django import forms

# *****************************************************************************
class fNewPoiUser(forms.Form):
    poiUser = forms.CharField(label='Users Name', max_length=30)

<file_sep>/reddit/views/vcomment.py
from django.http import HttpResponse
from django.shortcuts import redirect, render
from django.core.exceptions import ObjectDoesNotExist
# from django.utils.safestring import mark_safe
from ..config import clog
from ..models import mcomment
from ..forms import fuser
# import pprint

# *****************************************************************************
def user(request, username):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    # qs = mcomment.objects.filter(username=username).order_by('rcreated')
    # qs = mcomment.objects.filter(username=username).order_by('-rcreated')
    qs = mcomment.objects.filter(username=username).order_by('-rcreated_utc')
    return render(request, 'vcomment_user.html', {'comments': qs, 'username': username})

















<file_sep>/reddit/models/mcomment.py
from __future__ import unicode_literals
from django.db import models
from django.core.exceptions import ObjectDoesNotExist
from .mbase import mbase
from ..config import clog
# import praw
# import pprint

# *****************************************************************************
class mcommentManager(models.Manager):
    def addOrUpdate(self, username, prawComment):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        # NOTE SIMILAR CHANGE AS MUSER TO HANDLE DUPLIICATE COMMENTS

        try:
            qs = mcomment.objects.filter(username=username, name=prawComment.name, thread=prawComment.link_id, subreddit=prawComment.subreddit_id)
            if qs.count() > 0:
                i_mcomment = qs[0]
                redditFieldDict = i_mcomment.getRedditFieldDict()
                changedCount = i_mcomment.updateRedditFields(prawComment, redditFieldDict)
                if changedCount == 0: i_mcomment.addOrUpdateTempField = "oldUnchanged"
                else:                 i_mcomment.addOrUpdateTempField = "oldChanged"
            else:
                i_mcomment = self.create(username=username, name=prawComment.name, thread=prawComment.link_id, subreddit=prawComment.subreddit_id)
                redditFieldDict = i_mcomment.getRedditFieldDict()
                i_mcomment.addRedditFields(prawComment, redditFieldDict)
                i_mcomment.addOrUpdateTempField = "new"
        except ObjectDoesNotExist:
            i_mcomment = self.create(username=username, name=prawComment.name, thread=prawComment.link_id, subreddit=prawComment.subreddit_id)
            redditFieldDict = i_mcomment.getRedditFieldDict()
            i_mcomment.addRedditFields(prawComment, redditFieldDict)
            i_mcomment.addOrUpdateTempField = "new"

        i_mcomment.save()
        return i_mcomment

# *****************************************************************************
# class mcomment(mbase, models.Model):
class mcomment(mbase):
    name                        = models.CharField(max_length=12)
    thread                      = models.CharField(max_length=12)
    subreddit                   = models.CharField(max_length=12, db_index=True)
    username                    = models.CharField(max_length=30, db_index=True)
    # properties
    pdeleted                    = models.BooleanField(default=False)
    puseradded                  = models.BooleanField(default=False)
    # Reddit fields
    rapproved_by                = models.CharField(max_length=21, default='', blank=True)
    rbanned_by                  = models.CharField(max_length=21, default='', blank=True)
    rid                         = models.CharField(max_length=10, default='', blank=True)
    rparent_id                  = models.CharField(max_length=16, default='', blank=True)
    rsubreddit_name_prefixed    = models.CharField(max_length=64, default='', blank=True)

    rbody                       = models.TextField(default='', blank=True)
    rmod_reports                = models.TextField(default='', blank=True)
    ruser_reports               = models.TextField(default='', blank=True)

    rcontroversiality           = models.IntegerField(default=0)
    rdowns                      = models.IntegerField(default=0)
    rscore                      = models.IntegerField(default=0)
    rups                        = models.IntegerField(default=0)

    rarchived                   = models.BooleanField(default=False)
    rstickied                   = models.BooleanField(default=False)

    rcreated                    = models.DecimalField(default=0, max_digits=12,decimal_places=1)
    rcreated_utc                = models.DecimalField(default=0, max_digits=12,decimal_places=1)
    redited                     = models.DecimalField(default=0, max_digits=12,decimal_places=1)

    objects = mcommentManager()

    # -------------------------------------------------------------------------
    def getRedditFieldDict(self):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        redditFieldDict = {
            # mThreadFieldName          redditFieldName                 convertMethodPtr
            'rapproved_by':             ("approved_by",                 self.getRedditUserNameAsString),  # special
            'rbanned_by':               ("banned_by",                   self.getRedditUserNameAsString),  # special
            'rid':                      ("id",                          None),      # string
            'rparent_id':               ("parent_id",                   None),      # string
            'rsubreddit_name_prefixed': ("subreddit_name_prefixed",     None),      # string

            'rbody':                    ("body",                        None),      # string
            'rmod_reports':             ("mod_reports",                 None),      # [[u'mod reported text', u'stp2007']],  OR [[u'Spam', u'stp2007']]
            'ruser_reports':            ("user_reports",                None),      # [[u'Text for other reason', 1]]        OR [[u'Spam', 1]]

            'rcontroversiality':        ("controversiality",            int),       # int
            'rdowns':                   ("downs",                       int),       # int
            'rscore':                   ("score",                       int),       # int
            'rups':                     ("ups",                         int),       # int

            'rarchived':                ("archived",                    None),      # bool
            'rstickied':                ("stickied",                    None),      # bool

            'rcreated':                 ("created",                     None),      # 1493534605.0,
            'rcreated_utc':             ("created_utc",                 None),      # 1493534605.0,

            'redited':                  ("edited",                      self.getEditedOrFalseValueAsZero),  # False or timestamp

        }
        return redditFieldDict

    # -------------------------------------------------------------------------
    def __str__(self):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        s = self.username
        # s += " [" + self.name + "]"
        # s += " [submisson_id=" + self.submission_id + "]"
        # s += " [parent_id=" + self.parent_id + "]"
        return format(s)

# # ----------------------------------------------------------------------------
# # COMMENT ATTRIBUTES COMING FROM A USER
# {
#   '_fetched'                      : True,
#   '_info_params'                  : {},
#   '_mod'                          : None,
#   '_reddit'                       : <praw.reddit.Reddit object at 0x7f4e76125eb8>,
#   '_replies'                      : [],
#   '_submission'                   : None,
#   'approved_by'                   : None,
#   'archived'                      : False,
#   'author_flair_css_class'        : None,
#   'author_flair_text'             : None,
#   'author'                        : Redditor(name='stp2007'),
#   'banned_by'                     : None,
#   'body_html'                     : '<div class="md"><p>Thanks</p>\n</div>',
#   'body'                          : 'Thanks\n',
#   'can_gild'                      : True,
#   'controversiality'              : 0,
#   'created_utc'                   : 1492955714.0,
#   'created'                       : 1492984514.0,
#   'distinguished'                 : None,
#   'downs'                         : 0,
#   'edited'                        : False,
#   'gilded'                        : 0,
#   'id'                            : 'dgn3ae1',
#   'likes'                         : None,
#   'link_author'                   : 'stp2007',
#   'link_id'                       : 't3_66ybk6',
#   'link_permalink'                : 'https://www.reddit.com/r/TheWarNerd/comments/66ybk6/radio_war_nerd_questions/',
#   'link_title'                    : 'Radio War Nerd questions',
#   'link_url'                      : 'https://www.reddit.com/r/TheWarNerd/comments/66ybk6/radio_war_nerd_questions/',
#   'mod_reports'                   : [],
#   'name'                          : 't1_dgn3ae1',
#   'num_comments'                  : 4,
#   'num_reports'                   : None,
#   'over_18'                       : False,
#   'parent_id'                     : 't1_dgmugew',
#   'quarantine'                    : False,
#   'removal_reason'                : None,
#   'report_reasons'                : None,
#   'saved'                         : False,
#   'score_hidden'                  : False,
#   'score'                         : 1,
#   'stickied'                      : False,
#   'subreddit_id'                  : 't5_3b93s',
#   'subreddit_name_prefixed'       : 'r/TheWarNerd',
#   'subreddit_type'                : 'public',
#   'subreddit'                     : Subreddit(display_name='TheWarNerd'),
#   'ups'                           : 1,
#   'user_reports'                  : []
# }

# # ----------------------------------------------------------------------------
# # COMMENT ATTRIBUTES COMING FROM A SUBMISSION
# {
#   '_fetched': True,
#   '_info_params': {},
#   '_mod': None,
#   '_reddit': <praw.reddit.Reddit object at 0x7fe085ff5f28>,
#   '_replies': <praw.models.comment_forest.CommentForest object at 0x7fe085433d68>,
#   '_submission': Submission(id='69a8hl'),
#   'approved_by': None,
#   'archived': False,
#   'author_flair_css_class': None,
#   'author_flair_text': None,
#   'author': Redditor(name='AutoModerator'),
#   'banned_by': None,
#   'body_html': '<div class="md"><p>As a reminder, this subreddit <a ',
#   'body': '\nAs a reminder, this subreddit [is for civil ',
#   'can_gild': True,
#   'controversiality': 0,
#   'created_utc': 1493930841.0,
#   'created': 1493959641.0,
#   'depth': 0,
#   'distinguished': 'moderator',
#   'downs': 0,
#   'edited': False,
#   'gilded': 0,
#   'id': 'dh4zdvm',
#   'likes': None,
#   'link_id': 't3_69a8hl',
#   'mod_reports': [],
#   'name': 't1_dh4zdvm',
#   'num_reports': None,
#   'parent_id': 't3_69a8hl',
#   'removal_reason': None,
#   'report_reasons': None,
#   'saved': False,
#   'score_hidden': True,
#   'score': 1,
#   'stickied': True,
#   'subreddit_id': 't5_2cneq',
#   'subreddit_name_prefixed': 'r/politics',
#   'subreddit_type': 'public',
#   'subreddit': Subreddit(display_name='politics'),
#   'ups': 1,
#   'user_reports': []
# }






<file_sep>/reddit/models/mbase.py
from __future__ import unicode_literals
from django.db import models
from ..config import clog
import praw
import inspect
# import pprint

CONST_CLIENT_ID                     = "kcksu9E4VgC0TQ"
CONST_SECRET                        = "<KEY>"
CONST_GRANT_TYPE                    = "client_credentials"
CONST_DEV_USERNAME                  = "OldDevLearningLinux"
CONST_DEV_PASSWORD                  = "<PASSWORD>"
CONST_USER_AGENT                    = "test app by /u/OldDevLearningLinux, ver 0.01"

# *****************************************************************************
class mbase(models.Model):
    # p_updatePriority                = models.IntegerField(default=0)
    #
    # class Meta:
    #     abstract = True

    # def __str__(self):
    #     return format("mbase")
    pass

    # # -------------------------------------------------------------------------
    # @staticmethod
    # def getDictOfClassFieldNames(classModel):
    #     mi = clog.dumpMethodInfo()
    #     clog.logger.info(mi)
    #
    #     rvDict = {}
    #     fields = classModel._meta.get_fields()
    #     for field in fields:
    #         rvDict[field.name] = None
    #     return rvDict

    # -------------------------------------------------------------------------
    @staticmethod
    def getPrawRedditInstance():
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        prawReddit = praw.Reddit(
            client_id=CONST_CLIENT_ID,
            client_secret=CONST_SECRET,
            user_agent=CONST_USER_AGENT,
            username=CONST_DEV_USERNAME,
            password=<PASSWORD>
        )
        return prawReddit

    # -------------------------------------------------------------------------
    def getRedditUserNameAsString(self, redditUser):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        if isinstance(redditUser, praw.models.Redditor): return redditUser.name         # if it is a praw Reddit object
        elif redditUser == None:                         return '[None]'
        else:                                            return redditUser              # will also return values of '[deleted]' or ''[removed]'

    # -------------------------------------------------------------------------
    # if None will return ''
    def getStringOrNoneAsEmptyString(self, inputString):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        if inputString == None: return ''
        else:                   return inputString

    # -------------------------------------------------------------------------
    # Reddit edited field has a value of False or a timestamp.
    # This method will return timestamp or a value of zero if it was False
    def getEditedOrFalseValueAsZero(self, inputString):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        if inputString == False:    return 0
        else:                       return inputString

    # -------------------------------------------------------------------------
    def addRedditFields(self, prawData, redditFieldDict):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        for mFieldName in redditFieldDict:
            redditFieldName     = redditFieldDict[mFieldName][0]  # ex: author
            convertMethodPtr    = redditFieldDict[mFieldName][1]  # ex: self.getRedditUserNameAsString
            redditValue         = getattr(prawData, redditFieldName)  # ex: prawData.author

            if convertMethodPtr: setattr(self, mFieldName, convertMethodPtr(redditValue))
            else:                setattr(self, mFieldName, redditValue)

    # -------------------------------------------------------------------------
    # return count of fields that were changed or zero
    def updateRedditFields(self, prawData, redditFieldDict):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        # clog.logger.debug("METHOD NOT COMPLETED")

        for mFieldName in redditFieldDict:
            redditFieldName     = redditFieldDict[mFieldName][0]  # ex: author
            convertMethodPtr    = redditFieldDict[mFieldName][1]  # ex: self.getRedditUserNameAsString
            redditValue         = getattr(prawData, redditFieldName)  # ex: prawData.author

            # Instead of only putting value in self
            # - get current value in self
            # - see if different then new value
            # - if different archive/backup current value
            # - then replace current value with new value

            # if convertMethodPtr: setattr(self, mFieldName, convertMethodPtr(redditValue))
            # else:                setattr(self, mFieldName, redditValue)


        return 0




<file_sep>/reddit/lib/checkInternetStatus.py
#!/usr/bin/env python
import requests
import argparse
import time
# import os
# import sys
# import pprint

CONST_CONTINUE              = 0
CONST_FINI_SUCCESS          = 1
CONST_FINI_KEYBOARD_QUIT    = 2

# *****************************************************************************
def doRequestToUrl(url, count, verbose):
    processingStatus = CONST_CONTINUE

    try:
        r = requests.get(url)
        processingStatus = CONST_FINI_SUCCESS
    except KeyboardInterrupt:
        processingStatus = CONST_FINI_KEYBOARD_QUIT
    except:
        if verbose:
            print ("*** Try  #%d to url %s failed." % (count, url))
        else:
            print ("*** Try #%d failed." % (count))

    return processingStatus

# *****************************************************************************
def parseCommandLineArguments():

    parser = argparse.ArgumentParser()

    # --------------------
    # Positional arguments
    # args.url
    parser.add_argument("url", help="URL to test");

    # type and valid choices specified
    # args.iterations
    parser.add_argument("iterations", help="number of iterations to test. 0 = run forever.", type=int);

    # --------------------
    # Optional arguments

    # Both -v and --verbose acceptable
    # if specified then args.verbose will equal true otherwise false
    # args.verbose
    parser.add_argument("-v", "--verbose", help="verbosity of output message", action="store_true");

    # Different way to specify option arguments
    # args.signage
    # allows input = -d  -> args.delay = 1
    # allows input = -dd  -> args.delay = 2
    # allows input = -ddd  -> args.delay = 3
    # default value if argument not provided
    parser.add_argument("-d", "--delay", help="delay in seconds between tests", action="count", default=0);

    args = parser.parse_args()

    print ("")
    print ("args.url = %s" %(args.url))
    print ("args.iterations = %d" %(args.iterations))
    print ("args.verbose = %s" %(args.verbose))
    print ("args.delay = %d" %(args.delay))

    return args


# *****************************************************************************
# url               ex http://www.example.com
# verbose           True or False
# iterations        int, 0 = forever
# delay             int, seconds, 0 = no delay
def checkInternet(url, verbose, iterations, delay):
    print("===================")

    processingStatus = CONST_CONTINUE
    count = 1
    while processingStatus == CONST_CONTINUE:
        processingStatus = doRequestToUrl(url, count, verbose);
        if iterations > 0 and count >= iterations:
            print("Fini: %d iterations tested" % (iterations))
            break
        count += 1

        if delay > 0 and processingStatus == CONST_CONTINUE:
            time.sleep(delay)


    if processingStatus == CONST_FINI_SUCCESS:
        print("=================== success")
    elif processingStatus == CONST_FINI_KEYBOARD_QUIT:
        print("=================== keyboard quit")





# *****************************************************************************
if __name__ == "__main__":
    args = parseCommandLineArguments()
    checkInternet(args.url, args.verbose, args.iterations, args.delay)



















<file_sep>/reddit/views/vuser.py
from django.http import HttpResponse
from django.shortcuts import redirect, render
from django.core.exceptions import ObjectDoesNotExist
from django.utils.safestring import mark_safe
from ..config import clog
from ..models import muser
from ..forms import fuser
# import pprint

# *****************************************************************************
def list(request, xData=None):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    qs = muser.objects.filter(ppoi=True).order_by('pprioritylevel','-pcommentsupdatetimestamp')
    return render(request, 'vuser_list.html', {'users': qs, 'rightCol': mark_safe(request.session.get(xData, ''))})

# *****************************************************************************
def delAll(request):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    vs = ''
    qs = muser.objects.all()
    vs += str(qs.count()) + " musers deleted"
    qs.delete()

    clog.logger.info(vs)
    sessionKey = 'blue'
    request.session[sessionKey] = vs
    return redirect('vbase.main', xData=sessionKey)

# *****************************************************************************
def formNewPoiUser(request):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    if request.method == 'POST':                # if this is a POST request we need to process the form data
        form = fuser.fNewPoiUser(request.POST)  # create a form instance and populate it with data from the request:
        if form.is_valid():                     # check whether it's valid:
            # process the data in form.cleaned_data as required
            # pprint.pprint(form.cleaned_data)

            # add user as poi
            prawReddit = muser.getPrawRedditInstance()
            prawRedditor = prawReddit.redditor(form.cleaned_data['poiUser'])

            i_muser = muser.objects.addOrUpdate(prawRedditor)
            i_muser.ppoi = True
            i_muser.save()

            vs = form.cleaned_data['poiUser']
            if i_muser.addOrUpdateTempField == "new": vs += ' poi user added.'
            else:                                     vs += ' poi user already existed.'

            clog.logger.info(vs)
            sessionKey = 'blue'
            request.session[sessionKey] = vs
            return redirect('vbase.main', xData=sessionKey)
    else:                                       # if a GET (or any other method) we'll create a blank form
        form = fuser.fNewPoiUser()

    return render(request, 'fSimpleSubmitForm.html', {'form': form})




















<file_sep>/reddit/tasks/treddit.py
from celery import task
from django.core.exceptions import ObjectDoesNotExist
import time
from ..config import clog
from ..models import mcomment
from ..models import msubreddit
from ..models import mthread
from ..models import muser
from .tbase import getBaseP, getBaseC
import praw
import prawcore
# import pprint

# --------------------------------------------------------------------------
@task()
def TASK_updateUsersForAllComments(numberToProcess):
    mi = clog.dumpMethodInfo()
    ts = time.time()

    # create PRAW prawReddit instance
    prawReddit = mcomment.getPrawRedditInstance()
    countUsersAdded = 0

    qsCount = mcomment.objects.filter(puseradded=False).count()
    clog.logger.info("%s %d comments pending, processing %d" % (getBaseP(mi),  qsCount, numberToProcess))
    while qsCount > 0:
        # Look at first result
        i_mcomment = mcomment.objects.filter(puseradded=False)[0]

        # AddOrUpdate that user
        prawRedditor = prawReddit.redditor(i_mcomment.username)
        i_muser = muser.objects.addOrUpdate(prawRedditor)
        if i_muser.addOrUpdateTempField == "new":
            countUsersAdded += 1

        # set puseradded True for any false comments for that user
        qs2 = mcomment.objects.filter(puseradded=False).filter(username=i_mcomment.username)
        for item in qs2:
            item.puseradded = True
            item.save()

        # are there any puseradded False comments left
        qsCount = mcomment.objects.filter(puseradded=False).count()

        numberToProcess -= 1
        if numberToProcess <= 0:
            break

    clog.logger.info("%s %d comments pending, %d users added" % (getBaseC(mi, ts), qsCount, countUsersAdded))
    return ""

# --------------------------------------------------------------------------
@task()
def TASK_updateThreadCommentsByForest(numberToProcess):
    mi = clog.dumpMethodInfo()
    ts = time.time()

    # create PRAW prawReddit instance
    prawReddit = mcomment.getPrawRedditInstance()
    countCommentsAdded = 0

    qs = mthread.objects.filter(pdeleted=False, pforestgot=False).order_by("-rcreated")
    clog.logger.info("%s %d threads pending, processing %d" % (getBaseP(mi),  qs.count(), numberToProcess))

    countNew = 0
    countOldChanged = 0
    countOldUnchanged = 0
    countDeleted = 0

    for i_mthread in qs:
        try:
            params={};
            # params['before'] = i_mthread.getBestCommentBeforeValue(????)

            prawSubmissionObject = prawReddit.submission(id=i_mthread.fullname[3:])
            prawSubmissionObject.comment_sort = "new"

            # prawSubmissionObject.comments.replace_more(limit=None)
            prawSubmissionObject.comments.replace_more(limit=16)

            for prawComment in prawSubmissionObject.comments.list():
                if prawComment.author == None:
                    countDeleted += 1
                else:
                    i_mcomment = mcomment.objects.addOrUpdate(prawComment.author.name, prawComment)
                    if i_mcomment.addOrUpdateTempField == "new":            countNew += 1
                    if i_mcomment.addOrUpdateTempField == "oldUnchanged":   countOldUnchanged += 1
                    if i_mcomment.addOrUpdateTempField == "oldChanged":     countOldChanged += 1
        except praw.exceptions.APIException as e:
            clog.logger.info("%s %s PRAW_APIException: error_type = %s, message = %s" % (getBaseP(mi), username, e.error_type, e.message))

        i_mthread.pforestgot = True
        i_mthread.pcount += countNew
        i_mthread.save()

        numberToProcess -= 1
        if numberToProcess <= 0:
            break

    clog.logger.info("%s %d new comments, %d old, %d oldChanged, %d deleted" % (getBaseC(mi, ts), countNew, countOldUnchanged, countOldChanged, countDeleted))
    return ""

# --------------------------------------------------------------------------
@task()
def TASK_updateCommentsForUser(username):
    mi = clog.dumpMethodInfo()
    ts = time.time()

    try:
        i_muser = muser.objects.get(name=username)
        clog.logger.info("%s %s" % (getBaseP(mi), username))

        prawReddit = i_muser.getPrawRedditInstance()

        params={};
        params['before'] = i_muser.getBestCommentBeforeValue(prawReddit)
        # clog.logger.info("before = %s" % (params['before']))

        # iterate through submissions saving them
        countNew = 0
        countOldChanged = 0
        countOldUnchanged = 0
        try:
            for prawComment in prawReddit.redditor(i_muser.name).comments.new(limit=None, params=params):
                i_mcomment = mcomment.objects.addOrUpdate(i_muser.name, prawComment)
                if i_mcomment.addOrUpdateTempField == "new":            countNew += 1
                if i_mcomment.addOrUpdateTempField == "oldUnchanged":   countOldUnchanged += 1
                if i_mcomment.addOrUpdateTempField == "oldChanged":     countOldChanged += 1
                i_mcomment.puseradded = True
                i_mcomment.save()
        except praw.exceptions.APIException as e:
            clog.logger.info("%s %s APIException: error_type = %s, message = %s" % (getBaseC(mi, ts), username, e.error_type, e.message))
        except prawcore.exceptions.NotFound as e:
            clog.logger.info("%s %s PrawcoreException: %s, *** USER PROBABLY DELETED ***" % (getBaseC(mi, ts), username, e))
            i_muser.pdeleted = True
        except prawcore.exceptions.PrawcoreException as e:
            clog.logger.info("%s %s PrawcoreException: %s, *** USER PROBABLY SUSPENDED ***" % (getBaseC(mi, ts), username, e))
            i_muser.pdeleted = True

        beforeWarningString = ' [before]'
        if countNew == 0 and params['before'] != '':
            beforeWarningString = ' [** %s **]' % (params['before'])
        clog.logger.info("%s %s %d new, %d old, %d oldC %s" % (getBaseC(mi, ts), username, countNew, countOldUnchanged, countOldChanged, beforeWarningString))

        i_muser.commentsUpdated(countNew, countOldUnchanged, countOldChanged)
    except ObjectDoesNotExist:
        clog.logger.info("%s %s %s" % (getBaseC(mi, ts), username, "ERROR does not exist"))
    return ""

# --------------------------------------------------------------------------
@task()
def TASK_updateCommentsForAllUsers(userCount, priority):
    mi = clog.dumpMethodInfo()
    ts = time.time()

    qs = None
    if priority >= 0:
        qs = muser.objects.filter(ppoi=True).filter(pprioritylevel=priority).filter(pdeleted=False).order_by('pcommentsupdatetimestamp','name')[:userCount]
    else:
        qs = muser.objects.filter(ppoi=True).filter(pdeleted=False).order_by('pcommentsupdatetimestamp','name')[:userCount]

    if qs.count() > 0:
        clog.logger.info("%s %d users being processed, priority = %d" % (getBaseP(mi), qs.count(), priority))
        countOfTasksSpawned = 0
        for i_muser in qs:
            TASK_updateCommentsForUser.delay(i_muser.name)
            countOfTasksSpawned += 1
        clog.logger.info("%s %d tasks spawned" % (getBaseC(mi, ts), countOfTasksSpawned))
    else:
        clog.logger.info("%s zero users available at priority = %d" % (getBaseC(mi, ts), priority))
    return ""

# --------------------------------------------------------------------------
@task()
def TASK_updateThreadsForSubreddit(subredditName):
    mi = clog.dumpMethodInfo()
    ts = time.time()

    try:
        i_msubreddit = msubreddit.objects.get(name=subredditName)
        clog.logger.info("%s %s" % (getBaseP(mi), subredditName))

        prawReddit = i_msubreddit.getPrawRedditInstance()

        params={};
        params['before'] = i_msubreddit.getThreadsBestBeforeValue(prawReddit)
        # clog.logger.info("before = %s" % (params['before']))

        countNew = 0
        countOldUnchanged = 0
        countOldChanged = 0
        try:
            for prawThread in prawReddit.subreddit(i_msubreddit.name).new(limit=None, params=params):
                i_mthread = mthread.objects.addOrUpdate(i_msubreddit, prawThread)
                if i_mthread.addOrUpdateTempField == "new":             countNew += 1
                if i_mthread.addOrUpdateTempField == "oldUnchanged":    countOldUnchanged += 1
                if i_mthread.addOrUpdateTempField == "oldChanged":      countOldChanged += 1
        except praw.exceptions.APIException as e:
            clog.logger.info("%s %s PRAW_APIException: error_type = %s, message = %s" % (getBaseC(mi, ts), subredditName, e.error_type, e.message))
        clog.logger.info("%s %s: %d new, %d old, %d oldChanged" % (getBaseC(mi, ts), subredditName, countNew, countOldUnchanged, countOldChanged))

        i_msubreddit.threadsUpdated(countNew, countOldUnchanged, countOldChanged)
    except ObjectDoesNotExist:
        clog.logger.info("%s %s, %s" % (getBaseC(mi, ts), subredditName, "ERROR does not exist"))
    return ""

# --------------------------------------------------------------------------
@task()
def TASK_updateThreadsForAllSubreddits(subredditCount, priority):
    mi = clog.dumpMethodInfo()
    ts = time.time()

    qs = msubreddit.objects.filter(ppoi=True).filter(pprioritylevel=priority).order_by('pthreadupdatetimestamp','name')[:subredditCount]
    if qs.count() > 0:
        clog.logger.info("%s %d subreddits being processed, priority = %d" % (getBaseP(mi), qs.count(), priority))
        countOfTasksSpawned = 0
        for i_msubreddit in qs:
            TASK_updateThreadsForSubreddit.delay(i_msubreddit.name)
            countOfTasksSpawned += 1
        clog.logger.info("%s %d tasks spawned" % (getBaseC(mi, ts), countOfTasksSpawned))
    else:
        clog.logger.info("%s zero subreddits available at priority = %d" % (getBaseC(mi, ts), priority))
    return ""




<file_sep>/reddit/models/__init__.py
from .mbase import mbase
from .muser import muser
from .msubreddit import msubreddit
from .mthread import mthread
from .mcomment import mcomment













<file_sep>/reddit/urls.py
from django.conf.urls import url
from .views import vbase
from .views import vsubreddit
from .views import vthread
from .views import vuser
from .views import vcomment
from .views import vanalysis

urlpatterns = [
    url(r'^vbase/main/$',                               vbase.main,                         name='vbase.main'),
    url(r'^vbase/main/xData/(?P<xData>\w+)/$',          vbase.main,                         name='vbase.main'),
    url(r'^vbase/test/$',                               vbase.test,                         name='vbase.test'),
    url(r'^vbase/map/$',                                vbase.map,                          name='vbase.map'),

    url(r'^vcomment/user/(?P<username>\w+)/$',          vcomment.user,                      name='vcomment.user'),

    url(r'^vuser/list/$',                               vuser.list,                         name='vuser.list'),
    url(r'^vuser/list/xData/(?P<xData>\w+)/$',          vuser.list,                         name='vuser.list'),
    url(r'^vuser/formNewPoiUser/$',                     vuser.formNewPoiUser,               name='vuser.formNewPoiUser'),

    url(r'^vsubreddit/list/$',                          vsubreddit.list,                    name='vsubreddit.list'),
    url(r'^vsubreddit/list/xData/(?P<xData>\w+)/$',     vsubreddit.list,                    name='vsubreddit.list'),
    url(r'^vsubreddit/formNewPoiSubreddit/$',           vsubreddit.formNewPoiSubreddit,     name='vsubreddit.formNewPoiSubreddit'),

    url(r'^vthread/list/(?P<subreddit>\w+)/$',          vthread.list,                       name='vthread.list'),

    url(r'^vanalysis/poiUsersOfSubreddit/(?P<subreddit>\w+)/(?P<minNumComments>\w+)/$',     vanalysis.poiUsersOfSubreddit,          name='vanalysis.poiUsersOfSubreddit'),
    url(r'^vanalysis/moderatorsOfSubreddit/(?P<subreddit>\w+)/$',                           vanalysis.moderatorsOfSubreddit,        name='vanalysis.moderatorsOfSubreddit'),


]






# url(r'^vuser/delAll/$',                             vuser.delAll,                   name='vuser.delAll'),
# url(r'^vsubreddit/delAll/$',                        vsubreddit.delAll,              name='vsubreddit.delAll'),
# url(r'^vthread/delAll/$',                           vthread.delAll,                 name='vthread.delAll'),

<file_sep>/reddit/views/vsubreddit.py
from django.http import HttpResponse
from django.shortcuts import redirect, render
from django.core.exceptions import ObjectDoesNotExist
from django.utils.safestring import mark_safe
from ..config import clog
from ..models import msubreddit
from ..forms import fsubreddit
# import pprint

# *****************************************************************************
def list(request, xData=None):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    # qs = msubreddit.objects.all().order_by('name')
    qs = msubreddit.objects.all().order_by('pprioritylevel','-pthreadupdatetimestamp')
    return render(request, 'vsubreddit_list.html', {'subreddits': qs, 'rightCol': mark_safe(request.session.get(xData, ''))})

# *****************************************************************************
def delAll(request):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    vs = ''
    qs = msubreddit.objects.all()
    vs += str(qs.count()) + " msubreddits deleted"
    qs.delete()

    clog.logger.info(vs)
    sessionKey = 'blue'
    request.session[sessionKey] = vs
    return redirect('vbase.main', xData=sessionKey)

# *****************************************************************************
def formNewPoiSubreddit(request):
    mi = clog.dumpMethodInfo()
    clog.logger.info(mi)

    if request.method == 'POST':                            # if this is a POST request we need to process the form data
        form = fsubreddit.fNewPoiSubreddit(request.POST)    # create a form instance and populate it with data from the request:
        if form.is_valid():                                 # check whether it's valid:
            # process the data in form.cleaned_data as required
            # pprint.pprint(form.cleaned_data)

            # add subreddit as poi
            prawReddit = msubreddit.getPrawRedditInstance()
            prawSubreddit = prawReddit.subreddit(form.cleaned_data['poiSubreddit'])

            i_msubreddit = msubreddit.objects.addOrUpdate(form.cleaned_data['poiSubreddit'], prawSubreddit)
            i_msubreddit.ppoi = True
            i_msubreddit.save()

            vs = form.cleaned_data['poiSubreddit']
            if i_msubreddit.addOrUpdateTempField == "new": vs += ' poi poiSubreddit added.'
            else:                                          vs += ' poi poiSubreddit already existed.'

            clog.logger.info(vs)
            sessionKey = 'blue'
            request.session[sessionKey] = vs
            return redirect('vbase.main', xData=sessionKey)
    else:                                                   # if a GET (or any other method) we'll create a blank form
        form = fsubreddit.fNewPoiSubreddit()

    return render(request, 'fSimpleSubmitForm.html', {'form': form})

























<file_sep>/reddit/models/muser.py
from __future__ import unicode_literals
from django.db import models
from django.core.exceptions import ObjectDoesNotExist
from .mbase import mbase
from ..config import clog
from datetime import datetime
from django.utils import timezone
import praw
# import pprint

# *****************************************************************************
class muserManager(models.Manager):
    def addOrUpdate(self, prawRedditor):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        # NOTE: ENDED UP WITH MULITIPLE USERS WITH SAME NAME BECAUSE TWO TASKS operating
        # AT THE SAME TIME CREATED BOTH USERS. THIS CAUSES THE GET CALL BELOW TO FAIL.
        # FOR NOW, BEFORE FIXING THAT ISSUE AND WRITING CODE TO FIX DUPLICATE NAMES AM
        # CHANGING QUERY TO FILTER AND HACKING IN DUPLICATE OF OJBECT DOE SNOT EXIST CODE
        try:
            qs = muser.objects.filter(name=prawRedditor.name)
            if qs.count() > 0:
                i_muser = qs[0]
                redditFieldDict = i_muser.getRedditFieldDict()
                changedCount = i_muser.updateRedditFields(prawRedditor, redditFieldDict)
                if changedCount == 0: i_muser.addOrUpdateTempField = "oldUnchanged"
                else:                 i_muser.addOrUpdateTempField = "oldChanged"
            else: # hack put in object does not exist code below
                i_muser = self.create(name=prawRedditor.name)
                redditFieldDict = i_muser.getRedditFieldDict()
                i_muser.addRedditFields(prawRedditor, redditFieldDict)
                i_muser.addOrUpdateTempField = "new"
        except ObjectDoesNotExist:
            i_muser = self.create(name=prawRedditor.name)
            redditFieldDict = i_muser.getRedditFieldDict()
            i_muser.addRedditFields(prawRedditor, redditFieldDict)
            i_muser.addOrUpdateTempField = "new"

        i_muser.save()
        return i_muser

# *****************************************************************************
# class muser(mbase, models.Model):
class muser(mbase):
    name                        = models.CharField(max_length=30, db_index=True)
    # properties
    ppoi                        = models.BooleanField(default=False)
    pprioritylevel              = models.IntegerField(default=0)
    pcommentsupdatetimestamp    = models.DateTimeField(default=timezone.make_aware(datetime(2000, 1, 1, 1, 0, 0)))
    pupdateswithnochanges       = models.IntegerField(default=0)
    pcountnew                   = models.IntegerField(default=0)
    pcountOldUnchanged          = models.IntegerField(default=0)
    pcountOldChanged            = models.IntegerField(default=0)
    pdeleted                    = models.BooleanField(default=False)
    # Redditor fields
    r_path                      = models.CharField(max_length=40, default='', blank=True)

    objects = muserManager()

    # -------------------------------------------------------------------------
    def getRedditFieldDict(self):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        redditFieldDict = {
            # mThreadFieldName      redditFieldName     convertMethodPtr
            'r_path':               ("_path",           None),      # string
        }
        return redditFieldDict

    # -------------------------------------------------------------------------
    def __str__(self):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)
        s = self.name
        if self.ppoi: s += " (ppoi)"
        return format(s)

    # --------------------------------------------------------------------------
    def getBestCommentBeforeValue(self, prawReddit):
        from .mcomment import mcomment
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        youngestRV = ''
        qs = mcomment.objects.filter(username=self.name, pdeleted=False).order_by('-name')
        for item in qs:
            try:
                # CAN I BATCH THIS QUERY UP TO GET MULTIPLE COMMENTS FROM REDDIT AT ONCE?
                isValidComment = prawReddit.comment(item.name[3:])
                # if isValidComment.author != None:
                # if isValidComment.author != None and isValidComment.author.name != '[deleted]' and isValidComment.author.name != '[removed]':
                if isValidComment.author != None and isValidComment.author != '[deleted]' and isValidComment.author != '[removed]':
                    youngestRV = item.name
                    # clog.logger.debug("youngestRV = %s" % (youngestRV))
                    break
                else: # Update item as deleted.
                    item.pdeleted = True
                    item.save()
                    # clog.logger.debug("userCommentIndex %s flagged as deleted" % (item.name))
            except praw.exceptions.APIException as e:
                clog.logger.error("PRAW APIException: error_type = %s, message = %s" % (e.error_type, e.message))

        return youngestRV

    # --------------------------------------------------------------------------
    def commentsUpdated(self, countNew, countOldUnchanged, countOldChanged):
        mi = clog.dumpMethodInfo()
        # self.precentlyupdated =True

        if countNew > 0:
            self.pprioritylevel -= 1
            self.pupdateswithnochanges = 0
            if self.pprioritylevel < 0:
                self.pprioritylevel = 0;
        else:
            self.pprioritylevel += 1
            self.pupdateswithnochanges += 1
            if self.pprioritylevel > 3:
                self.pprioritylevel = 3;

        self.pcountnew          = countNew
        self.pcountOldUnchanged = countOldUnchanged
        self.pcountOldChanged   = countOldChanged

        # self.pcommentsupdatetimestamp = datetime.now()
        self.pcommentsupdatetimestamp = timezone.now()
        self.save()
        return

# # ----------------------------------------------------------------------------
# # REDDITOR attributes
# # Ex
# # rUser = reddit.redditor('stp2007')
# # logger.info(rUser.name) # to make it non-lazy
# # pprint.pprint(vars(rUser))
# {
#   '_fetched':           False,
#   '_info_params':       {},
#   '_listing_use_sort':  True,
#   '_path':              'user/stp2007/',
#   '_reddit':            <praw.reddit.Reddit object at 0x7f45d99f9128>,
#   'name':               'stp2007'
#  }







<file_sep>/reddit/tasks/ttest.py
from celery import task
import time
from django.db.models import Count
from ..config import clog
from ..models import mcomment
from ..models import msubreddit
from ..models import mthread
from ..models import muser
from .tbase import getBaseP, getBaseC
# import pprint

# --------------------------------------------------------------------------
@task()
def TASK_testLogLevels():
    mi = clog.dumpMethodInfo()
    ts = time.time()

    clog.logger.critical("%s %s" % (getBaseC(mi, ts), 'critical'))
    clog.logger.error   ("%s %s" % (getBaseC(mi, ts), 'error'))
    clog.logger.warning ("%s %s" % (getBaseC(mi, ts), 'warning'))
    clog.logger.info    ("%s %s" % (getBaseC(mi, ts), 'info'))
    clog.logger.debug   ("%s %s" % (getBaseC(mi, ts), 'debug'))
    clog.logger.trace   ("%s %s" % (getBaseC(mi, ts), 'trace'))
    return ""

# --------------------------------------------------------------------------
@task()
def TASK_testForDuplicateComments():
    mi = clog.dumpMethodInfo()
    ts = time.time()
    clog.logger.info("%s" % (getBaseP(mi)))

    qs = mcomment.objects.values('username','name','thread','subreddit').annotate(num_count=Count('name')).filter(num_count__gt=1)
    if qs.count() > 0:
        clog.logger.info("%s WARNING: at least %d duplicate comments" % (getBaseC(mi, ts), qs.count()))
        for i_mac in qs:
            value = int(i_mac['num_count'])
            while value > 1:
                clog.logger.info("%s WARNING: deleting %s, thread %s, subreddit %s, username %s" % (getBaseC(mi, ts), i_mac['name'], i_mac['thread'], i_mac['subreddit'], i_mac['username']))
                qs2 = mcomment.objects.filter(username=i_mac['username'], name=i_mac['name'], thread=i_mac['thread'], subreddit=i_mac['subreddit'])
                qs2.delete()
                value -= 1
    else:
        clog.logger.info("%s no duplicate comments found" %    (getBaseC(mi, ts)))
    return ""

# --------------------------------------------------------------------------
@task()
def TASK_testForDuplicateThreads():
    mi = clog.dumpMethodInfo()
    ts = time.time()
    clog.logger.info("%s" % (getBaseP(mi)))

    qs = mthread.objects.values('fullname').annotate(num_count=Count('fullname')).filter(num_count__gt=1)
    if qs.count() > 0:
        clog.logger.info("%s WARNING: at least %d duplicate threads" % (getBaseC(mi, ts), qs.count()))
        for i_mac in qs:
            value = int(i_mac['num_count'])
            while value > 1:
                clog.logger.info("%s WARNING: deleting %s" % (getBaseC(mi, ts), i_mac['fullname']))
                qs2 = mthread.objects.filter(fullname=i_mac['fullname'])
                qs2.delete()
                value -= 1
    else:
        clog.logger.info("%s no duplicate threads found" %    (getBaseC(mi, ts)))
    return ""

# --------------------------------------------------------------------------
@task()
def TASK_testForDuplicateUsers():
    mi = clog.dumpMethodInfo()
    ts = time.time()
    clog.logger.info("%s" % (getBaseP(mi)))

    qs = muser.objects.values('name').annotate(num_count=Count('name')).filter(num_count__gt=1)
    if qs.count() > 0:
        clog.logger.info("%s WARNING: at least %d duplicate users" % (getBaseC(mi, ts), qs.count()))
        for i_mac in qs:
            value = int(i_mac['num_count'])
            while value > 1:
                clog.logger.info("%s WARNING: deleting %s" % (getBaseC(mi, ts), i_mac['name']))
                qs2 = muser.objects.filter(name=i_mac['name']).order_by('-pcommentsupdatetimestamp')
                qs2.delete()
                value -= 1
    else:
        clog.logger.info("%s no duplicate users found" %    (getBaseC(mi, ts)))
    return ""




<file_sep>/reddit/lib/cLogger.py
#!/usr/bin/env python
import logging
import datetime
import time
import inspect
import os
# import pprint

cLoggerLevel_CRITICAL                       = logging.CRITICAL      # 50
cLoggerLevel_ERROR                          = logging.ERROR         # 40
cLoggerLevel_WARNING                        = logging.WARNING       # 30
cLoggerLevel_INFO                           = logging.INFO          # 20
cLoggerLevel_DEBUG                          = logging.DEBUG         # 10
cLoggerLevel_TRACE                          = 6                     # 6, value of 5 conflicts with Celery "SUBDEBUG" level
cLoggerLevel_NOTSET                         = logging.NOTSET        # 0

cLoggerFile_archiveOlder                    = 100
cLoggerFile_overwriteOlder                  = 200
cLoggerFile_appendOlder                     = 300

cLoggerFilter_None                          = 10
cLoggerFilter_SpecificLevelOnly             = 20
cLoggerFilter_GreaterThanOrEqualToLevel     = 30

# *****************************************************************************
# cLoggerFilter_SpecificLevelOnly
class filterSpecificLevelOnly(object):
    def __init__(self, level):
        self.__level = level

    def filter(self, logRecord):
        if logRecord.levelno == self.__level: return True
        else:                                 return False

# *****************************************************************************
# cLoggerFilter_GreaterThanOrEqualToLevel
class filterGreaterThanOrEqualToLevelOnly(object):
    def __init__(self, level):
        self.__level = level

    def filter(self, logRecord):
        if logRecord.levelno >= self.__level: return True
        else:                                 return False

# *****************************************************************************
class cLogger(object):

    # --------------------------------------------------------------------------
    def __init__(self, name):
        self.name = name
        self.baseLoggerLevel        = logging.CRITICAL
        self.handlerLevels          = {}
        self.loggerInfoLevel        = logging.DEBUG
        self.methodInfoLevel        = logging.DEBUG
        self.logger = logging.getLogger(self.name)
        self.logger.setLevel(self.baseLoggerLevel)
        self.funcdict = {}

    # --------------------------------------------------------------------------
    def getArchiveFilename(self, fNameKey, archiveIndex, fNameType):
        return "%s%03d.%s" % (fNameKey, archiveIndex, fNameType)

    # --------------------------------------------------------------------------
    def archiveOlderVersions(self, pathFileName):
        # get path and filename
        pathFileTuple = pathFileName.rpartition('/')    # if success: path = pathFileTuple[0], filename = pathFileTuple[2]
        # get fname and ftype
        nameTypeTuple = pathFileTuple[2].partition('.') # if success: fname = nameTypeTuple[0], type = nameTypeTuple[2]
        # print("archiveOlderVersions A")

        if not (pathFileTuple[0] and pathFileTuple[1] and nameTypeTuple[0] and nameTypeTuple[1]):
            self.logger.critical("Error splitting %s into path and file and or %s into name and type" % (pathFileName, pathFileTuple[2]))
        else:
            # print("archiveOlderVersions B")
            fNameKey    = nameTypeTuple[0]         # if success: fNameKey = element 0
            fNameType   = nameTypeTuple[2]         # if success: fNameType = element 2
            # get names of all previously archived files
            fileList = []
            for fname in os.listdir(pathFileTuple[0]):
                # print("archiveOlderVersions C: fname = %s" % (fname))
                if fname.startswith(fNameKey):
                    # print("archiveOlderVersions D: appending = %s" % (fname))
                    fileList.append(fname)

            # remove base filename if in list
            if (pathFileTuple[2] in fileList):
                # print("archiveOlderVersions E: removing = %s" % (pathFileTuple[2]))
                fileList.remove(pathFileTuple[2])

            # reverse sort fileList
            fileList.sort(reverse=True)

            # len(fileList) = number of archived files found.
            # therefore increment by one for archive_index for renaming
            archiveIndex = len(fileList) + 1
            # print("archiveOlderVersions f: archiveIndex = %s" % (archiveIndex))

            # fileList is sorted with oldest names first: ex log_004.txt, log_003.txt, log_002.txt, log_001.txt
            # for each rename with older archiveIndex
            for x in fileList:
                # print("archiveOlderVersions g renaming %s" % (x))
                archiveFilename = self.getArchiveFilename(fNameKey, archiveIndex, fNameType)
                # self.logger.critical("%s to be renamed to %s" % (x, archiveFilename))
                os.rename(pathFileTuple[0]+'/'+x, pathFileTuple[0]+'/'+archiveFilename)
                archiveIndex -= 1

            # finally rename newest log file
            archiveFilename = self.getArchiveFilename(fNameKey, archiveIndex, fNameType)
            if (os.path.isfile(pathFileName)):
                os.rename(pathFileName, pathFileTuple[0]+'/'+archiveFilename)

    # --------------------------------------------------------------------------
    def addConsoleLogger(self, loggingLevel, name, filterVal):
        handler = logging.StreamHandler()
        handler.setLevel(loggingLevel)
        self.handlerLevels[name]=loggingLevel
        formatter = logging.Formatter(self.getFormatString(), datefmt='%H:%M:%S')
        formatter.converter = time.localtime
        handler.setFormatter(formatter)
        if   filterVal == cLoggerFilter_GreaterThanOrEqualToLevel:  handler.addFilter(filterGreaterThanOrEqualToLevelOnly(loggingLevel))
        elif filterVal == cLoggerFilter_SpecificLevelOnly:          handler.addFilter(filterSpecificLevelOnly(loggingLevel))
        self.logger.addHandler(handler)
        if loggingLevel < self.baseLoggerLevel:
            self.baseLoggerLevel = loggingLevel
            self.logger.setLevel(self.baseLoggerLevel)

    # --------------------------------------------------------------------------
    def getFileHandler(self, pathFileName, loggingLevel, name, archiveType, filterVal):
        fileMode = 'w'
        if archiveType == cLoggerFile_archiveOlder:
            self.archiveOlderVersions(pathFileName)
        elif archiveType == cLoggerFile_appendOlder:
            fileMode = 'a'
        # handler = logging.FileHandler(pathFileName, mode=fileMode, encoding=None, delay=False)
        handler = logging.FileHandler(pathFileName, mode=fileMode, encoding=None, delay=True)
        handler.setLevel(loggingLevel)
        self.handlerLevels[name]=loggingLevel
        formatter = logging.Formatter(self.getFormatString(), datefmt='%H:%M:%S')
        formatter.converter = time.localtime
        handler.setFormatter(formatter)
        if   filterVal == cLoggerFilter_GreaterThanOrEqualToLevel:  handler.addFilter(filterGreaterThanOrEqualToLevelOnly(loggingLevel))
        elif filterVal == cLoggerFilter_SpecificLevelOnly:          handler.addFilter(filterSpecificLevelOnly(loggingLevel))
        return handler

    # --------------------------------------------------------------------------
    def addFileLogger(self, pathFileName, loggingLevel, name, archiveType, filterVal):
        handler = self.getFileHandler(pathFileName, loggingLevel, name, archiveType, filterVal)
        self.logger.addHandler(handler)
        if loggingLevel < self.baseLoggerLevel:
            self.baseLoggerLevel = loggingLevel
            self.logger.setLevel(self.baseLoggerLevel)

    # --------------------------------------------------------------------------
    def addTraceLoggingLevel(self):
        logging.TRACE = cLoggerLevel_TRACE
        logging.addLevelName(logging.TRACE, "TRACE")
        def trace(self, message, *args, **kws):
            if self.isEnabledFor(logging.TRACE):
                self._log(logging.TRACE, message, args, **kws)
        logging.Logger.trace = trace

    # --------------------------------------------------------------------------
    def generateFuncDict(self):
        self.funcdict['CRITICAL']   = self.logger.critical
        self.funcdict['ERROR']      = self.logger.error
        self.funcdict['WARNING']    = self.logger.warning
        self.funcdict['INFO']       = self.logger.info
        self.funcdict['DEBUG']      = self.logger.debug
        self.funcdict['TRACE']      = self.logger.trace

    # --------------------------------------------------------------------------
    # ref: https://docs.python.org/2/library/logging.html#logrecord-attributes
    # %(levelname).1s has a precision of 1 so that many characters
    def getFormatString(self):
        return '%(asctime)-8s %(levelname).1s %(filename)-15s (line %(lineno)4s): %(message)s'

    # --------------------------------------------------------------------------
    def getMethodPtr(self, loggingLevel):
        # self.logger.info("getMethodPtr")
        # self.logger.info("loggingLevel = %s" % (loggingLevel))
        # self.logger.info("loggingLevelName = %s" % (logging.getLevelName(loggingLevel)))
        # self.logger.info("self.funcdict = %s" % (pprint.pformat(self.funcdict)))
        methodPtr = self.funcdict[logging.getLevelName(loggingLevel)]
        return methodPtr

    # --------------------------------------------------------------------------
    def setLoggerInfoLevel(self, loggingLevel):
        self.loggerInfoLevel = loggingLevel

    # --------------------------------------------------------------------------
    def setMethodInfoLevel(self, loggingLevel):
        self.methodInfoLevel = loggingLevel

    # --------------------------------------------------------------------------
    def dumpLoggerInfo(self):
        mPtr = self.getMethodPtr(self.loggerInfoLevel)
        now = datetime.datetime.now()
        mPtr("************************************************************")
        mPtr("%-24s: %s" % ("Initialized", self.name))
        mPtr("%-24s: %s" % ("Time", str(now)))
        mPtr("%-24s: %s" % ("Base Logger Level", logging.getLevelName(self.baseLoggerLevel)))
        for k in self.handlerLevels:
            mPtr("%-24s: %s" % (k, logging.getLevelName(self.handlerLevels[k])))
        mPtr("%-24s: %s" % ("Logger Info Level", logging.getLevelName(self.loggerInfoLevel)))
        mPtr("%-24s: %s" % ("Method Info Level", logging.getLevelName(self.methodInfoLevel)))
        mPtr("************************************************************")
        return

    # --------------------------------------------------------------------------
    # inspect.stack()[1]    is parent of current or what called methodTrace.
    # (
    #     <frame object at 0x7fcc7718e228>,
    #     '/media/shared/work/datagrab/reddit/vbase.py',
    #     7,
    #     'main',
    #     ['    config.clog.methodTrace()\n'],
    #     0
    # )
    # inspect.stack()[1][0]
    def dumpMethodInfo(self):
        methodName = inspect.stack()[1][3]
        fileNameWithType = inspect.stack()[1][1].rpartition('/')[2] # right partition string by backslash and return third part of tuple
        fileName = fileNameWithType.partition('.')[0]

        os = "%-20s: " % (fileName + "." + methodName + "()")
        args, _, _, values = inspect.getargvalues(inspect.stack()[1][0])
        firstIteration = True
        for i in args:
            if firstIteration: os += "("
            else: os += ", "
            firstIteration = False
            os += "%s=%s" % (i, values[i])
        if not firstIteration: os += ")"

        # Full methodInfo to TRACE
        mPtr = self.getMethodPtr(cLoggerLevel_TRACE)
        mPtr(os)

        return methodName + "(): "

















<file_sep>/datagrab/celery.py
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery


# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'datagrab.settings')

app = Celery('datagrab', broker='redis://localhost:6379/0')

# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
#   should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')

# Config options
# Restart worker after changing these
app.conf.worker_log_color=False
app.conf.worker_hijack_root_logger=False
app.conf.worker_log_format='%(asctime)-8s %(levelname).1s %(filename)-18s (line %(lineno)4s): %(message)s'

# ref: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
app.conf.timezone='Canada/Eastern'

# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
<file_sep>/reddit/tasks/tbase.py
# from datagrab.celery import app as celery_app               # for beat tasks
from celery import current_task
import time


# --------------------------------------------------------------------------
def getTaskId():
    return current_task.request.id[:8]

# --------------------------------------------------------------------------
def getTaskP():
    return 'P'
    # return 'Processing'

# --------------------------------------------------------------------------
def getTaskC():
    return 'C'
    # return 'Completed '

# --------------------------------------------------------------------------
def getMI(mi):
    rv = mi[:-2]
    return rv.ljust(36,' ')  # max length of task name plus trailing paranthesis is 36

# --------------------------------------------------------------------------
def getTimeDif(ts):
    td = round(time.time()-ts, 0)
    return '[' + str(int(td)) + ']'

# --------------------------------------------------------------------------
def getBaseP(mi):
    return "%s: %s %s:" % (getTaskId(), getMI(mi), getTaskP())

# --------------------------------------------------------------------------
def getBaseC(mi, ts):
    return "%s: %s %s: %s:" % (getTaskId(), getMI(mi), getTaskC(), getTimeDif(ts))

# --------------------------------------------------------------------------
def getLine():
    return "*******************************************"








<file_sep>/datagrab/__init__.py
# *****************************************************************************
# Import datagrab/datagrab/celery.py
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ['celery_app']

# *****************************************************************************
# Update sys.path to allow importing python files from libPython directory
import sys
sys.path.insert(0, "/home/delta/work/datagrab/reddit/lib")

# *****************************************************************************
# Set up logger
from reddit.config import initializeCLogger
initializeCLogger()



<file_sep>/reddit/tasks/__init__.py
from datagrab.celery import app as celery_app               # for beat tasks
from celery.schedules import crontab                        # for crontab periodic tasks
from .tmisc import TASK_template, TASK_inspectTaskQueue, TASK_displayModelCounts
from .treddit import TASK_updateUsersForAllComments, TASK_updateCommentsForAllUsers, TASK_updateCommentsForUser, TASK_updateThreadsForAllSubreddits, TASK_updateThreadCommentsByForest
from .ttest import TASK_testLogLevels, TASK_testForDuplicateUsers, TASK_testForDuplicateComments, TASK_testForDuplicateThreads

CONST_SECONDS_05            = 5

CONST_MINUTES_01            = 60
CONST_MINUTES_02            = CONST_MINUTES_01*2
CONST_MINUTES_05            = CONST_MINUTES_01*5
CONST_MINUTES_10            = CONST_MINUTES_01*10
CONST_MINUTES_15            = CONST_MINUTES_01*15
CONST_MINUTES_20            = CONST_MINUTES_01*20
CONST_MINUTES_30            = CONST_MINUTES_01*30

CONST_HOURS___01            = CONST_MINUTES_01*60
CONST_HOURS___02            = CONST_HOURS___01*2
CONST_HOURS___03            = CONST_HOURS___01*3
CONST_HOURS___04            = CONST_HOURS___01*4
CONST_HOURS___05            = CONST_HOURS___01*5
CONST_HOURS___06            = CONST_HOURS___01*6
CONST_HOURS___07            = CONST_HOURS___01*7
CONST_HOURS___08            = CONST_HOURS___01*8
CONST_HOURS___09            = CONST_HOURS___01*9
CONST_HOURS___10            = CONST_HOURS___01*10
CONST_HOURS___11            = CONST_HOURS___01*11
CONST_HOURS___12            = CONST_HOURS___01*12

# --------------------------------------------------------------------------
# can add expires parameter (in seconds)
# Ex: sender.add_periodic_task(60.0,      TASK_template.s(), expires=10)
@celery_app.on_after_finalize.connect
def setup_periodic_tasks(sender, **kwargs):
    # sender.add_periodic_task(CONST_SECONDS_05,  TASK_testLogLevels.s())
    # sender.add_periodic_task(CONST_SECONDS_05,  TASK_template.s())

    sender.add_periodic_task(CONST_MINUTES_02,  TASK_inspectTaskQueue.s(),                      expires=CONST_MINUTES_02-10)

    sender.add_periodic_task(CONST_MINUTES_15,  TASK_updateThreadsForAllSubreddits.s(20, 0),    expires=CONST_MINUTES_15-10)
    sender.add_periodic_task(CONST_HOURS___01,  TASK_updateThreadsForAllSubreddits.s(20, 1),    expires=CONST_HOURS___01-10)
    sender.add_periodic_task(CONST_HOURS___02,  TASK_updateThreadsForAllSubreddits.s(20, 2),    expires=CONST_HOURS___02-10)
    sender.add_periodic_task(CONST_HOURS___02,  TASK_updateThreadsForAllSubreddits.s(20, 3),    expires=CONST_HOURS___02-10)

    sender.add_periodic_task(CONST_HOURS___01,  TASK_updateCommentsForAllUsers.s(100, 0),       expires=CONST_HOURS___01-10)
    sender.add_periodic_task(CONST_HOURS___02,  TASK_updateCommentsForAllUsers.s(100, 1),       expires=CONST_HOURS___02-10)
    sender.add_periodic_task(CONST_HOURS___03,  TASK_updateCommentsForAllUsers.s(100, 2),       expires=CONST_HOURS___03-10)
    sender.add_periodic_task(CONST_HOURS___04,  TASK_updateCommentsForAllUsers.s(100, 3),       expires=CONST_HOURS___04-10)

    sender.add_periodic_task(CONST_MINUTES_10,  TASK_displayModelCounts.s(),                    expires=CONST_MINUTES_10-10)

    sender.add_periodic_task(CONST_MINUTES_02,  TASK_updateThreadCommentsByForest.s(30),        expires=CONST_MINUTES_02-10)
    sender.add_periodic_task(CONST_MINUTES_05,  TASK_updateUsersForAllComments.s(100),          expires=CONST_MINUTES_05-10)
    sender.add_periodic_task(CONST_HOURS___03,  TASK_testForDuplicateUsers.s(),                 expires=CONST_HOURS___03-10)
    sender.add_periodic_task(CONST_HOURS___03,  TASK_testForDuplicateComments.s(),              expires=CONST_HOURS___03-10)
    sender.add_periodic_task(CONST_HOURS___03,  TASK_testForDuplicateThreads.s(),               expires=CONST_HOURS___03-10)

    pass

# --------------------------------------------------------------------------
@celery_app.on_after_finalize.connect
def launch_tasks_on_startup(sender, **kwargs):
    # TASK_inspectTaskQueue.delay()
    # TASK_displayModelCounts.delay()

    # TASK_updateThreadCommentsByForest.delay(30)
    # TASK_updateUsersForAllComments.delay(100)
    # TASK_testForDuplicateUsers.delay()
    # TASK_testForDuplicateComments.delay()
    # TASK_testForDuplicateThreads.delay()
    # TASK_updateCommentsForAllUsers.delay(2000, -1)
    # TASK_updateThreadsForAllSubreddits.delay(51, 3)
    # TASK_updateThreadsForAllSubreddits.delay(51, 2)
    # TASK_updateThreadsForAllSubreddits.delay(51, 1)
    # TASK_updateThreadsForAllSubreddits.delay(51, 0)

    pass



# --------------------------------------------------------------------------
    # sender.add_periodic_task(
    #     crontab(hour=7, minute=30, day_of_week=1),
    #     test.s('Happy Mondays!'),
    # )



    # task.delay(arg1,arg2)       #will be async
    # task.delay(arg1,arg2).get() #will be sync
    # task.delay(arg1,arg2).get() #will be sync
<file_sep>/reddit/apps.py
from __future__ import unicode_literals
from django.apps import AppConfig

class RedditConfig(AppConfig):
    name = 'reddit'
<file_sep>/reddit/views/vanalysis.py
from django.http import HttpResponse
from django.db.models import Count
from django.shortcuts import redirect
from django.core.exceptions import ObjectDoesNotExist
from ..config import clog
from ..models import msubreddit, mcomment, muser
import pprint

# # *****************************************************************************
def poiUsersOfSubreddit(request, subreddit, minNumComments):
    mi = clog.dumpMethodInfo()
    # clog.logger.info(mi)

    vs = mi

    prawReddit = msubreddit.getPrawRedditInstance()
    prawSubreddit = prawReddit.subreddit(subreddit)

    subredditDict = {}

    # Get a list of users who've posted in this subreddit
    # and sort this by number of comments
    qs = mcomment.objects.filter(subreddit=prawSubreddit.name).values('username').annotate(num_count=Count('username')).order_by('-num_count')
    for i_mcommentAnnotatedUserCount in qs:
        # {'num_count': 785, 'username': 'RobRoyWithTwist'}
        if i_mcommentAnnotatedUserCount['num_count'] >= int(minNumComments):
            # print(i_mcommentAnnotatedUserCount)

            # Add this user as poi
            prawRedditor = prawReddit.redditor(i_mcommentAnnotatedUserCount['username'])
            i_muser = muser.objects.addOrUpdate(prawRedditor)
            i_muser.ppoi = True
            i_muser.save()
            if i_muser.addOrUpdateTempField == "new":
                vs += "<br>" + i_mcommentAnnotatedUserCount['username']

            # # Get subreddits this person also comments in
            qs2 = mcomment.objects.filter(username=i_mcommentAnnotatedUserCount['username']).values('rsubreddit_name_prefixed').annotate(num_count2=Count('rsubreddit_name_prefixed')).order_by('-num_count2')
            for i_mcommentAnnotatedSubredditCount in qs2:
                if i_mcommentAnnotatedSubredditCount['rsubreddit_name_prefixed'] not in subredditDict:
                    subredditDict[i_mcommentAnnotatedSubredditCount['rsubreddit_name_prefixed']] = 0
                subredditDict[i_mcommentAnnotatedSubredditCount['rsubreddit_name_prefixed']] += i_mcommentAnnotatedSubredditCount['num_count2']

    topCount = 10
    vs += "<br>Top " + str(topCount) + " subreddits these poi also post to are:"
    from operator import itemgetter
    sortedList = sorted(subredditDict.items(), key=itemgetter(1), reverse=True)
    for v in sortedList:
        # print(v)
        # # ('r/The_Donald', 34890)
        # # ('r/politics', 224)
        if topCount >= 1:
            vs += "<br>" + v[0]
        topCount -= 1

    sessionKey = 'blue'
    request.session[sessionKey] = vs
    return redirect('vsubreddit.list', xData=sessionKey)

# # *****************************************************************************
def moderatorsOfSubreddit(request, subreddit):
    mi = clog.dumpMethodInfo()
    # clog.logger.info(mi)

    vs = mi

    prawReddit = msubreddit.getPrawRedditInstance()
    prawSubreddit = prawReddit.subreddit(subreddit)

    for moderator in prawSubreddit.moderator():
        # print('{}: {}'.format(moderator, moderator.mod_permissions))
        # DatabaseCentral: ['wiki', 'posts', 'access', 'mail', 'config', 'flair']

        # Add this user as poi
        prawRedditor = prawReddit.redditor(moderator.name)
        i_muser = muser.objects.addOrUpdate(prawRedditor)
        i_muser.ppoi = True
        i_muser.save()
        if i_muser.addOrUpdateTempField == "new":
            vs += "<br> added: " + moderator.name

    # Get list of subreddits these mods also moderate
    # Seems like there isn't an API endpoint for this in Reddit or Praw api.
    # could manually scrape the page but nope not right now.

    sessionKey = 'blue'
    request.session[sessionKey] = vs
    return redirect('vsubreddit.list', xData=sessionKey)





<file_sep>/reddit/admin.py
from django.contrib import admin

# from .models import subredditSubmissionRaw
#
# admin.site.register(subredditSubmissionRaw)
<file_sep>/reddit/models/mthread.py
from __future__ import unicode_literals
from django.db import models
from django.core.exceptions import ObjectDoesNotExist
from .mbase import mbase
from .msubreddit import msubreddit
from ..config import clog
# import praw
# import pprint

# *****************************************************************************
class mthreadManager(models.Manager):
    def addOrUpdate(self, i_msubreddit, prawThread):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        try:
            i_mthread = self.get(subreddit=i_msubreddit, fullname=prawThread.name)
            redditFieldDict = i_mthread.getRedditFieldDict()
            changedCount = i_mthread.updateRedditFields(prawThread, redditFieldDict)
            if changedCount == 0: i_mthread.addOrUpdateTempField = "oldUnchanged"
            else:                 i_mthread.addOrUpdateTempField = "oldChanged"
        except ObjectDoesNotExist:
            i_mthread = self.create(subreddit=i_msubreddit, fullname=prawThread.name)
            redditFieldDict = i_mthread.getRedditFieldDict()
            i_mthread.addRedditFields(prawThread, redditFieldDict)
            i_mthread.addOrUpdateTempField = "new"

        # clog.logger.debug("i_mthread = %s" % (pprint.pformat(vars(i_mthread))))
        i_mthread.save()
        return i_mthread

# *****************************************************************************
# class mthread(mbase, models.Model):
class mthread(mbase):
    subreddit       = models.ForeignKey(msubreddit, on_delete=models.CASCADE,)
    fullname        = models.CharField(max_length=12)
    # properties
    pdeleted        = models.BooleanField(default=False)
    pforestgot      = models.BooleanField(default=False)
    pcount          = models.PositiveIntegerField(default=0)
    # Redditor fields
    rapproved_by    = models.CharField(max_length=21, default='', blank=True)
    rauthor         = models.CharField(max_length=21, default='', blank=True)
    rbanned_by      = models.CharField(max_length=21, default='', blank=True)
    rdomain         = models.CharField(max_length=64, default='', blank=True)
    rid             = models.CharField(max_length=10, default='', blank=True)
    rtitle          = models.CharField(max_length=301, default='', blank=True)
    rurl            = models.CharField(max_length=2084, default='', blank=True)

    rmod_reports    = models.TextField(default='', blank=True)
    rselftext       = models.TextField(default='')
    ruser_reports   = models.TextField(default='', blank=True)

    rdowns          = models.IntegerField(default=0)
    rnum_comments   = models.IntegerField(default=0)
    rscore          = models.IntegerField(default=0)
    rups            = models.IntegerField(default=0)

    rhidden         = models.BooleanField(default=False)
    ris_self        = models.BooleanField(default=False)
    rlocked         = models.BooleanField(default=False)
    rquarantine     = models.BooleanField(default=False)

    rcreated        = models.DecimalField(default=0, max_digits=12,decimal_places=1)
    rcreated_utc    = models.DecimalField(default=0, max_digits=12,decimal_places=1)
    redited         = models.DecimalField(default=0, max_digits=12,decimal_places=1)

    objects = mthreadManager()

    # -------------------------------------------------------------------------
    def getRedditFieldDict(self):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        redditFieldDict = {
            # mThreadFieldName      redditFieldName     convertMethodPtr
            'rapproved_by':         ("approved_by",     self.getRedditUserNameAsString),  # special
            'rauthor':              ("author",          self.getRedditUserNameAsString),  # special
            'rbanned_by':           ("banned_by",       self.getRedditUserNameAsString),  # special
            'rdomain':              ("domain",          None),      # string
            'rid':                  ("id",              None),      # string
            'rtitle':               ("title",           None),      # string
            'rurl':                 ("url",             None),      # string

            'rmod_reports':         ("mod_reports",     None),      # [[u'mod reported text', u'stp2007']],  OR [[u'Spam', u'stp2007']]
            'rselftext':            ("selftext",        None),      # string
            'ruser_reports':        ("user_reports",    None),      # [[u'Text for other reason', 1]]        OR [[u'Spam', 1]]

            'rdowns':               ("downs",           int),       # int
            'rnum_comments':        ("num_comments",    int),       # int
            'rscore':               ("score",           int),       # int
            'rups':                 ("ups",             int),       # int

            'rhidden':              ("hidden",          None),      # bool
            'ris_self':             ("is_self",         None),      # bool
            'rlocked':              ("locked",          None),      # bool
            'rquarantine':          ("quarantine",      None),      # bool

            'rcreated':             ("created",         None),      # 1493534605.0,
            'rcreated_utc':         ("created_utc",     None),      # 1493534605.0,

            'redited':              ("edited",          self.getEditedOrFalseValueAsZero),  # False or timestamp
        }
        return redditFieldDict

    # -------------------------------------------------------------------------
    def __str__(self):
        # mi = clog.dumpMethodInfo()
        # clog.logger.info(mi)

        s = self.subreddit.name
        s += " [" + self.fullname + "]"
        s += " [" + str(self.pcount) + "]"
        if self.pforestgot: s += " (pforestgot = True)"
        else:               s += " (pforestgot = False)"
        return format(s)

# # ----------------------------------------------------------------------------
# # SUBMISSION attributes
# # EX:
# # subreddit = reddit.subreddit('redditdev')
# # submission = subreddit.hot(limit=1).next()
# # logger.info(submission.title)  # to make it non-lazy
# # pprint.pprint(vars(submission))
# {
#   '_comments_by_id': {},
#   '_fetched': False,
#   '_flair': None,
#   '_info_params': {},
#   '_mod': None,
#   '_reddit': <praw.reddit.Reddit object at 0x7fee4ac8beb8>,
#   'approved_by': None,
#   'archived': False,
#   'author_flair_css_class': None,
#   'author_flair_text': None,
#   'author': Redditor(name='bboe'),
#   'banned_by': None,
#   'brand_safe': True,
#   'can_gild': True,
#   'clicked': False,
#   'comment_limit': 2048,
#   'comment_sort': 'best',
#   'contest_mode': False,
#   'created_utc': 1493534605.0,
#   'created': 1493563405.0,
#   'distinguished': None,
#   'domain': 'self.redditdev',
#   'downs': 0,
#   'edited': False,
#   'gilded': 0,
#   'hidden': False,
#   'hide_score': False,
#   'id': '68e6pp',
#   'is_self': True,
#   'likes': None,
#   'link_flair_css_class': None,
#   'link_flair_text': None,
#   'locked': False,
#   'media_embed': {},
#   'media': None,
#   'mod_reports': [],
#   'name': 't3_68e6pp',
#   'num_comments': 0,
#   'num_reports': None,
#   'over_18': False,
#   'permalink': '/r/redditdev/comments/68e6pp/praw_450_released/',
#   'quarantine': False,
#   'removal_reason': None,     # This attribute is deprecated. Please use mod_reports and user_reports instead.
#   'report_reasons': None,     # This attribute is deprecated. Please use mod_reports and user_reports instead.
#   'saved': False,
#   'score': 18,
#   'secure_media_embed': {},
#   'secure_media': None,
#   'selftext_html': '<!-- SC_OFF --><div class="md"><p>Notable additions:</p>\n',
#   'selftext': 'Notable additions:\n',
#   'spoiler': False,
#   'stickied': True,
#   'subreddit_id': 't5_2qizd',
#   'subreddit_name_prefixed': 'r/redditdev',
#   'subreddit_type': 'public',
#   'subreddit': Subreddit(display_name='redditdev'),
#   'suggested_sort': None,
#   'thumbnail': '',
#   'title': 'PRAW 4.5.0 Released',
#   'ups': 18,
#   'url': 'https://www.reddit.com/r/redditdev/comments/68e6pp/praw_450_released/',
#   'user_reports': [],
#   'view_count': None,
#   'visited': False
# }


<file_sep>/reddit/tasks/tmisc.py
from celery import task
from celery.task.control import inspect                     # for ispectTasks
import time
from ..config import clog
from .tbase import getBaseP, getBaseC, getLine
from ..models import mcomment
from ..models import msubreddit
from ..models import mthread
from ..models import muser
# import pprint

# --------------------------------------------------------------------------
@task()
def TASK_template():
    mi = clog.dumpMethodInfo()
    ts = time.time()

    # # create PRAW prawReddit instance
    # prawReddit = mcomment.getPrawRedditInstance()

    clog.logger.info("%s" % (getBaseP(mi)))

    clog.logger.info("%s" % (getBaseC(mi, ts)))
    return ""

# --------------------------------------------------------------------------
heartbeatTickString = "TICK"
@task()
def TASK_inspectTaskQueue():
    mi = clog.dumpMethodInfo()
    ts = time.time()

    thisTaskName    = 'reddit.tasks.tmisc.TASK_inspectTaskQueue'
    workerName      = "celery<EMAIL>"

    i = inspect()

    # clog.logger.info("scheduled: %s" % (pprint.pformat(i.scheduled())))
    # clog.logger.info("active: %s" % (pprint.pformat(i.active())))
    # clog.logger.info("reserved: %s" % (pprint.pformat(i.reserved())))

    # Scheduled Tasks
    scheduledCount = 0
    scheduled = i.scheduled()
    if len(scheduled) > 0 and workerName in scheduled:
        scheduledList = scheduled[workerName]
        for item in scheduledList:
            if item['name'] != thisTaskName:    # Ignore THIS task
                scheduledCount += 1

    # Active Tasks
    activeCount = 0
    active = i.active()
    if len(active) > 0 and workerName in active:
        activeList = active[workerName]
        for item in activeList:
            if item['name'] != thisTaskName:    # Ignore THIS task
                activeCount += 1

    # Reserved Tasks
    reservedCount = 0
    reserved = i.reserved()
    if len(reserved) > 0 and workerName in reserved:
        reservedList = reserved[workerName]
        for item in reservedList:
            if item['name'] != thisTaskName:    # Ignore THIS task
                reservedCount += 1

    global heartbeatTickString
    if heartbeatTickString == 'TICK': heartbeatTickString = 'tock'
    else:                             heartbeatTickString = 'TICK'

    if scheduledCount or activeCount or reservedCount:
        clog.logger.info("%s %s: %d active, %d scheduled, %d reserved" % (getBaseC(mi, ts), heartbeatTickString, activeCount, scheduledCount, reservedCount))
    else:
        clog.logger.info("%s %s: %s" % (getBaseC(mi, ts), heartbeatTickString, "*** no pending tasks ***"))
    return ""

# --------------------------------------------------------------------------
@task()
def TASK_generateModelCountData():
    mi = clog.dumpMethodInfo()

    users_poi               = muser.objects.filter(ppoi=True).count()
    users_notPoi            = muser.objects.filter(ppoi=False).count()

    users_poi_pri_0         = muser.objects.filter(ppoi=True).filter(pprioritylevel=0).count()
    users_poi_pri_1         = muser.objects.filter(ppoi=True).filter(pprioritylevel=1).count()
    users_poi_pri_2         = muser.objects.filter(ppoi=True).filter(pprioritylevel=2).count()
    users_poi_pri_3         = muser.objects.filter(ppoi=True).filter(pprioritylevel=3).count()

    comments_usersAdded     = mcomment.objects.filter(puseradded=True).count()
    comments_notUsersAdded  = mcomment.objects.filter(puseradded=False).count()

    subreddits_poi          = msubreddit.objects.filter(ppoi=True).count()
    subreddits_notPoi       = msubreddit.objects.filter(ppoi=False).count()

    subreddits_poi_pri_0    = msubreddit.objects.filter(ppoi=True).filter(pprioritylevel=0).count()
    subreddits_poi_pri_1    = msubreddit.objects.filter(ppoi=True).filter(pprioritylevel=1).count()
    subreddits_poi_pri_2    = msubreddit.objects.filter(ppoi=True).filter(pprioritylevel=2).count()
    subreddits_poi_pri_3    = msubreddit.objects.filter(ppoi=True).filter(pprioritylevel=3).count()

    threads_forestGot       = mthread.objects.filter(pforestgot=True).count()
    threads_notForestGot    = mthread.objects.filter(pforestgot=False).count()

    listOfModelCountStrings = []

    listOfModelCountStrings.append("%-30s %8d" % ("Users  poi",                  users_poi))
    listOfModelCountStrings.append("%-30s %8d" % ("Users !poi",                  users_notPoi))
    listOfModelCountStrings.append("%s" % ("---------------------------------------"))
    listOfModelCountStrings.append("%-30s %8d" % ("Users  poi priority 0",       users_poi_pri_0))
    listOfModelCountStrings.append("%-30s %8d" % ("Users  poi priority 1",       users_poi_pri_1))
    listOfModelCountStrings.append("%-30s %8d" % ("Users  poi priority 2",       users_poi_pri_2))
    listOfModelCountStrings.append("%-30s %8d" % ("Users  poi priority 3",       users_poi_pri_3))
    listOfModelCountStrings.append("%s" % ("---------------------------------------"))
    listOfModelCountStrings.append("%-30s %8d" % ("Comments  users added",       comments_usersAdded))
    listOfModelCountStrings.append("%-30s %8d" % ("Comments !users added",       comments_notUsersAdded))
    listOfModelCountStrings.append("%-30s %8d" % ("Comments  total",             comments_usersAdded + comments_notUsersAdded))
    listOfModelCountStrings.append("%s" % ("---------------------------------------"))
    listOfModelCountStrings.append("%-30s %8d" % ("Subreddits  poi",             subreddits_poi))
    listOfModelCountStrings.append("%-30s %8d" % ("Subreddits !poi",             subreddits_notPoi))
    listOfModelCountStrings.append("%s" % ("---------------------------------------"))
    listOfModelCountStrings.append("%-30s %8d" % ("Subreddits  poi  priority 0", subreddits_poi_pri_0))
    listOfModelCountStrings.append("%-30s %8d" % ("Subreddits  poi  priority 1", subreddits_poi_pri_1))
    listOfModelCountStrings.append("%-30s %8d" % ("Subreddits  poi  priority 2", subreddits_poi_pri_2))
    listOfModelCountStrings.append("%-30s %8d" % ("Subreddits  poi  priority 3", subreddits_poi_pri_3))
    listOfModelCountStrings.append("%s" % ("---------------------------------------"))
    listOfModelCountStrings.append("%-30s %8d" % ("Threads  forestGot",          threads_forestGot))
    listOfModelCountStrings.append("%-30s %8d" % ("Threads !forestGot",          threads_notForestGot))
    listOfModelCountStrings.append("%-30s %8d" % ("Threads  total",              threads_forestGot + threads_notForestGot))

    return listOfModelCountStrings

# --------------------------------------------------------------------------
@task()
def TASK_displayModelCounts():
    mi = clog.dumpMethodInfo()
    ts = time.time()

    listOfModelCountStrings = TASK_generateModelCountData()

    clog.logger.info("%s %s"    % (getBaseC(mi, ts), getLine()))
    for line in listOfModelCountStrings:
        # clog.logger.info("%s * %s *" % (getBaseC(mi, ts), line))
        clog.logger.info("%s   %s  " % (getBaseC(mi, ts), line))
    clog.logger.info("%s %s"    % (getBaseC(mi, ts), getLine()))
    return
