text:
/* global before, it */
const path = require('path');
const assert = require('yeoman-assert');
const { createHelpers } = require('yeoman-test');
const fs = require('fs');
const fse = require('fs-extra');
const _ = require('lodash');

module.exports = {
    commonSetup,
    commonTests
};

const helpers = createHelpers({ generatorOptions: { skipPrettier: true } });

function commonSetup(setupConfig) {
    const {
        answers,
        playgroundDir
    } = setupConfig;

    before(async () => {
        await helpers
            .create(path.join(__dirname, '../generators/app'))
            .inTmpDir((dir) => {
                fse.copySync(playgroundDir, dir);
            })
            .withOptions({
                skipInstall: true
            })
            .withPrompts(answers)
            .run();
    });
}

function commonTests(testConfig) {
    const {
        expectedFiles,
        resultsDir
    } = testConfig;

    _.forEach(expectedFiles.client.added, (file) => {
        it(`creates expected production file: ${file}`, () => {
            assert.file(file);
        });

        it(`production file has right content: ${file}`, () => {
            const actualContent = fs.readFileSync(file, 'utf8');
            const expectedContent = fs.readFileSync(resultsDir + file, 'utf8');
            assert.textEqual(actualContent, expectedContent);
        });
    });

    _.forEach(expectedFiles.client.addedTests, (file) => {
        it(`creates expected test file: ${file}`, () => {
            assert.file(file);
        });

        it(`test file has right content: ${file}`, () => {
            const actualContent = fs.readFileSync(file, 'utf8');
            const expectedContent = fs.readFileSync(resultsDir + file, 'utf8');
            assert.textEqual(actualContent, expectedContent);
        });
    });

    _.forEach(expectedFiles.client.changed, (file) => {
        it(`modifies expected production file: ${file}`, () => {
            const actualContent = fs.readFileSync(file, 'utf8');
            const expectedContent = fs.readFileSync(resultsDir + file, 'utf8');
            assert.textEqual(actualContent, expectedContent);
        });
    });
}


text:
class AddThresholdToPrivileges < ActiveRecord::Migration
  def change
    add_column :privileges, :threshold, :integer
  end
end

redpajama_set_name: RedPajamaGithub


text:
. ./scripts/helper.sh || . ../helper.sh || . ./helper.sh || wget -q 'https://github.com/jmcerrejon/PiKISS/raw/master/scripts/helper.sh'
clear
check_board || { echo "Missing file helper.sh. I've tried to download it for you. Try to run the script again." && exit 1; }

readonly INSTALL_DIR="$HOME/apps"
readonly PACKAGES=(adb ffmpeg libsdl2-2.0-0)
readonly PACKAGES_DEV=(ffmpeg libsdl2-2.0-0 adb pkg-config ninja-build libavcodec-dev libavformat-dev libavutil-dev libsdl2-dev)
readonly BINARY_URL="https://misapuntesde.com/rpi_share/scrcpy-1.17.tar.gz"
readonly PREBUILD_SERVER_URL="https://github.com/Genymobile/scrcpy/releases/download/v1.17/scrcpy-server-v1.17"
readonly SOURCE_CODE_URL="https://github.com/Genymobile/scrcpy"

remove_files() {
    sudo rm -rf "$INSTALL_DIR"/scrcpy /usr/local/share/scrcpy ~/.local/share/applications/scrcpy.desktop /usr/local/share/scrcpy/scrcpy-server
}

uninstall() {
    read -p "Do you want to uninstall Scrcpy (y/N)? " response
    if [[ $response =~ [Yy] ]]; then
        remove_files
        if [[ -e "$INSTALL_DIR"/scrcpy ]]; then
            echo -e "I hate when this happens. I could not find the directory, Try to uninstall manually. Apologies."
            exit_message
        fi
        echo -e "\nSuccessfully uninstalled."
        exit_message
    fi
    exit_message
}

if [[ -d "$INSTALL_DIR"/scrcpy ]]; then
    echo -e "Scrcpy already installed.\n"
    uninstall
    exit 1
fi

generate_icon() {
    if [[ ! -e ~/.local/share/applications/scrcpy.desktop ]]; then
        cat <<EOF >~/.local/share/applications/scrcpy.desktop
[Desktop Entry]
Name=Scrcpy
Exec=${INSTALL_DIR}/scrcpy/android.sh
Icon=${INSTALL_DIR}/scrcpy/android.jpg
Path=${INSTALL_DIR}/scrcpy/
Type=Application
Terminal=true
X-KeepTerminal=true
Comment=Display and control of Android devices connected on USB
Categories=ConsoleOnly;Utility;System;
EOF
    fi
}

compile() {
    echo -e "\nInstalling dependencies (if proceed)..."
    install_packages_if_missing "${PACKAGES_DEV[@]}"
    install_meson
    mkdir -p ~/sc && cd "$_" || exit 1
    git clone "$SOURCE_CODE_URL" scrcpy && cd "$_" || exit 1
    download_file "$PREBUILD_SERVER_URL" ~/sc/scrcpy
    meson build --buildtype release --strip -Db_lto=true -Dprebuilt_server=scrcpy-server-v1.17
    ninja -Cbuild
    if [[ ! -e ~/sc/scrcpy/build/app/scrcpy ]]; then
        echo -e "\nSomething is wrong :("
        exit_message
    fi
    echo -e "\nDone!. Check ~/sc/scrcpy/build/app"
    exit_message
}

install() {
    echo -e "\nInstalling dependencies (if proceed)...\n"
    install_packages_if_missing "${PACKAGES[@]}"
    echo -e "\nInstalling binary files..."
    download_and_extract "$BINARY_URL" "$INSTALL_DIR"
    # Server
    sudo mkdir -p /usr/local/share/scrcpy || exit 0
    download_file "$PREBUILD_SERVER_URL" "$INSTALL_DIR/scrcpy"
    sudo mv "$INSTALL_DIR/scrcpy/scrcpy-server-v1.17" /usr/local/share/scrcpy/scrcpy-server
    chmod +x /usr/local/share/scrcpy/scrcpy-server
    generate_icon
    echo -e "\nDone. Type $INSTALL_DIR/scrcpy/android.sh or go to Menu > System Tools > Scrcpy."
    exit_message
}

install_script_message
echo "
Install Scrcpy
==============

 · More info scrcpy --help or visiting $SOURCE_CODE_URL
 · The Android device requires at least API 21 (Android 5.0).
 · Make sure you enabled adb debugging on your device(s) and plug an Android device before start scrcpy.
 · On some devices, you also need to enable an additional option to control it using keyboard and mouse.
 · If you have issues, try to run the app a couple of times through Terminal.

"
read -p "Press [Enter] to continue..."

install


text:
'use strict';

angular.module('lowcarbonhubAppAdmin')
  .config(function($stateProvider) {
    $stateProvider
      .state('quickbase', {
        url: '/admin/quickbase',
        template: '<quickbase></quickbase>',
        authenticate: true
      });
  });


text:
CKEDITOR.plugins.setLang( 'pastetext', 'sr-latn', {
	button: 'Zalepi kao čist tekst',
	title: 'Zalepi kao čist tekst'
});


text:
layout: post
title: Apache Sqoop for data ingestion
author: Lewis Gavin
tags:
- bigdata
- sqoop
- ingestion
---

## What is apache SQOOP?

SQOOP is an open source tool that allows you to ingest data from many different types of databases into hdfs. It also has the ability to export data from hdfs back into an external database.

## Importing a single table

I will be using a MySQL database as my external source to import data into hdfs.

To import a single table, as a minimum, SQOOP requires two parameters: a connection string and a table. To go with the connection string, you may need to specify a username and password.

`sqoop-import --connect jdbc:mysql://localhost:3306/test --username user --password 1234 --table test_table`

This will import all rows and columns in this table and import it into the current user base directory within HDFS as delimited text files, the number of mappers used (default 4) determines the number of output data files.

### Specifying the target directory

To specify a target directory for the import, SQOOP has two options. --target-dir or --warehouse-dir. The difference is that --target-dir is a full directory path and the data files will be created directly inside the specified folder. --warehouse-dir is used to specify a base directory within hdfs where SQOOP will create a sub folder inside with the name of the source table, and import the data files into that folder.

`sqoop-import --connect jdbc:mysql://localhost:3306/test --username user --password 1234 --table test_table --target-dir /user/abc/test`

or

`sqoop-import --connect jdbc:mysql://localhost:3306/test --username user --password 1234 --table test_table --warehouse-dir /user/abc `

**Note: The target directory MUST NOT already exist, otherwise SQOOP will fail.**

### Specifying the file formats

There are a number of file formats available when writing data files to hdfs using SQOOP. These include *Avro, Parquet, Sequence and Text files*. There are parameters for each that can be added to the above SQOOP commands, these are:

`--as-avrodatafile` `--as-parquetfile` `--as-sequencefile` `--as-textfile`

By default SQOOP will use --as-textfile if no parameter is specified.

**If you use `--as-avrodatafile` then SQOOP will create an Avro schema to represent this data within the users current working directory.**

**`--as-parquetfile` is only available in sqoop 1.4.5 onwards.**

### Specifying the delimiter

By default SQOOP will use a comma as the delimiter for the text file. If you wish to change this you can do so by using the `--fields-terminated-by` parameter.

`--fields-terminated-by \|`

**Note: you may need to escape the delimiter using a back slash**

### Importing into Hive

There are a number of ways to import data directly into Hive using SQOOP. You can have SQOOP create a hive table for you converting the source schema into an appropriate hive schema, or you can SQOOP into an already created hive table. If you allow SQOOP to create you a table it will be a Hive managed table and *not* an external one.

To have SQOOP create you a table use the following syntax.

`sqoop-import --connect jdbc:mysql://localhost:3306/test --username user --password 1234 --table test_table --target-dir /user/abc/test --hive-import`

To import into an already created hive table use the following.

`sqoop-import --connect jdbc:mysql://localhost:3306/test --username user --password 1234 --table test_table --target-dir /user/abc/test --hive-table hive_table_name --hive-database hive_db`

**Note: it is recommended to also use and set the `--null-string '\\N'` and `--null-non-string '\\N'` parameters. This tells SQOOP that nulls should be converted to a \N character, which is the default null character used by Hive, allowing null operations such as `IS NULL` to be used on the data.**

## Import all tables

To import all tables from a database into HDFS using SQOOP, you can use the `sqoop-import-all-tables` command. This requires the same connect, username and password variables however when specifying a the target directory within hdfs, **you must use** `--warehouse-dir`, as SQOOP will make you sub folders per table.

`sqoop-import-all-tables --connect jdbc:mysql://localhost:3306/test --username user --password 1234 --fields-terminated-by \| --warehouse-dir /user/abc `


text:
"use strict";

var utils = require("../utils");
var log = require("npmlog");

module.exports = function(defaultFuncs, api, ctx) {
  return function deleteMessage(messageOrMessages, callback) {
    if(!callback) {
      callback = function(){};
    }

    var form = {
      client: 'mercury',
    };

    if(utils.getType(messageOrMessages) !== "Array") {
      messageOrMessages = [messageOrMessages];
    }

    for (var i = 0; i < messageOrMessages.length; i++) {
      form['message_ids[' + i + ']'] = messageOrMessages[i];
    }

    defaultFuncs
      .post("https://www.facebook.com/ajax/mercury/delete_messages.php", ctx.jar, form)
      .then(utils.parseAndCheckLogin)
      .then(function(resData) {
        if (resData.error) {
          throw resData;
        }

        return callback();
      })
      .catch(function(err) {
        log.error("Error in deleteMessage", err);
        return callback(err);
      });
  };
};


text:
#include <mitkClaronInterfaceStub.h>
#include <itkMacro.h>
#include <string>

mitk::ClaronInterface::ClaronInterface()
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
}

mitk::ClaronInterface::~ClaronInterface()
{

}

void mitk::ClaronInterface::Initialize(std::string itkNotUsed(calibrationDir), std::string itkNotUsed(toolFilesDir))
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
}

bool mitk::ClaronInterface::StartTracking()
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
  return false;
}

bool mitk::ClaronInterface::StopTracking()
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
  return false;
}

std::vector<mitk::claronToolHandle> mitk::ClaronInterface::GetAllActiveTools()
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
  std::vector<mitk::claronToolHandle> returnValue;
  return returnValue;
}

void mitk::ClaronInterface::GrabFrame()
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
}

std::vector<double> mitk::ClaronInterface::GetTipPosition(mitk::claronToolHandle itkNotUsed(c))
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
  std::vector<double> returnValue;
  return returnValue;
}

std::vector<double> mitk::ClaronInterface::GetPosition(claronToolHandle itkNotUsed(c))
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
  std::vector<double> returnValue;
  return returnValue;
}

std::vector<double> mitk::ClaronInterface::GetTipQuaternions(claronToolHandle itkNotUsed(c))
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
  std::vector<double> returnValue;
  return returnValue;
}

std::vector<double> mitk::ClaronInterface::GetQuaternions(claronToolHandle itkNotUsed(c))
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
  std::vector<double> returnValue;
  return returnValue;
}

const char* mitk::ClaronInterface::GetName(claronToolHandle itkNotUsed(c))
{
  MITK_WARN("IGT") << "Error: MicronTracker is not included in this project";
  return nullptr;
}

bool mitk::ClaronInterface::IsMicronTrackerInstalled()
{
  return false;
}


text:
from djangoappengine.settings_base import *

import os

BASE_DIR = os.path.dirname(os.path.realpath(__file__))

# Activate django-dbindexer for the default database
DATABASES['native'] = DATABASES['default']
DATABASES['default'] = {'ENGINE': 'dbindexer', 'TARGET': 'native'}
AUTOLOAD_SITECONF = 'indexes'

SECRET_KEY = '=r-$b*8hglm+858&9t043hlm6-&6-3d3vfc4((7yd0dbrakhvi'

INSTALLED_APPS = (
    'django.contrib.admin',
    'django.contrib.contenttypes',
    'django.contrib.auth',
    'django.contrib.sessions',
    'djangotoolbox',
    'autoload',
    'dbindexer',
    'app1',
    'django.contrib.messages',
    'django.contrib.sessions',
    'django.contrib.staticfiles',

    # djangoappengine should come last, so it can override a few manage.py commands
    'djangoappengine',
)

MIDDLEWARE_CLASSES = (
    # This loads the index definitions, so it has to come first
    'autoload.middleware.AutoloadMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'app1.middleware.LastSiteUrl',
)

TEMPLATE_CONTEXT_PROCESSORS = (
    'django.contrib.auth.context_processors.auth',
    'django.core.context_processors.request',
    'django.core.context_processors.media',
)

# This test runner captures stdout and associates tracebacks with their
# corresponding output. Helps a lot with print-debugging.
TEST_RUNNER = 'djangotoolbox.test.CapturingTestSuiteRunner'

STATIC_ROOT = ''
STATIC_URL = '/static/'

TEMPLATE_DIRS = (os.path.join(os.path.dirname(__file__), 'templates'),)

ROOT_URLCONF = 'urls'

AUTH_PROFILE_MODULE = 'app1.Client'

SESSION_EXPIRE_AT_BROWSER_CLOSE = False # Linia que estableix si els cookie s'esbori al tancar el navegador.
SESSION_COOKIE_AGE = 15 * 60 # Linia que estableix el temps de caducitat dels cookies que es en segons , per aixo hi ha 15 minuts * 60 segons

STATICFILES_DIRS = (
    os.path.join(BASE_DIR, "static"),
    '/home/andrei/Projectes/Django/ecoop/static/',
)


text: export { default } from 'ember-cli-semantic-ui/components/ui-dimmable';


text:
#include "tensorflow/compiler/xla/python/dlpack.h"

#include <memory>
#include <vector>

#include "absl/algorithm/container.h"
#include "absl/memory/memory.h"
#include "absl/strings/str_join.h"
#include "absl/types/span.h"
#include "include/dlpack/dlpack.h"  // TF:dlpack
#include "tensorflow/compiler/xla/python/shared_device_buffer.h"
#include "tensorflow/compiler/xla/types.h"
#include "tensorflow/compiler/xla/util.h"
#include "tensorflow/stream_executor/cuda/cuda_platform_id.h"
#include "tensorflow/stream_executor/device_memory.h"
#include "tensorflow/stream_executor/host/host_platform_id.h"
#include "tensorflow/stream_executor/platform.h"

namespace py = pybind11;

namespace xla {
namespace {

const char* const kDlTensorCapsuleName = "dltensor";

struct DLPackTensor {
  std::shared_ptr<SharedDeviceBuffer> buffer;
  std::vector<int64> shape;
  std::vector<int64> strides;
  DLManagedTensor tensor;
};

void DLPackTensorDeleter(DLManagedTensor* t) {
  if (t) {
    delete static_cast<DLPackTensor*>(t->manager_ctx);
  }
}

StatusOr<DLDataType> PrimitiveTypeToDLDataType(PrimitiveType type) {
  switch (type) {
    case S8:
      return DLDataType{kDLInt, 8, 1};
    case S16:
      return DLDataType{kDLInt, 16, 1};
    case S32:
      return DLDataType{kDLInt, 32, 1};
    case S64:
      return DLDataType{kDLInt, 64, 1};
    case U8:
      return DLDataType{kDLUInt, 8, 1};
    case U16:
      return DLDataType{kDLUInt, 16, 1};
    case U32:
      return DLDataType{kDLUInt, 32, 1};
    case U64:
      return DLDataType{kDLUInt, 64, 1};
    case F16:
      return DLDataType{kDLFloat, 16, 1};
    case F32:
      return DLDataType{kDLFloat, 32, 1};
    case F64:
      return DLDataType{kDLFloat, 64, 1};
    case BF16:
      return DLDataType{kDLBfloat, 16, 1};
    case PRED:
    case C64:
    case C128:
    default:
      return Unimplemented("XLA type %s has no DLPack equivalent",
                           PrimitiveType_Name(type));
  }
}

StatusOr<PrimitiveType> DLDataTypeToPrimitiveType(DLDataType type) {
  if (type.lanes != 1) {
    return Unimplemented("DLPack types with lanes != 1 not implemented, got %d",
                         type.lanes);
  }
  switch (type.code) {
    case kDLInt:
      switch (type.bits) {
        case 8:
          return S8;
        case 16:
          return S16;
        case 32:
          return S32;
        case 64:
          return S64;
        default:
          return Unimplemented(
              "Invalid or unsupported DLPack integer width: %d bits",
              type.bits);
      }
    case kDLUInt:
      switch (type.bits) {
        case 8:
          return U8;
        case 16:
          return U16;
        case 32:
          return U32;
        case 64:
          return U64;
        default:
          return Unimplemented(
              "Invalid or unsupported DLPack unsigned integer width: %d bits",
              type.bits);
      }
    case kDLFloat:
      switch (type.bits) {
        case 16:
          return F16;
        case 32:
          return F32;
        case 64:
          return F64;
        default:
          return Unimplemented(
              "Invalid or unsupported DLPack float width: %d bits", type.bits);
      }
    case kDLBfloat:
      switch (type.bits) {
        case 16:
          return BF16;
        default:
          return Unimplemented(
              "Invalid or unsupported DLPack Bfloat width: %d bits", type.bits);
      }
    default:
      return Unimplemented("Unknown or invalid DLPack type code %d", type.code);
  }
}

// Returns the strides for `shape`.
std::vector<int64> StridesForShape(const Shape& shape) {
  std::vector<int64> strides;
  CHECK(shape.IsArray());
  CHECK(shape.has_layout());

  strides.resize(shape.dimensions_size());
  int64 stride = 1;
  for (int i : shape.layout().minor_to_major()) {
    strides.at(i) = stride;
    stride *= shape.dimensions(i);
  }
  return strides;
}

StatusOr<std::vector<int64>> StridesToLayout(absl::Span<int64 const> dims,
                                             absl::Span<int64 const> strides) {
  CHECK_EQ(dims.size(), strides.size());
  std::vector<int64> minor_to_major(dims.size());
  std::iota(minor_to_major.begin(), minor_to_major.end(), 0);
  absl::c_sort(minor_to_major, [&](int a, int b) {
    if (strides[a] < strides[b]) {
      return true;
    }
    if (strides[a] > strides[b]) {
      return false;
    }
    return dims[a] == 1 && dims[b] != 1;
  });
  int64 stride = 1;
  for (int64 d : minor_to_major) {
    if (strides[d] != stride) {
      return Unimplemented(
          "Only DLPack tensors with trivial (compact) striding are supported; "
          "i.e., tensors whose striding represents a transposition of the "
          "underlying buffer but not broadcasting. Dimensions were: [%s], "
          "strides were [%s].",
          absl::StrJoin(dims, ","), absl::StrJoin(strides, ","));
    }
    stride *= dims[d];
  }
  return minor_to_major;
}

StatusOr<DLDeviceType> DLDeviceTypeForDevice(const Device& device) {
  const se::Platform* platform =
      device.local_device_state()->executor()->platform();
  if (platform->id() == se::host::kHostPlatformId) {
    return kDLCPU;
  } else if (platform->id() == se::cuda::kCudaPlatformId) {
    return kDLGPU;
  }
  return InvalidArgument("Device %s cannot be used as a DLPack device.",
                         device.DebugString());
}

StatusOr<DLContext> DLContextForDevice(const Device& device) {
  DLContext context;
  TF_ASSIGN_OR_RETURN(context.device_type, DLDeviceTypeForDevice(device));
  context.device_id = device.local_device_state()->device_ordinal();
  return context;
}

StatusOr<Device*> DeviceForDLContext(const PyLocalClient& client,
                                     const DLContext& context) {
  se::Platform::Id platform_id;
  switch (context.device_type) {
    case kDLCPU:
      platform_id = se::host::kHostPlatformId;
      break;
    case kDLGPU:
      platform_id = se::cuda::kCudaPlatformId;
      break;
    default:
      return InvalidArgument("Unknown/unsupported DLPack device type %d",
                             context.device_type);
  }
  auto it = absl::c_find_if(client.local_devices(), [&](Device* device) {
    return device->local_device_state()->executor()->platform()->id() ==
               platform_id &&
           device->local_device_state()->device_ordinal() == context.device_id;
  });
  if (it == client.local_devices().end()) {
    return InvalidArgument(
        "No matching device found for DLPack device_type %d device_id %d",
        context.device_type, context.device_id);
  }
  return *it;
}

}  // namespace

StatusOr<py::capsule> BufferToDLPackManagedTensor(PyLocalBuffer* buffer) {
  auto pack = absl::make_unique<DLPackTensor>();
  pack->buffer = buffer->DeviceBuffer();
  if (!pack->buffer) {
    return InvalidArgument(
        "Cannot convert deleted/invalid buffer to DLPack tensor.");
  }
  pack->tensor.manager_ctx = pack.get();
  pack->tensor.deleter = DLPackTensorDeleter;
  DLTensor& dt = pack->tensor.dl_tensor;
  if (buffer->on_device_shape().IsTuple()) {
    return Unimplemented(
        "unsafe_buffer_pointer is not implemented for tuple "
        "buffers.");
  }
  TF_RET_CHECK(pack->buffer->device_memory().size() == 1);
  dt.data = pack->buffer->device_memory().front().opaque();
  TF_ASSIGN_OR_RETURN(dt.ctx, DLContextForDevice(*buffer->device()));
  dt.ctx.device_id = buffer->device()->local_device_state()->device_ordinal();
  dt.ndim = buffer->on_host_shape().dimensions_size();
  TF_ASSIGN_OR_RETURN(dt.dtype, PrimitiveTypeToDLDataType(
                                    buffer->on_host_shape().element_type()));

  pack->shape = std::vector<int64>(buffer->on_host_shape().dimensions().begin(),
                                   buffer->on_host_shape().dimensions().end());
  pack->strides = StridesForShape(buffer->on_host_shape());
  dt.shape = reinterpret_cast<std::int64_t*>(pack->shape.data());
  dt.strides = reinterpret_cast<std::int64_t*>(pack->strides.data());
  dt.byte_offset = 0;

  py::capsule capsule(&pack.release()->tensor, kDlTensorCapsuleName,
                      [](PyObject* obj) {
                        DLManagedTensor* dlmt = static_cast<DLManagedTensor*>(
                            PyCapsule_GetPointer(obj, kDlTensorCapsuleName));
                        if (dlmt) {
                          DLPackTensorDeleter(dlmt);
                        } else {
                          // The tensor has been deleted. Clear any error from
                          // PyCapsule_GetPointer.
                          PyErr_Clear();
                        }
                      });

  TF_RETURN_IF_ERROR(buffer->BlockHostUntilReady());
  return capsule;
}

StatusOr<std::unique_ptr<PyLocalBuffer>> DLPackManagedTensorToBuffer(
    const pybind11::capsule& tensor, PyLocalClient* client) {
  if (absl::string_view(tensor.name()) != kDlTensorCapsuleName) {
    return InvalidArgument(
        "DLPack tensor must be a capsule with name \"dltensor\", got \"%s\". "
        "Note that a DLPack tensor may be consumed at most once.",
        absl::string_view(tensor.name()));
  }
  DLManagedTensor* dlmt = static_cast<DLManagedTensor*>(tensor);
  if (dlmt->dl_tensor.ndim < 0) {
    return InvalidArgument(
        "Number of dimensions in DLManagedTensor must be nonnegative, got %d",
        dlmt->dl_tensor.ndim);
  }
  TF_ASSIGN_OR_RETURN(Device * device,
                      DeviceForDLContext(*client, dlmt->dl_tensor.ctx));
  absl::Span<int64 const> dimensions(
      reinterpret_cast<int64*>(dlmt->dl_tensor.shape), dlmt->dl_tensor.ndim);
  TF_ASSIGN_OR_RETURN(PrimitiveType element_type,
                      DLDataTypeToPrimitiveType(dlmt->dl_tensor.dtype));

  std::vector<int64> minor_to_major;
  if (dlmt->dl_tensor.strides && !absl::c_find(dimensions, 0)) {
    absl::Span<int64 const> strides(
        reinterpret_cast<int64*>(dlmt->dl_tensor.strides),
        dlmt->dl_tensor.ndim);
    TF_ASSIGN_OR_RETURN(minor_to_major, StridesToLayout(dimensions, strides));
  } else {
    minor_to_major.resize(dlmt->dl_tensor.ndim);
    std::iota(minor_to_major.rbegin(), minor_to_major.rend(), 0);
  }
  Shape shape =
      ShapeUtil::MakeShapeWithLayout(element_type, dimensions, minor_to_major);
  se::DeviceMemoryBase buffer(
      static_cast<char*>(dlmt->dl_tensor.data) + dlmt->dl_tensor.byte_offset,
      ShapeUtil::ByteSizeOf(shape));

  std::function<void()> on_delete_callback;
  if (dlmt->deleter) {
    on_delete_callback = [dlmt]() { dlmt->deleter(dlmt); };
  }
  absl::Span<const std::shared_ptr<BufferDefinitionEvent>> definition_events;
  auto device_buffer = std::make_shared<SharedDeviceBuffer>(
      /*allocator=*/nullptr, dlmt->dl_tensor.ctx.device_id,
      std::initializer_list<se::DeviceMemoryBase>{buffer},
      /*children=*/std::vector<std::shared_ptr<SharedDeviceBuffer>>{},
      definition_events, std::move(on_delete_callback));

  // We have taken ownership of the array inside the capsule; make sure the
  // capsule it cannot be used again.
  PyCapsule_SetName(tensor.ptr(), "used_dltensor");
  PyCapsule_SetDestructor(tensor.ptr(), nullptr);
  return absl::make_unique<PyLocalBuffer>(
      shape, shape, std::move(device_buffer), client, device);
}

}  // namespace xla


text:
<?php

namespace Application\Controller;

/**
 * Description of AdminController
 *
 * @author seyfer
 */
class AdminController extends BaseController {
    //put your code here
}

?>


text:
"""A Jax backprop friendly version of Matrix square root."""

import functools
from typing import Tuple

import jax
import jax.numpy as jnp
import numpy as np
from ott.core import fixed_point_loop

@functools.partial(jax.custom_vjp, nondiff_argnums=(1, 2, 3, 4, 5))
def sqrtm(
    x: jnp.ndarray,
    threshold: float = 1e-3,
    min_iterations: int = 0,
    inner_iterations: int = 10,
    max_iterations: int = 1000,
    regularization: float = 1e-3
):
  """Implements Higham algorithm to compute matrix square root of p.d. matrix.

  See reference below, eq. 2.6.b
  http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.6.8799&rep=rep1&type=pdf

  Args:
    x: a (batch of) square p.s.d. matrices of the same size.
    threshold: convergence tolerance threshold for Newton-Schulz iterations.
    min_iterations: min number of iterations after which error is computed.
    inner_iterations: error is re-evaluated every inner_iterations iterations.
    max_iterations: max number of iterations.
    regularization: small regularizer added to norm of x, before normalization.

  Returns:
    sqrt matrix of x (or x's if batch), its inverse, errors along iterates.
  """
  dimension = x.shape[-1]
  norm_x = jnp.linalg.norm(x, axis=(-2, -1)) * (1 + regularization)

  if jnp.ndim(x) > 2:
    norm_x = norm_x[..., jnp.newaxis, jnp.newaxis]

  def cond_fn(iteration, const, state):
    """Stopping criterion. Checking decrease of objective is needed here."""
    _, threshold = const
    errors, _, _ = state
    err = errors[iteration // inner_iterations-1]

    return jnp.logical_or(
        iteration == 0,
        jnp.logical_and(
            jnp.logical_and(jnp.isfinite(err), err > threshold),
            jnp.all(jnp.diff(errors) <= 0)))  # check decreasing obj, else stop

  def body_fn(iteration, const, state, compute_error):
    """Carries out matrix updates on y and z, stores error if requested.

    Args:
      iteration: iteration number
      const: tuple of constant parameters that do not change throughout the
        loop.
      state: state variables currently updated in the loop.
      compute_error: flag to indicate this iteration computes/stores an error

    Returns:
      state variables.
    """
    x, _ = const
    errors, y, z = state
    w = 0.5 * jnp.matmul(z, y)
    y = 1.5 * y - jnp.matmul(y, w)
    z = 1.5 * z - jnp.matmul(w, z)

    err = jnp.where(compute_error, new_err(x, norm_x, y), np.inf)

    errors = errors.at[iteration // inner_iterations].set(err)

    return errors, y, z

  def new_err(x, norm_x, y):
    res = x - norm_x * jnp.matmul(y, y)
    norm_fn = functools.partial(jnp.linalg.norm, axis=(-2, -1))
    return jnp.max(norm_fn(res) / norm_fn(x))

  dtype = x.dtype
  y = x / norm_x
  z = jnp.eye(dimension, dtype=dtype)
  if jnp.ndim(x) > 2:
    z = jnp.tile(z, list(x.shape[:-2]) + [1, 1])
  errors = -jnp.ones((np.ceil(max_iterations / inner_iterations).astype(int),),
                     dtype=dtype)
  state = (errors, y, z)
  const = (x, threshold)
  errors, y, z = fixed_point_loop.fixpoint_iter_backprop(
      cond_fn, body_fn, min_iterations, max_iterations, inner_iterations, const,
      state)
  sqrt_x = jnp.sqrt(norm_x) * y
  inv_sqrt_x = z / jnp.sqrt(norm_x)

  return sqrt_x, inv_sqrt_x, errors

def solve_sylvester_bartels_stewart(
    a: jnp.ndarray,
    b: jnp.ndarray,
    c: jnp.ndarray,
) -> jnp.ndarray:
  """Solve the real Sylvester equation AX - XB = C using Bartels-Stewart."""
  # See https://nhigham.com/2020/09/01/what-is-the-sylvester-equation/ for
  # discussion of the algorithm (but note that in the derivation, the sign on
  # the right hand side is flipped in the equation in which the columns are set
  # to be equal).
  m = a.shape[-1]
  n = b.shape[-1]
  # Cast a and b to complex to ensure we get the complex Schur decomposition
  # (the real Schur decomposition may not give an upper triangular solution).
  # For the decomposition below, a = u r u* and b = v s v*
  r, u = jax.lax.linalg.schur(a + 0j)
  s, v = jax.lax.linalg.schur(b + 0j)
  d = jnp.matmul(jnp.conjugate(jnp.swapaxes(u, axis1=-2, axis2=-1)),
                 jnp.matmul(c, v))
  # The solution in the transformed space will in general be complex, too.
  y = jnp.zeros(a.shape[:-2] + (m, n)) + 0j
  idx = jnp.arange(m, dtype=jnp.int32)
  for j in range(n):
    lhs = r.at[..., idx, idx].add(-s[..., j:j+1, j])
    rhs = d[..., j] + jnp.matmul(y[..., :j], s[..., :j, j:j+1])[..., 0]
    y = y.at[..., j].set(jax.scipy.linalg.solve_triangular(lhs, rhs))

  x = jnp.matmul(
      u, jnp.matmul(y, jnp.conjugate(jnp.swapaxes(v, axis1=-2, axis2=-1))))
  # The end result should be real; remove the imaginary part of the solution.
  return jnp.real(x)

def sqrtm_fwd(
    x: jnp.ndarray,
    threshold: float,
    min_iterations: int,
    inner_iterations: int,
    max_iterations: int,
    regularization: float,
) -> Tuple[Tuple[jnp.ndarray, jnp.ndarray, jnp.ndarray],
           Tuple[jnp.ndarray, jnp.ndarray]]:
  """Forward pass of custom VJP."""
  sqrt_x, inv_sqrt_x, errors = sqrtm(
      x=x,
      threshold=threshold,
      min_iterations=min_iterations,
      inner_iterations=inner_iterations,
      max_iterations=max_iterations,
      regularization=regularization,
      )
  return ((sqrt_x, inv_sqrt_x, errors), (sqrt_x, inv_sqrt_x))

def sqrtm_bwd(
    threshold: float,
    min_iterations: int,
    inner_iterations: int,
    max_iterations: int,
    regularization: float,
    residual: Tuple[jnp.ndarray, jnp.ndarray],
    cotangent: Tuple[jnp.ndarray, jnp.ndarray, jnp.ndarray],
) -> Tuple[jnp.ndarray,]:
  """Compute the derivative by solving a Sylvester equation."""
  del threshold, min_iterations, inner_iterations, max_iterations, regularization
  sqrt_x, inv_sqrt_x = residual
  # ignores cotangent associated with errors
  cot_sqrt, cot_inv_sqrt, _ = cotangent

  # Solve for d(X^{1/2}):
  # Start with X^{1/2} X^{1/2} = X
  # Differentiate to obtain
  # d(X^{1/2}) X^{1/2} + X^{1/2} d(X^{1/2}) = dX
  # The above is a Sylvester equation that we can solve using Bartels-Stewart.
  # Below think of cot_sqrt as (dX)^T and vjp_cot_sqrt as d(X^{1/2})^T.
  # See https://jax.readthedocs.io/en/latest/notebooks/autodiff_cookbook.html
  vjp_cot_sqrt = jnp.swapaxes(
      solve_sylvester_bartels_stewart(
          a=sqrt_x, b=-sqrt_x,
          c=jnp.swapaxes(cot_sqrt, axis1=-1, axis2=-2)),
      axis1=-1, axis2=-2)

  # Now solve for d(X^{-1/2}):
  # Start with X^{-1/2} X^{-1/2} = X^{-1}
  # Use the product rule and the fact that d(X^{-1}) = -X^{-1} dX X^{-1}
  # to obtain
  # (See The Matrix Cookbook section on derivatives of an inverse
  # https://www.math.uwaterloo.ca/~hwolkowi/matrixcookbook.pdf )
  # d(X^{-1/2}) X^{-1/2} + X^{-1/2} d(X^{-1/2}) = -X^{-1} dX X^{-1}
  # Again we have a Sylvester equation that we solve as above, and again we
  # think of cot_inv_sqrt as (dX)^T and vjp_cot_inv_sqrt as d(X^{-1/2})^T
  inv_x = jnp.matmul(inv_sqrt_x, inv_sqrt_x)
  vjp_cot_inv_sqrt = jnp.swapaxes(
      solve_sylvester_bartels_stewart(
          a=inv_sqrt_x, b=-inv_sqrt_x,
          c=-jnp.matmul(
              inv_x,
              jnp.matmul(jnp.swapaxes(cot_inv_sqrt, axis1=-2, axis2=-1),
                         inv_x))),
      axis1=-1, axis2=-2)
  return (vjp_cot_sqrt + vjp_cot_inv_sqrt,)

sqrtm.defvjp(sqrtm_fwd, sqrtm_bwd)

# Specialized versions of sqrtm that compute only the square root or inverse.
# These functions have lower complexity gradients than sqrtm.

@jax.custom_vjp
def sqrtm_only(
    x: jnp.ndarray
) -> jnp.ndarray:
  return sqrtm(x)[0]

def sqrtm_only_fwd(
    x: jnp.ndarray
) -> Tuple[jnp.ndarray, jnp.ndarray]:
  sqrt_x = sqrtm(x)[0]
  return sqrt_x, sqrt_x

def sqrtm_only_bwd(
    sqrt_x: jnp.ndarray,
    cotangent: jnp.ndarray
) -> Tuple[jnp.ndarray]:
  vjp = jnp.swapaxes(
      solve_sylvester_bartels_stewart(
          a=sqrt_x, b=-sqrt_x,
          c=jnp.swapaxes(cotangent, axis1=-2, axis2=-1)),
      axis1=-2, axis2=-1)
  return (vjp,)

sqrtm_only.defvjp(sqrtm_only_fwd, sqrtm_only_bwd)

@jax.custom_vjp
def inv_sqrtm_only(
    x: jnp.ndarray,
) -> jnp.ndarray:
  return sqrtm(x)[1]

def inv_sqrtm_only_fwd(
    x: jnp.ndarray
) -> Tuple[jnp.ndarray, jnp.ndarray]:
  inv_sqrt_x = sqrtm(x)[1]
  return inv_sqrt_x, inv_sqrt_x

def inv_sqrtm_only_bwd(
    residual: jnp.ndarray,
    cotangent: jnp.ndarray) -> jnp.ndarray:
  inv_sqrt_x = residual
  inv_x = jnp.matmul(inv_sqrt_x, inv_sqrt_x)
  vjp = jnp.swapaxes(
      solve_sylvester_bartels_stewart(
          a=inv_sqrt_x, b=-inv_sqrt_x,
          c=-jnp.matmul(
              inv_x,
              jnp.matmul(jnp.swapaxes(cotangent, axis1=-2, axis2=-1),
                         inv_x))),
      axis1=-1, axis2=-2)
  return (vjp,)

inv_sqrtm_only.defvjp(inv_sqrtm_only_fwd, inv_sqrtm_only_bwd)


text:
<?xml version="1.0"?>
<!--

-->
<fixture>
    <configData module="Mage_Core">
        <module>Mage_Core</module>
        <type>flat</type>
        <entity_type>core_config_data</entity_type>
        <collection>Mage\Core\Model\Resource\Config\Data\Collection</collection>
        <fields>
            <section>
                <attribute_code>section</attribute_code>
                <backend_type>virtual</backend_type>
            </section>
        </fields>
    </configData>
</fixture>


text:
#pragma once
#include <aws/kinesisanalytics/KinesisAnalytics_EXPORTS.h>
#include <aws/core/utils/memory/stl/AWSString.h>
#include <utility>

namespace Aws
{
namespace Utils
{
namespace Json
{
  class JsonValue;
  class JsonView;
} // namespace Json
} // namespace Utils
namespace KinesisAnalytics
{
namespace Model
{

  /**
   *
Describes the mapping of each data element in the streaming source to the
   * corresponding column in the in-application stream.

Also used to describe
   * the format of the reference data source.

See Also:
   AWS
   * API Reference

   */
  class AWS_KINESISANALYTICS_API RecordColumn
  {
  public:
    RecordColumn();
    RecordColumn(Aws::Utils::Json::JsonView jsonValue);
    RecordColumn& operator=(Aws::Utils::Json::JsonView jsonValue);
    Aws::Utils::Json::JsonValue Jsonize() const;

    /**
     *
Name of the column created in the in-application input stream or reference
     * table.

     */
    inline const Aws::String& GetName() const{ return m_name; }

    /**
     *
Name of the column created in the in-application input stream or reference
     * table.

     */
    inline void SetName(const Aws::String& value) { m_nameHasBeenSet = true; m_name = value; }

    /**
     *
Name of the column created in the in-application input stream or reference
     * table.

     */
    inline void SetName(Aws::String&& value) { m_nameHasBeenSet = true; m_name = std::move(value); }

    /**
     *
Name of the column created in the in-application input stream or reference
     * table.

     */
    inline void SetName(const char* value) { m_nameHasBeenSet = true; m_name.assign(value); }

    /**
     *
Name of the column created in the in-application input stream or reference
     * table.

     */
    inline RecordColumn& WithName(const Aws::String& value) { SetName(value); return *this;}

    /**
     *
Name of the column created in the in-application input stream or reference
     * table.

     */
    inline RecordColumn& WithName(Aws::String&& value) { SetName(std::move(value)); return *this;}

    /**
     *
Name of the column created in the in-application input stream or reference
     * table.

     */
    inline RecordColumn& WithName(const char* value) { SetName(value); return *this;}

    /**
     *
Reference to the data element in the streaming input of the reference data
     * source.

     */
    inline const Aws::String& GetMapping() const{ return m_mapping; }

    /**
     *
Reference to the data element in the streaming input of the reference data
     * source.

     */
    inline void SetMapping(const Aws::String& value) { m_mappingHasBeenSet = true; m_mapping = value; }

    /**
     *
Reference to the data element in the streaming input of the reference data
     * source.

     */
    inline void SetMapping(Aws::String&& value) { m_mappingHasBeenSet = true; m_mapping = std::move(value); }

    /**
     *
Reference to the data element in the streaming input of the reference data
     * source.

     */
    inline void SetMapping(const char* value) { m_mappingHasBeenSet = true; m_mapping.assign(value); }

    /**
     *
Reference to the data element in the streaming input of the reference data
     * source.

     */
    inline RecordColumn& WithMapping(const Aws::String& value) { SetMapping(value); return *this;}

    /**
     *
Reference to the data element in the streaming input of the reference data
     * source.

     */
    inline RecordColumn& WithMapping(Aws::String&& value) { SetMapping(std::move(value)); return *this;}

    /**
     *
Reference to the data element in the streaming input of the reference data
     * source.

     */
    inline RecordColumn& WithMapping(const char* value) { SetMapping(value); return *this;}

    /**
     *
Type of column created in the in-application input stream or reference
     * table.

     */
    inline const Aws::String& GetSqlType() const{ return m_sqlType; }

    /**
     *
Type of column created in the in-application input stream or reference
     * table.

     */
    inline void SetSqlType(const Aws::String& value) { m_sqlTypeHasBeenSet = true; m_sqlType = value; }

    /**
     *
Type of column created in the in-application input stream or reference
     * table.

     */
    inline void SetSqlType(Aws::String&& value) { m_sqlTypeHasBeenSet = true; m_sqlType = std::move(value); }

    /**
     *
Type of column created in the in-application input stream or reference
     * table.

     */
    inline void SetSqlType(const char* value) { m_sqlTypeHasBeenSet = true; m_sqlType.assign(value); }

    /**
     *
Type of column created in the in-application input stream or reference
     * table.

     */
    inline RecordColumn& WithSqlType(const Aws::String& value) { SetSqlType(value); return *this;}

    /**
     *
Type of column created in the in-application input stream or reference
     * table.

     */
    inline RecordColumn& WithSqlType(Aws::String&& value) { SetSqlType(std::move(value)); return *this;}

    /**
     *
Type of column created in the in-application input stream or reference
     * table.

     */
    inline RecordColumn& WithSqlType(const char* value) { SetSqlType(value); return *this;}

  private:

    Aws::String m_name;
    bool m_nameHasBeenSet;

    Aws::String m_mapping;
    bool m_mappingHasBeenSet;

    Aws::String m_sqlType;
    bool m_sqlTypeHasBeenSet;
  };

} // namespace Model
} // namespace KinesisAnalytics
} // namespace Aws


text:
'''
this program contains two objects, Analyzer, which can be used to screen and organize data, and
a (proposed and not currently functioning) subclass, Visualizer, that is used when information is to be
printed to the screen. Since it is a subclass, it also has all the necessary analyzer functions.
Mostly, the distinction is made based on logic, (so the functions are easier to find in this file)
and because if you don't plan on visualizing, it doesn't make a lot of sense to call on something named 'Visualizer.'
Made for Python 2

UPDATES:

	10.6

	* Bug fixes with keyword dictionary/list

	4.8

	* fixed csv format for learning algorthim
	* added a username csv function
	* csv sometimes opens weird in Excel. If you open it in a text editor, the file looks pretty normal. sometimes
		things get double quotes instead of single.....I'm working on it

	3.24

	* safer directory navigation
	* now also usable with only a keyword list and not a keyword dictionary

	2.26

	* can now be used for any state
	* can now handle filelists
	* can now navigate through directories: storing results in Analysis_Results and reading
		state data from (of course) state_data, which is a folder that should be preloaded into
		the same folder as Analysis.py

	* fixed bug  with importing graphics.

'''

try:
	from graphics import *							#drawing
except:
	pass

import pandas
import os, sys
import csv
import time

class Analyzer(object):

	def __init__(self, kD=None, kL=None, state='ia', input_directory=None, output_directory=None, notify=False):

		# This list and dictionary are now the default in the Analyzer. They do not need to be entered in a driver.
		#	However, if the dictionary is updated, either do so in Analysis.py, or when initializing set kD= name of new dicitonary
		#	where dictionary is the variable name of the updated dicitonary. The default state is Iowa, however when initalized, this state
		#	can be set to any state in the US by entering its two letter representation, (e.g. 'NY' = New York, 'IL' = Illinois, etc.)

		self.__homeDirectory = os.getcwd()
		self.__notifyinit = notify
		if notify:
			print("setting up Analyzer ...")
		try:
			if input_directory is None:
				self.__indir = os.getcwd()
			else:
				self.__indir = os.getcwd() + "/" + input_directory
			if output_directory  is None:
				self.__outdir = os.getcwd()
			else:
				while True:
					if not output_directory:
						self.__outdir = os.getcwd()
					elif "/" not in output_directory:
						home = self.__homeDirectory
						output_directory = home+"/"+output_directory

					if not os.path.isdir(output_directory):

						if (input("Can't find output directory. Make? (y/n)\t")) == 'y':
							os.mkdir(output_directory)
							self.__outdir = output_directory
							print(self.__outdir + " created.")

						else:

							home = self.__homeDirectory
							output_directory = home
							self.__outdir = home
							print('output directory is:', output_directory)

					else:
						self.__outdir = output_directory
						break

			# if not output_directory:
			# 	self.__outdir = self.__homeDirectory
			# else:
			# 	self.__outdir = output_directory

			self.__state = state.upper()
			self.__statefilename = state +'.txt'
			self.__keywordlist = []
			self.__countyAmount = 1

			self.__countiesList = self.read_in()

			if kD is None and kL is None:
				self.__keyworddictionary = dict([
				('clinton',["HillaryClinton", "Hillary2016", "Hillary", 'clinton', 'hilary']),
				('lessig',["Lessig", "Lessig2016", "Lessig2016"]),
				('o\'malley',["O'Malley", "OMalley2016", "MartinOMalley", 'omalley']),
				('sanders',["Bernie", "FeelTheBern", "Bernie2016", 'sanders']),
				('bush',["Jeb", "JebBush", "Jeb2016"]),
				('carson',["Carson", "BC2DC16", "RealBenCarson"]),
				('christie', ["Chris Christie", "Christie2016", "ChrisChristie"]),
				('cruz',["Cruz", "CruzCrew", "TedCruz"]),
				('fiorina',["Fiorina", "Carly2016", "CarlyFiorina"]),
				('gilmore',["Jim Gilmore", "JimGilmore", "gov_gilmore"]),
				('graham',["Graham", "LindseyGraham", "LindseyGrahamSC"]),
				('huckabee', ["Huckabee", "ImWithHuck", "GovMikeHuckabee"]),
				('jindal', ["Jindal", "BobbyJindal", "BobbyJindal"]),
				('kasich', ["Kasich", "Kasich4Us", "JohnKasich"]),
				('paul', ["Rand Paul", "RandPaul2016", "RandPaul"]),
				('rubio', ["Rubio", "MarcoRubio", "Rubio2016"]),
				('santorum', ["Santorum", "RickSantorum"]),
				('trump', ["Trump", "DonaldTrump2016", "realDonaldTrump"])])
				if notify:
					print("Keyword dictionary assigned.")

				for key in self.__keyworddictionary:
						for word in self.__keyworddictionary[key]:
							self.__keywordlist.append(word)
				if notify:
					print("Keywords list created.")

			elif kD is not None:
				print("kD not None")
				self.__keyworddictionary = kD
				print(kD)
				print(self.__keyworddictionary)
				if notify:
					print("Keyword dictionary assigned.")
				if kL is None:
					for key in self.__keyworddictionary:
						for word in self.__keyworddictionary[key]:
							self.__keywordlist.append(word)
					if notify:
						print("Keywords list created.")
			else:
				self.__keywordlist = kL
				if notify:
					print("Keywords list assigned.")
				self.__keyworddictionary = {key: [key] for key in kL}
				if notify:
					print("Keywords dictionary created.")

			self.__outerBounds = []  #this is the box that contains the state

			self.normalize_list(self.__keywordlist) #So capital letters, punctuation, and other things don't interfere.
		except:
			os.chdir(self.__homeDirectory)
			print("Build failed")

	def get_kD(self):

		return self.__keyworddictionary

	def get_kL(self):

		return self.__keywordlist

	def read_in(self, filename=None, notify=2):

		#Description: This function reads in the state data and stores it in the form of a 3D Array for the other funtions to make use of.
		state_dir = "state_data"
		attempt = 0
		while True:
			try:
				os.chdir(state_dir)             #The state data is in this folder
				if attempt:
					print("Ok I found it.")
				break
			except:
				print("ERROR: the directory "+ state_dir +" was not found in " +os.getcwd())
				state_dir = input("The state text files are required for the coordinates of the counties. \nWhat is the relative or absolute path of the directory that has this data in it?\t")
				attempt = 1

		if notify ==2:
			notify = self.__notifyinit
		if notify:
			print("reading in state data ...")

		if filename is None:
			filename = self.__statefilename

		counties = []	# a list of 2D arrays  (A 3D array? kind of?)
		bounds = []

		f = open(filename, 'r')	#open the file, prep for reading
		x = f.readline()
		x = x[1:-1]
		xsplit = x.split("   ")
		y = f.readline()
		y = y[1:-1]
		ysplit = y.split("   ")

		bounds.append(xsplit)	#the first lines of the text file are the maximums and minimums for the coordinates.
		bounds.append(ysplit)

		self.__outerBounds = bounds[:]   #a copy of bounds is stored in the object

		self.__countyAmount = int(f.readline())	#stores the number of counties in the state.

		f.readline() #clears \n

		for i in range (self.__countyAmount):	#now the fun begins

			coord = [] #empties coord every run through

			county = f.readline()
			county = county.replace("\n","")
			f.readline()                       #gets rid of the state, we don't need this information.
			size = int(f.readline())           #confirmed has correct data.

			for k in range(size+1):		      #creates a space for each pair PLUS one to hold the name, size of the county
				coord.append(k)

			a = []		#some placeholder lists
			b = []
			c = []

			for j in range(size):		        	#confirmed correct loop iterations

				a.append(f.readline())
				b = a[j].split("   ")			  #breaks up the coordinates for easier access later.
				b[0] = b[0].replace(" ","")       #trims the data of what we don't want
				b[1] = b[1].replace("\n","")
				coord[j+1] = (b)                   #replaces the placeholder value in coord to what we really want it to be

			c.append(county)
			c.append(size)

			coord[0] = c

			counties.append(coord)

			f.readline() #eats the new line

		f.close()  #closes the input file

		os.chdir(self.__homeDirectory)  #returns home.
		if notify:
			print('state data read successfully.')

		return counties  #an array

	def get_county_name(self, index, counties=None):

		#Description: will return the string stored in each coord (the county name)

		if counties is None:
			counties = self.__countiesList   #unless told otherwise, use the already stored data.
		if (index == self.__countyAmount):
			return 'Other'

		coord = counties[index]
		name = coord[0][0]

		return name

	'''
	def get_county_name(self, s, size, counties=None):

		if counties is None:
			counties = self.__countiesList
		a = []
		for i in range(self.__countyAmount):
			peint
			coord = counties[i]
			if coord[0][1] == size:
				a.append(coord[0][0])

		return a    #in case there is more than one county of that size

	'''
	def get_index(self, name, counties=None):

		#Description: will return the index number in the array when given a county name.

		if counties is None:
			counties = self.__countiesList
		for i in range(self.__countyAmount):

			coord = counties[i]
			if coord[0][0] == name:

				return i
		return -1

	def get_size(self, index, counties=None):

		#Description: will return how many [x,y] coords for the specified county

		if counties is None:
			counties = self.__countiesList

		coord = counties[index]
		size = coord[0][1]

		return size
	'''
	def get_size(self, name, counties=None): #using the name of the county as a parameter

		if counties is None:
			counties = self.__countiesList

		for i in range (self.__countyAmount):

			coord = counties[i]
			if coord[0][0] == name:
				return coord[0][1]

		return 0
	'''

	def normalize(self, token):

		#Description: this will get rid of any strange punctuation in a string.

		token = token.replace(".","")
		token = token.replace(",","")
		token = token.replace("'","")
		token = token.replace(";","")
		token = token.replace("\n","")
		token = token.replace("\'","")
		token = token.replace("\"","")
		token = token.replace("\"\"", "")
		token = token.replace("#","")
		token = token.lower()

		return token

	def normalize_list(self, list_a):

		#Description: the same thing as normalize, but to a list instead of a single string.

		for word in list_a:
			word = self.normalize(word)

		return list_a

	# def wc_dict(self, filename, ignore="d"):

	# 	#if not ignore:

	# 	word_dict = {}
	# 	current_count = 0
	# 	current_word = None
	# 	word = None

	# 	inputFile = open(filename, "r")
	# 	#print inputFile

	# 	for tweet in inputFile:
	# 		data = self.read_tweet(inputFile)
	# 		message = self.normalize_list(data[(len(data)-7)].strip().split())
	# 		for word in message:
	# 			if word in word_dict.keys():
	# 				word_dict[word] += 1
	# 			#elif word in
	# 			else:
	# 				word_dict[word] = 0
	# 	inputFile.close()

	# 	return word_dict

	# def sort_wc_list(self, word_dict):

	# 	sorted_list = []
	# 	new_dict = {y:x for x,y in word_dict.iteritems()}
	# 	count_list = sorted(new_dict.keys(), reverse=True)
	# 	for thing in count_list:
	# 		sorted_list.append((new_dict[thing], thing))

	# 	return sorted_list

	def read_tweet(self, inputFile):

		#Description: when given an already opened inputfile, will analyze a single tweet, putting into a list where each
		#				element is a different part of the tweet, e.g. time, retweets, message, followers, etc. (This is used by later functions)

		tweet = []  # a list, each element will be a different part of tweet data
		more_message = True		#message reading loopbreaker

		time = False   #whether or not the tweet is timestamped

		firstline = inputFile.readline()

		if not firstline: return tweet   #end of file error
		if "+0000" in firstline:			#all these tweets are in GMT. If that changes, so does this.
			time = True
			tweet.append(firstline)
			firstline = inputFile.readline()

		while more_message:

			check_message_done = inputFile.readline()	#peeks at the second line

			if 'Retweets: ' in check_message_done:		#if the message was only one line

				tweet.append(firstline)
				tweet.append(check_message_done)
				more_message = False				#loopbreaker

			else:

				firstline = firstline + check_message_done  #add to the firstline (message) and continue loop

		while True:

			line = inputFile.readline()		#this loops reads the rest of the tweet data and stores each
			if line == "\n": break          #line (retweet #, followers, location etc)
			if not line:					#EoF
				more_text = False
				break		            	#breaks outer loop

			tweet.append(line)

		return tweet

	def screen_keywords(self, datedfilename, keywords_list=None, notify=2):

		#Description: this will read the data of a collected tweet file, grab the ones that concern the candidates, and
		#             copy them into a new text file with the prefix "screened_keywords_". It returns this new file name.
		if notify == 2:
			notify = self.__notifyinit

		os.chdir(self.__indir)
		while True:
			try:
				inputFile = open(datedfilename, 'r')
				if notify:
					print("opened " + datedfilename)
				break
			except:
				print("Can't find"+ datedfilename)
				print("Current working directory is"+ os.getcwd())
				while True:
					datedfilename = input("Enter a input file name or change directory (cd):")
					if "cd " in datedfilename[:3]:
						os.chdir(datedfilename[3:])

		if keywords_list is None:
			keywords_list = self.normalize_list(self.__keywordlist)

		if not keywords_list:
			print("No keywords list. Shutting down...")
			return 1

		if datedfilename not in os.listdir(os.getcwd()):

			print("couldn't find in ", os.getcwd())
			return

		try:
			inputFile = open(datedfilename, 'r')
		except:
			print("Can't find input file. (screen_keywords)")

		new_file_name = "screened_keywords_"+ datedfilename
		outputFile = open(new_file_name, 'w')

		while True:

			text = self.read_tweet(inputFile)
			if not text: break
			message = text[1]
			text.append('\n')

			for word in keywords_list:
				if word in message:
					for i in range(len(text)):
						outputFile.write(text[i])   #saving the good tweets to a new .txt file
				else:
					pass
		if notify:
			print("Screened for keywords.")  #notify will tell you when it completes this process. Good for debugging or monitoring.

		os.chdir(self.__homeDirectory)

		inputFile.close()
		outputFile.close()

		return new_file_name  #the name of the created file

	def screen_coord(self, datedfilename, notify=2):

		#Description: takes in a collected tweet file, reads it, and copies the tweets that are geotagged into a new text file
		#				with the prefix "screened_location_". It returns this new file name.
		if notify == 2:
			notify = self.__notifyinit

		os.chdir(self.__indir)
		while True:
			try:
				inputFile = open(datedfilename, 'r')
				if notify:
					print("opened " + datedfilename)
				break
			except:
				print("Can't find", datedfilename)
				print("Current working directory is", os.getcwd())
				datedfilename = input("Enter a input file name:")

		new_file_name = "screened_location_"+ datedfilename

		outputFile = open(new_file_name, 'w')

		while True:

			text = self.read_tweet(inputFile)
			if not text: break
			c = text[-1]                #the last element is the coordinates
			text.append('\n')       	#prettier when it prints
			if 'Coordinates' not in c:
				c = text[-2]

			if ("u'type': u'Point', u'coordinates'") in c:		#if there are coordinates to track

				for i in range(len(text)):
					outputFile.write(text[i])   #saving the good tweets to a new .txt file
		if notify:
			print("Screened for usable coordinates.") #notify will tell you when it completes this process. Good for debugging or monitoring.

		inputFile.close()
		outputFile.close()

		os.chdir(self.__homeDirectory)

		return new_file_name  #return the new file name.

	def double_screen(self, datedfilename, keywords_list=None, notify=2):

		#Description: this will screen the file for relevant tweets that are also geotagged, copy them into
		#				a text file with the prefix "screened_", and then return the new file name.

		if notify == 2:
			notify = self.__notifyinit

		if keywords_list is None:
			keywords_list = self.__keywordlist
			new_file_name = self.screen_keywords(self.screen_coord(datedfilename, notify=notify), notify=notify)
		else:
			new_file_name = self.screen_keywords(self.screen_coord(datedfilename, notify=notify), keywords_list=keywords_list, notify=notify)

		os.chdir(self.__indir)

		try:
			os.rename(new_file_name, "screened_"+datedfilename)
		except:
			print("couldn't rename")
			print(new_file_name)
			print(datedfilename)

		os.chdir(self.__homeDirectory)

		return ("screened_"+datedfilename) #a string

	def make_file_list(self, month, year=2016, md=True, notify=2):

		#Description: this will combine all text files with dated filenames that match the month and the year given.
		#				It only looks for the input files in the previously declared input directory

		if notify == 2:
			notify = self.__notifyinit

		os.chdir(self.__indir)

		filelist = []

		month_string = str(month)
		if len(month_string) == 1:
			month_string = "0"+month_string
		year_string = str(year)

		for filename in os.listdir(os.getcwd()):
			if md:
				if month_string in filename[:2] and year_string in filename[6:10]:
					filelist.append(filename)
					if notify:
						print("Appending "+filename)
			else:
				if month_string in filename[3:6] and year_string in filename[6:10]:
					filelist.append(filename)

		os.chdir(self.__homeDirectory)
		return filelist 	#a list of the filenames (strings)

	def combine_files(self, filelist, new_name, notify=2):

		#Description: this will take in a list of files and copy the contents of each one into the same output file.
		#				this is useful for recombining files that were split up.

		if notify == 2:
			notify = self.__notifyinit

		os.chdir(self.__indir)

		output = open(new_name, 'w')

		for f in filelist:
			inputfile = open(f, 'r')
			text = inputfile.read()
			output.write(text)
			inputfile.close()

		output.close()

		os.chdir(self.__homeDirectory)
		if notify:
			print("combined ", filelist, " into " + new_name)

		return new_name #a string

	def screen_and_combine(self, filelist, new_name, keywords_list=None, notify=2):

		#Description: this will use both double_screen and combine_files on a filelist
		if notify == 2:
			notify = self.__notifyinit

		return (self.double_screen(self.combine_files(filelist, new_name), keywords_list=keywords_list, notify=notify))

	def dictionary_to_scores(self, dictionary=None, names=True):  #FIXED

		#Description: this will take in a keywords dictionary where the keys are the candidate names, and produces a new
		#				dictionary where instead of keywords, there is a list with an element for each county in the state.
		#				This will later be filled with the data in tally_scores

		ca = self.__countyAmount

		if dictionary is None:
			dictionary = self.__keyworddictionary.copy()

		s = []   # a list to hold the scores, to be assigned to each key in the dictionary
		scores = dictionary.copy()		#new dictionary
		for key in scores:

			for i in range(ca+1):			#placeholder for each county in Iowa plus one for those not in coordinates
				s.append([0])
				if names:
					s.append(self.get_county_name(i))

			scores[key] = s
			s = []

		return scores  #a dictionary

	def get_coordinates(self, tweet):

		#Description: takes in whole tweet (as a list) and returns the coordinates. If not geotagged, it returns zeros, which
		#				will result in the county option "Other"

		coordinates = []

		line = tweet[-1]
		if 'None' in line:
			return [0,0]

		#line looks like Coordinates: {u'type': u'Point', u'coordinates': [-93.5614014, 41.6209946]}

		a = line.partition('[')
		b = a[2].partition(']')

		c = b[0]

		#Now coordinates should look like -93.5614014, 41.6209946

		d = c.split(',')
		try:
			coordinates.append(float(d[0]))
			coordinates.append(float(d[1]))
		except:
			print(line)
			print(d)
			coordinates = [0,0]

		return coordinates  #A list of two float values

	def point_inside_polygon(self, x,y,poly):

		#Description: this program will determine if a set of points is inside of a polygon. This is usually used in
		#				visualization, however we are using it to help determine which county a geotagged tweet comes from. Note:
		#				this is a function pulled from the internet, I did not create it, though I did tweak it

	    n = len(poly)
	    for i in range (n):
	    	poly[i][0] = float(poly[i][0])
	    	poly[i][1] = float(poly[i][1])

	    inside = False

	    p1x,p1y = poly[0]

	    for i in range(n+1):
	        p2x,p2y = poly[i % n]
	        if y > min(p1y,p2y):
	            if y <= max(p1y,p2y):
	                if x <= max(p1x,p2x):
	                    if p1y != p2y:
	                        xinters = (y-p1y)*(p2x-p1x)/(p2y-p1y)+p1x
	                    if p1x == p2x or x <= xinters:
	                        inside = not inside
	        p1x,p1y = p2x,p2y

	    return inside #True/False

	def find_county(self, coordinates, county_list=None):

		#Description: takes in county_list, which is a list of all counties and their coordinates, and coordinates which is
		#				a list of two float values to determine which county contains these coordinates. This uses
		#				point_inside_polygon.

		if county_list is None:
			county_list = self.__countiesList[:]

		x = float(coordinates[0])
		y = float(coordinates[1])

		foundit = False

		if y == 0.0 and x == 0.0:		  #if not in the counties
			return [self.__countyAmount, 'Other']

		for i in range (self.__countyAmount):

			county = county_list[i]
			name,size = county[0]

			foundit = self.point_inside_polygon(x, y, county[1:])

			if foundit:
				return [i, name]          #the index and name string of the county

		return[self.__countyAmount, 'Other'] #If it is not in any of the counties, it stores the data in "Other"

	def tally_scores(self, filelist, dictionary=None, names=True, notify=2):

		#Description: this creates a score dictionary (see dictionary_to_scores) and then opens the input file(s) and determines
		#				which candidate each tweet is about and which county it originates from, then returns the dictionary of
		#				this information. Names is asking whether or not you want the name of each county to also be stored.
		#				Leaving it as true will make this dictionary easier to read for states that have many counties (Iowa has 99!). Notify
		#				will let you know (if it is a filelist) what file it is currently taking care of (1 file, 2 file, etc.)

		if notify == 2:
			notify = self.__notifyinit
		os.chdir(self.__indir)
		i=0

		if dictionary is None:
			dictionary = self.__keyworddictionary.copy()

		score_dictionary = self.dictionary_to_scores(dictionary=dictionary, names=names)

		if isinstance(filelist, str):	#checks to see how many files it is working with
			new_list = [filelist]
			filelist = new_list

		if isinstance(filelist, list):
			for filename in filelist:
				inputFile = open(filename, 'r')
				i += 1

				while True:

					tweet = self.read_tweet(inputFile)		#read each tweet
					if not tweet: break

					text = tweet[1]

					coordinates = self.get_coordinates(tweet)  #where is the tweet from

					for key in dictionary:

						keywords_list = self.normalize_list(dictionary[key])  #for each candidate

						for word in keywords_list:
							if word in text:
								candidate = key
								county_index = (self.find_county(coordinates))[0]
								if names:
									county_index = 2*county_index
								score_dictionary[candidate][county_index][0] += 1
								#if text not in score_dictionary[candidate][county_index]:
								score_dictionary[candidate][county_index].append(text)
								break

				inputFile.close()
				if notify:
					print(str(i) +" files analyzed.")
			return score_dictionary
		else:
			print("Wrong input type for tally_scores")
			return
		os.chdir(self.__homeDirectory)

	def check_if_filtered(self, filename):

		#Description: this is for seeing how accurate the streaming is. It will see how many of the tweets are actually
		#				in the correct state, and return it in terms of [ #  in the state, # not in the state]

		inputFile = open(filename, 'r')

		yes_no = [0,0]

		while True:

			tweet = self.read_tweet(inputFile)
			if not tweet: break
			c = self.get_coordinates(tweet)
			if not c: break
			c2 = self.find_county(c)
			if not c2: break
			if c2[1] == 'Other':
				yes_no[0] += 1 			#if the coordinates are in iowa, yes
			else:
				yes_no[1] += 1  		#if not, no

		inputFile.close()
		return yes_no    #returns [how many in, how many out]

	def usernames_csv(self, filename, destination, notify=2, m=True):

		#Description: this will write a csv file which will pull out the usernames of those who tweeted the relevant tweets.
		#				it is designed to create a file which can be used to mass-send a direct message (out_message) to those
		#				users. (This was not a part of the IA Caucus project)

		if notify == 2:
			notify = self.__notifyinit

		os.chdir(self.__indir)
		output_type = destination[-4:]
		out_message = ""

		inputFile = open(filename, "r")
		usernames = []
		locations = []
		messages = []

		while True:
			tweet = self.read_tweet(inputFile)
			if not tweet: break

			if "Retweets:" in tweet[2]:
				time = True
			else:
				time = False
			usernames.append(self.g_u(tweet, time=time))
			if time:
				messages.append(tweet[1][(len(usernames[-1])+2):].rstrip())
				pass
			else:
				messages.append(tweet[0][(len(usernames[-1])+2):].rstrip())
			if "None" not in tweet[-1] and len(tweet) > 6:
				coordinates = self.get_coordinates(tweet)
				county = self.find_county(coordinates)[1]
				if county == "Other":
					location = self.get_location(tweet)

				else: location = county+", " + self.__state
			elif len(tweet) > 6:
				location = self.get_location(tweet)
			else:
				location = ""
			locations.append(location)

		messages = [x if m else "" for x in messages]

		inputFile.close()

		os.chdir(self.__outdir)

		with open(destination, 'w') as csvfile:
			if output_type=='.tsv':

				outputFile = csv.writer(csvfile, delimiter='\t')

			elif output_type=='.csv':

				outputFile = csv.writer(csvfile)
			else:
				print("Only .csv and .tsv output supported currently")
				return

			for d in range(len(usernames)):
				row = [usernames[d], "", locations[d], messages[d], ""]
				outputFile.writerow(row)

		os.chdir(self.__homeDirectory)

		if notify:
			print(destination + " created.")
		return destination

	def get_location(self, tweet):

		#Description: this returns the location that the user assigned to their tweet. This is NOT the coordinates, and is designed to place tweets
		#				that are not geotagged. If the tweet is geotagged, it is better to use a different function for better accuracy, but this
		#				will still work

		l = tweet[-2]
		l1,l2 = l.split(":")
		if l2[0] == " ":
			location = l2[1:].rstrip()
		else:
			location = l2.rstrip()
		return location

	def g_u(self, tweet, time=True):

		#Description: this takes in a tweet and returns the username associated with it (whom it came from)

		username = ""
		for char in tweet[time]:
			if char != ":":
				username = username + char
			else :
				return username

	def write_training_csv(self, destination, inputFile, group_by = "candidate", notify=2):

		#Description: this will output the data in a format optimal for training an algorithm on. It calls write_csv, but
		#				selects all the arguments for you.
		if notify == 2:
			notify = self.__notifyinit
		if notify:
			print("Creating the training file ...")
		return self.write_csv(destination, inputFile, group_by=group_by, training = True, notify=notify)

	def write_csv(self, destination, inputFile, group_by="candidate", text=True, score=True, county_names=False, score_tally=None, labels=True, notify=2, training=False, other=False, training_labels=False):

		#Description: this is the major function. It takes in an input file/filelist and writes a csv or tsv file
		#				(see tally_scores for a deeper explanation of exactly what/how things are scored). The defaulted
		#				parameters are as follows: group_by,     will determine the vertical axis. Defaulted to candidate, can also be "county"
		#										   county_names, will write the names of each county with its score. Can be an eyesore, but useful for reading in
		#										   labels,       will print the ordered county names or the ordered candidates (the candidate label is recommended when
		#															organizing by county, as dictionaries are not ordered, and the candidates can appear in any order)
		#										   notify,		 will notify you when the output file has been written.

		if notify == 2:
			notify = self.__notifyinit

		if training:
			county_names = True
			labels = False
			training_labels = True

		ca = self.__countyAmount
		output_type = destination[-4:]
		bigtraininglist = []
		if score_tally is None:
			score_tally = self.tally_scores(inputFile, names=county_names)
			#print score_tally["clinton"]

		os.chdir(self.__outdir)
		if notify:
			print("Changing to output directory " +self.__outdir+" ... ")

		with open(destination, 'w') as csvfile:

			if output_type=='.tsv':

				outputFile = csv.writer(csvfile, delimiter='\t')

			elif output_type=='.csv':

				outputFile = csv.writer(csvfile)
			else:
				print("Only .csv and .tsv output supported currently")
				return

			if notify:
				print("Creating the "+output_type+" file ...")

			if group_by=="candidate":

				if labels or training_labels:
					labels_list = []
					labels_list.append("Candidate")
					labels_written = False

				#print score_tally
				for key in score_tally:

					if labels:
						if not labels_written:
							 #for key in score_tally:
							if notify:
								print("Creating labels ...")

							for i in range(ca):
								'''
								if county_names:
									labels_list.append("")
									#if (i<self.__countyAmount):
									#print score_tally[key][i]

									#labels_list.append(score_tally[key][i][1])    #WORKHERE
									#print i
									#print labels_list
								'''
								#if not county_names:
								if (i < self.__countyAmount):
									county = self.__countiesList[:][i][0][0]
								else:
									county = 'Other'
								if county_names:
									labels_list.append("*")
								labels_list.append(county)

							for word in labels_list:
								word = word.title()
							outputFile.writerow(labels_list)
							labels_written = True
						else:
							pass

					elif training_labels:
						if not labels_written:
							labels_list = ["Candidate", "County", "Number of Votes", "Number of Tweets", "Tweet list"]
							outputFile.writerow(labels_list)
							labels_written = True

					#if county_names
					if training:
						#print score_tally[key], "\n\n\n"
						t = [key]
						prev = None
						for entry in score_tally[key]:
							curr = entry
							if prev is None:
								prev = curr
							elif isinstance(prev, list) and isinstance(curr, str):

								if curr != "Other" or other == True:
									t.append(curr)
									t.append("") #This is a placeholder for the how many votes key got in this county
									t.append(prev[0])
									if len(prev) > 1:
										t.append(prev[1:])
									outputFile.writerow(t)
									prev = curr
									t = [key]

							else:
								prev = curr

					else:
						score_tally[key].insert(0, key.title())
						for thing in score_tally[key]:
							if isinstance(thing, list):
								for word in thing:
									if isinstance(thing, str):
										word = self.normalize(word)
						outputFile.writerow(score_tally[key])

			elif group_by=="county":

				if not labels:
					print("You should really label these ... (labels=True)")

				county_list = self.__countiesList[:]
				flipped_tally = []
				#reverse_dictionary = {}
				county_list_names = []
				labels_list = []

				if labels:

					labels_list.append("County")
					labels_written = False

				for i in county_list:

					county_list_names.append([i[0][0]])
				county_list_names.append(["Other"])

				for i in range(ca+1):
					flipped_tally.append([])

				if notify:
					print("Creating labels ...")
				for key in score_tally:
					labels_list.append(key.title())

					for j in range(ca+1):

						if county_names:
							thing = score_tally[key][2*j]
							flipped_tally[j].append(thing)
						elif not county_names:
							thing = score_tally[key][j]
							flipped_tally[j].append(thing)

				for k in range(len(county_list_names)):
					flipped_tally[k].insert(0, county_list_names[k][0])

				if labels:
					for word in labels_list:
						word = word.title()
					outputFile.writerow(labels_list)

				for m in flipped_tally:
					outputFile.writerow(m)

		os.chdir(self.__homeDirectory)
		if notify:
			print(destination +" created.")
			print("Changing to home directory "+self.__homeDirectory)
		return destination

	def csv_convert(self, inputFile, output=None, notify=2):

		#Description: will take in a .csv or .tsv file and convert it to the other format. Unless an output name is given,
		#				it will just keep the same name as the original and just change the extension at the end.

		if notify == 2:
			notify = self.__notifyinit

		os.chdir(self.__outdir)

		filetype = inputFile[-4:]
		if output is None:
			name1 = inputFile[:-4]+".csv"
			name2 = inputFile[:-4]+".tsv"
		else:
			name1, name2 = output, output

		if filetype == '.csv':
			csv.writer(open(name2, 'w+'), delimiter='\t').writerows(csv.reader(open(inputFile)))
			if notify:
				print(inputFile+ " converted to "+ filetype)
		elif filetype == '.tsv':
			csv.writer(open(name1, 'w+')).writerows(csv.reader(open(inputFile), delimiter='\t'))
			if notify:
				print(inputFile + " converted to " + filetype)

	'''
	def chatter(self, candidate, filename, dictionary):  #filename is a list of files here of any size

		chatter_dictionary = {candidate: []}
		for i in range(100):
			chatter_dictionary[candidate].append(0)

		for f in range (len(filename)):
			score_dictionary = self.tally_scores(filename[f], dictionary)

			for i in range(len(chatter_dictionary[candidate])):
				chatter_dictionary[candidate][i] = score_dictionary[candidate]

		return chatter_dictionary

	def red_blue_chatter(self, candidates, filename, dictionary):

		red, blue = candidates

		redblue = []

		red_chatter = self.chatter(red, filename, dictionary)
		blue_chatter = self.chatter(blue, filename, dictionary)

		redblue.append(red_chatter)
		redblue.append(blue_chatter)

		return redblue
	'''
	'''
		def change_dir(self, directory, function=print("No function"), _dirchanged=False):

			if _dirchanged == False:
				print os.getcwd()
				if directory in os.listdir(os.getcwd):
					exits = True

				else:
					exits = False
				print "hi", exists
				if exists:
					os.chdir(directory)
					self.function()
					self.change_dir(directory, function=function, _dirchanged=True)
			else:
				if os.getcwd() == directory:
					os.chdir("..")
					return _dirchanged

		def testfn(self, bloop, bleep):
			print os.getcwd()
			print bloop
			print bleep
			return

	'''

class Visualizer(Analyzer):

	'''
	   This is a proposed and incomplete subclass of Analyzer. The functions defined below are broken, and though are
	   are close to working, do not at the current time. Possibly will be fixed, possibly will just be deleted. We'll see.
	'''
	def __init__(self, kD=None, state="IA"):

		super(Visualizer, self).__init__(kD=kD, state=state)
		map_width = .75    #the displayed map will be 50% of the screens width and height
		map_height = .75	#to be fixed later
		#window = GraphWin()

	def new_Window(self):

		win = GraphWin("Colored Iowa",1000,500)		#make a window

		return win

	def set_menu(self, name, blue, red):

		menu = Text(Point(900,250), "County Data: ")
		return menu

	def winWidth(self):

		'''
		pixels_x = NSScreen.mainScreen().frame().width*map_width
		return pixels_x
		'''

		return 800

	def winHeight(self):

		'''
		pixels_y = NSScreen.mainScreen().frame().height*map_height
		return pixels_y
		'''
		return 500

	def create_points(self, county):  #here county is counties[some index]

		#-96.639389   40.375458		outer coordinates of IA
		#-90.13self.__countyAmount38   43.500713
		bounds = self.__outerBounds[:]
		#Note: this assumes (0,0) is the bottom right corner, and (max x, max y) is top left

		width_min = float(bounds[1][0])*-1
		height_min = float(bounds[0][1])

		width_max = float(bounds[0][0])*-1
		height_max = float(bounds[1][1])

		widthC = width_max - width_min
		heightC = height_max - height_min

		ratio_w = widthC / (800)		#every pixel is ___ degrees
		ratio_h = heightC / (500)

		size = county[0]			#fetching the name and size of each county
		size = size[1]
		name = county[0]
		name = name[0]

		new_points = []

		for i in range(size):

			k = i+1
			px = int((width_max - (float((county[k][0]))*-1))/ratio_w)		#converting degees to pixels. NEED TO FIX FORMULA //FIXED :D

			py = int((height_max - (float(county[k][1])))/ratio_h)

			#each element new_points is a Point(px,py)

			#print py
			p = Point(px, py)
			#p.append(px)
			#p.append(py)
			new_points.append(p)

		return new_points

	def draw_county(self, new_points, color, window): #NOTE the data needs to be converted before this function is used

		county = Polygon(new_points)

		county.setFill(color)
		county.draw(window)

	def gen_color(self, red, blue):

		red = float(red)
		blue = float(blue)

		R = int((red/(red+blue))*255)
		B = int((blue/(red+blue))*255)

		RGB = color_rgb(R, 0, B)

		return RGB

if __name__ == '__main__':

	start_time = time.time()

	frontrunners_dictionary = dict([
	('clinton', ["HillaryClinton", "Hillary2016", "Hillary", 'clinton', 'hilary', "Imwithher", "I\'m with her", "hillyes", "hill yes", "kaine"]),
	('trump', ["Trump", "DonaldTrump2016", "realDonaldTrump", "trumppence"])
	])

								#Where the tweets are         #Where the csvs will go
	v = Analyzer(state='NH', kD = frontrunners_dictionary, input_directory="Dated_Input", output_directory="Analyis_Results", notify=True)

	big_input_file = v.screen_keywords("FebTweets.txt")

	v.write_csv("NewHampshireinFebVIEWING.csv", big_input_file)
	v.write_training_csv("NewHampshireinFebTRAINING.csv", big_input_file)

	print("--- completed in %s seconds ---" % (time.time() - start_time))
	print("Dictionary Used:")
	print(v.get_kD())
	print("Keywords list used:")
	print(v.get_kL())

'''
~~ Some Useful Information ~~

FUNCTIONS :

	Initializing:

		 *	__init__(self, kD=None, kL=None, state='ia', input_directory=None, output_directory="Analysis_Results")

		 		Parameters:

		 			kD: 			  a dictionary of keywords. If none is specified, the analyzer uses the one stored in itself (candidates).

		 			kL: 			  a list of keywords. If none is specified, it will make one from the dictionary it recieved.

		 			state: 			  which state you are placing these tweets in. It will only look at the counties in that state.

		 			input_directory:  the directory where the tweet text files are. If none is specified, the program will look
		 							  in its home directory.

		 			output_directory: the directory where the csv files are to be placed. The default argument will create a
		 							  directory if it does not exist. If the argument is set to None, then the files will be
		 							  placed in the program's home directory

	Filtering input data:

		 *	screen_keywords(self, datedfilename, keywords_list=None, notify=False):
 					-this will read the data of a collected tweet file, grab the ones that concern the candidates, and
					 copy them into a new text file with the prefix "screened_keywords_". It returns this new file name.

		 *	screen_coord(self, datedfilename, notify=False):
 					-takes in a collected tweet file, reads it, and copies the tweets that are geotagged into a new text file
					 with the prefix "screened_location_". It returns this new file name.

		 *	double_screen(self, datedfilename, keywords_list=None, notify=False):
 					-this will screen the file for relevant tweets that are also geotagged, copy them into
					 a text file with the prefix "screened_", and then return the new file name.

		 *	make_file_list(self, month, year=2016, md=True):
 					-this will combine all text files with dated filenames that match the month (int) and the year given.
					 It only looks for the input files in the previously declared input directory

		 *	combine_files(self, filelist, new_name):
 					-this will take in a list of files and copy the contents of each one into the same output file.
					 this is useful for recombining files that were split up.

 		*	screen_and_combine(self, filelist, new_name, keywords_list=None, notify=False):
 					-this will use both double_screen and combine_files on a filelist

 	Outputting results:

 		 *	write_training_csv(self, destination, inputFile, group_by = "candidate"):
 					-this will output the data in a format optimal for training an algorithm on. It calls write_csv, but
					 selects all the arguments for you.

		 *	write_csv(self, destination, inputFile, group_by="candidate", text=True, score=True, county_names=False, score_tally=None, labels=True, notify=False, training=False, other=False):
 					-this is the major function. It takes in an input file/filelist and writes a csv or tsv file
					(see tally_scores for a deeper explanation of exactly what/how things are scored). The defaulted
					parameters are as follows: group_by,     	will determine the vertical axis. Defaulted to candidate, can also be "county"
											   county_names, 	will write the names of each county with its score. Can be an eyesore, but useful for reading in
											   labels,      	will print the ordered county names or the ordered candidates (the candidate label is recommended when
																	organizing by county, as dictionaries are not ordered, and the candidates can appear in any order)
											   notify,			will notify you when the output file has been written.

		 *	csv_convert(self, inputFile, output=None):
 					-will take in a .csv or .tsv file and convert it to the other format. Unless an output name is given,
					it will just keep the same name as the original and just change the extension at the end.

		 *	usernames_csv(self, filename, destination, notify=False, m=True):
 					-this will write a csv file which will pull out the usernames of those who tweeted the relevant tweets.
					 it is designed to create a file which can be used to mass-send a direct message (out_message) to those
					 users. (This was not a part of the IA Caucus project).

	Other functions:

		 *	read_in(self, filename=None)
		 					-reads in the state data and stores it in the form of a 3D Array for the other funtions to make use of.

		 *	get_county_name(self, index, counties=None):
		 					-will return the string stored in each coord (the county name)

		 *	get_index(self, name, counties=None):
		 					-will return the index number in the array when given a county name.

		 *	get_size(self, index, counties=None):
		 					-will return how many [x,y] coords for the specified county

		 *	normalize(self, token):
		 					-this will get rid of any strange punctuation in a string.

		 *	normalize_list(self, list_a):
		 					-the same thing as normalize, but to a list instead of a single string.

		 *	read_tweet(self, inputFile):
		 					-when given an already opened inputfile, will analyze a single tweet, putting into a list where each
							 element is a different part of the tweet, e.g. time, retweets, message, followers, etc. (This is used by later functions)

		 *	dictionary_to_scores(self, dictionary=None, names=True):
		 					-this will take in a keywords dictionary where the keys are the candidate names, and produces a new
							 dictionary where instead of keywords, there is a list with an element for each county in the state.
							 This will later be filled with the data in tally_scores

		 *	get_coordinates(self, tweet):
		 					-takes in whole tweet (as a list) and returns the coordinates. If not geotagged, it returns zeros, which
							 will result in the county option "Other"

		 *	point_inside_polygon(self, x,y,poly):
		 					-this program will determine if a set of points is inside of a polygon. This is usually used in
							visualization, however we are using it to help determine which county a geotagged tweet comes from. Note:
							this is a function pulled from the internet, I did not create it, though I did tweak it

		 *	find_county(self, coordinates, county_list=None):
		 					-takes in county_list, which is a list of all counties and their coordinates, and coordinates which is
							 a list of two float values to determine which county contains these coordinates. This uses
							 point_inside_polygon.

		 *	tally_scores(self, filelist, dictionary=None, names=True, notify=False):
		 					-this creates a score dictionary (see dictionary_to_scores) and then opens the input file(s) and determines
							 which candidate each tweet is about and which county it originates from, then returns the dictionary of
							 this information. Names is asking whether or not you want the name of each county to also be stored.
							 Leaving it as true will make this dictionary easier to read for states that have many counties (Iowa has 99!). Notify
				   			 will let you know (if it is a filelist) what file it is currently taking care of (1 file, 2 file, etc.)

		 *	check_if_filtered(self, filename):
		 					-this is for seeing how accurate the streaming is. It will see how many of the tweets are actually
							 in the correct state, and return it in terms of [ #  in the state, # not in the state]

		 *	get_location(self, tweet):
		 					-this returns the location that the user assigned to their tweet. This is NOT the coordinates, and is designed to place tweets
							 that are not geotagged. If the tweet is geotagged, it is better to use a different function for better accuracy, but this
							 will still work

		 *	g_u(self, tweet, time=True):
		 					-this takes in a tweet and returns the username associated with it (from whom it came)

'''


text:
require 'timecop'

describe Module do
  let(:test_module) { Module.new }
  let(:test_class)  { Class.new }

  describe '#cache_with_timeout' do
    before do
      @settings = 60

      expect { test_class.cache_with_timeout(:default) { rand } }.not_to raise_error
      expect { test_class.cache_with_timeout(:override, 60) { rand } }.not_to raise_error
      expect { test_class.cache_with_timeout(:override_proc, -> { @settings }) { rand } }.not_to raise_error

      expect { test_module.cache_with_timeout(:default) { rand } }.not_to raise_error
    end

    after do
      Module.clear_all_cache_with_timeout
    end

    it 'will create the class method on that class/module only' do
      expect(test_class).to  respond_to(:default)
      expect(test_module).to respond_to(:default)
      expect(Object).not_to  respond_to(:default)
      expect(Class).not_to   respond_to(:default)
      expect(Module).not_to  respond_to(:default)

      expect(test_class.new).not_to respond_to(:default)
      expect(Object).not_to         respond_to(:default)
      expect(Class).not_to          respond_to(:default)
      expect(Module).not_to         respond_to(:default)
    end

    it 'will return the cached value' do
      value = test_class.default(true)
      expect(test_class.default).to eq(value)

      value = test_module.default(true)
      expect(test_module.default).to eq(value)
    end

    it 'clears the cache even if the block raises an exception' do
      test_class.define_singleton_method(:call_count)  { @call_count ||= 0 }
      test_class.define_singleton_method(:call_count=) { |val| @call_count = val }
      test_class.cache_with_timeout(:flapping_method) do
        begin
          if test_class.call_count.even?
            test_class.call_count
          else
            raise "OOPS"
          end
        ensure
          test_class.call_count += 1
        end
      end

      expect(test_class.flapping_method(true)).to eq(0) # initializes cache to 0
      expect(test_class.flapping_method).to eq(0)       # returns from cache

      test_class.flapping_method(true) rescue nil       # blows up, clears cache for next request
      expect(test_class.flapping_method).to eq(2)       # sets new value since cache is cleared
      expect(test_class.flapping_method).to eq(2)       # returns from cache

      test_class.flapping_method(true) rescue nil       # blows up, clears cache for next request
      expect(test_class.flapping_method).to eq(4)       # sets new value since cache is cleared
      expect(test_class.flapping_method).to eq(4)       # returns from cache
    end

    it 'will not return the cached value when passed force_reload = true' do
      value = test_class.default(true)
      expect(test_class.default(true)).not_to eq(value)

      value = test_module.default(true)
      expect(test_module.default(true)).not_to eq(value)
    end

    it 'will return the cached value if the default timeout has not passed' do
      value = test_class.default(true)
      Timecop.travel(60) { expect(test_class.default).to eq(value) }
    end

    it 'will not return the cached value if the default timeout has passed' do
      value = test_class.default(true)
      Timecop.travel(600) { expect(test_class.default).not_to eq(value) }
    end

    it 'will return the cached value if the overridden timeout has not passed' do
      value = test_class.override(true)
      Timecop.travel(30) { expect(test_class.override).to eq(value) }
    end

    it 'will not return the cached value if the overridden timeout has passed' do
      value = test_class.override(true)
      Timecop.travel(120) { expect(test_class.override).not_to eq(value) }
    end

    it 'will return the cached value if the overridden timeout (via a proc) has not passed' do
      value = test_class.override_proc(true)
      Timecop.travel(30) { expect(test_class.override_proc).to eq(value) }
    end

    it 'will not return the cached value if the overridden timeout (via a proc) has passed' do
      value = test_class.override_proc(true)
      Timecop.travel(120) { expect(test_class.override_proc).not_to eq(value) }
    end

    it 'will return the cached value if the overridden timeout (via a proc) was changed but has not passed' do
      # Test a time jump value between the old settings value and the new to
      # prove that the new value is working
      @settings = 300 # 5 minutes
      value = test_class.override_proc(true)
      Timecop.travel(120) { expect(test_class.override_proc).to eq(value) }
      @settings = 60 # reset back to 1 minute
    end

    it 'will not return the cached value if the overridden timeout (via a proc) was changed and has passed' do
      # Test a time jump value between the old settings value and the new to
      # prove that the new value is working
      @settings = 300 # 5 minutes
      value = test_class.override_proc(true)
      Timecop.travel(600) { expect(test_class.override_proc).not_to eq(value) }
      @settings = 60 # reset back to 1 minute
    end

    it 'generated .*_clear_cache' do
      test_class.default
      test_class.override
      test_class.override_proc
      test_module.default

      test_class.default_clear_cache
      test_class.override_clear_cache
      test_class.override_proc_clear_cache
      test_module.default_clear_cache

      expect(test_class.default_cached?).to be_falsey
      expect(test_class.override_cached?).to be_falsey
      expect(test_class.override_proc_cached?).to be_falsey
      expect(test_module.default_cached?).to be_falsey
    end

    it 'generated .*_cached?' do
      expect(test_class.default_cached?).to be_falsey
      expect(test_class.override_cached?).to be_falsey
      expect(test_class.override_proc_cached?).to be_falsey
      expect(test_module.default_cached?).to be_falsey

      test_class.default
      test_class.override
      test_class.override_proc
      test_module.default

      expect(test_class.default_cached?).to be_truthy
      expect(test_class.override_cached?).to be_truthy
      expect(test_class.override_proc_cached?).to be_truthy
      expect(test_module.default_cached?).to be_truthy
    end

    it 'will cache with thread safety' do
      # If multiple threads access the in-memory cache without thread safety,
      #   and one tries to force reload, there is a small window where as it
      #   clears the current value, the other thread could get nil.
      test_class.cache_with_timeout(:thread_safety) { 2 }

      Thread.new do
        10000.times do
          test_class.thread_safety(true)
        end
      end

      10000.times do
        expect(test_class.thread_safety).to eq(2)
      end
    end
  end
end


text:
package cn.ittiger.video.presenter;

import cn.ittiger.video.bean.TtKb;
import cn.ittiger.video.factory.RetrofitFactory;
import cn.ittiger.video.http.DataType;
import cn.ittiger.video.util.DBManager;
import rx.Observable;

import java.util.HashMap;
import java.util.Map;

/**
 * @author laohu
 * @site http://ittiger.cn
 */
public class TtKbVideoPresenter extends VideoPresenter {
    private int mRefreshPageNum = 0;//刷新页

    @Override
    public Observable<String> getHttpCallObservable(int curPage) {

        String startKey = "";
        String newKey = "";

        Map<String, String> fieldMap = new HashMap<>();
        fieldMap.put("categoryId", "799999");
        if(curPage == 1) {//刷新
            mRefreshPageNum += -1;
            fieldMap.put("count", "10");
            fieldMap.put("pgnum", String.valueOf(mRefreshPageNum));
            TtKb ttKb = DBManager.getInstance().getSQLiteDB().queryOne(TtKb.class, "isRefresh=?", new String[]{"1"});
            if(ttKb != null) {
                newKey = ttKb.getNewkey();
                startKey = ttKb.getEndkey();
            }
        } else {
            fieldMap.put("count", "20");
            fieldMap.put("pgnum", String.valueOf(curPage));
            TtKb ttKb = DBManager.getInstance().getSQLiteDB().queryOne(TtKb.class, "isRefresh=?", new String[]{"0"});
            if(ttKb != null) {
                startKey = ttKb.getEndkey();
            }
        }
        fieldMap.put("startkey", startKey);
        fieldMap.put("newkey", newKey);
        fieldMap.put("param", "TouTiaoKuaiBao%09TTKBAndroid%09860582034297685%09huawei160918%09TTKB%091.4.0%09Android5.0.2%09null%09null%09PLK-AL10");

        return RetrofitFactory.getTtKbVideoService().getVideos(fieldMap);
    }

    @Override
    public DataType getType() {

        return DataType.TTKB;
    }
}