text:
/*
 * walk-table.h - Definition of walk table and walk table cell of
 *                Needleman-Wunsch alignment reconstruction.  Contains
 *                prototypes for functions implemented in walk_table.c.
 */

#ifndef __WALK_TABLE_H__
#define __WALK_TABLE_H__

#include <pthread.h>

/* arrow_t: Directions in a walk_table_t. */
typedef enum {left, up, diag} arrow_t;

/* walk_table_cell_t: Cell in a walk_table_t. */
typedef struct walk_table_cell {
        int diag;
        int left;
        int up;
        int diag_done;
        int left_done;
        int up_done;
        arrow_t src_direction;
        int in_optimal_path;
} walk_table_cell_t;

/* walk_table_t: An MxN table of walk_table_cells (i.e. matrix of
 *               walk_table_cell_t). */
typedef struct walk_table {
        int M;
        int N;
        walk_table_cell_t **cells;
        unsigned int branch_count;
        pthread_rwlock_t branch_count_rwlock;
} walk_table_t;

/*
 * Prototypes
 */

walk_table_t *alloc_walk_table(int M, int N);

void free_walk_table(walk_table_t *W, unsigned int nthreads);

void inc_branch_count(walk_table_t *W, unsigned int nthreads);

unsigned int get_branch_count(walk_table_t *W, unsigned int nthreads);

#endif /* __WALK_TABLE_H__ */


text:
/**
* @file LineGraphPercept.h
*
* @author Heinrich Mellmann
* Definition of class ProbabilisticQuadCompas
*/

#ifndef _LineGraphPercept_h_
#define _LineGraphPercept_h_

#include <vector>

#include "Tools/Math/Vector2.h"
#include "Tools/ImageProcessing/Edgel.h"
#include <Tools/DataStructures/Serializer.h>

class LineGraphPercept
{
public:
  void reset() {
    edgelsOnField.clear();

    edgelsInImage.clear();
    edgelsInImageTop.clear();

    lineGraphs.clear();
    lineGraphsTop.clear();
  }

public:
  //TODO: should this be double?
  std::vector<Edgel>  edgelsOnField;

  std::vector<EdgelD> edgelsInImage;
  std::vector<EdgelD> edgelsInImageTop;

  std::vector<std::vector<EdgelD>> lineGraphs;
  std::vector<std::vector<EdgelD>> lineGraphsTop;
};

namespace naoth
{
template<>
class Serializer<LineGraphPercept>
{
public:
	static void serialize(const LineGraphPercept& representation, std::ostream& stream);
	static void deserialize(std::istream& stream, LineGraphPercept& representation);
};
}

#endif // _LineGraphPercept_h_

redpajama_set_name: RedPajamaGithub


text:
package dummyApp.app;

import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Currency;
import java.util.Date;

import com.google.inject.Injector;
import com.premiumminds.billy.core.exceptions.SeriesUniqueCodeNotFilled;
import com.premiumminds.billy.core.persistence.dao.DAOInvoiceSeries;
import com.premiumminds.billy.core.persistence.entities.InvoiceSeriesEntity;
import com.premiumminds.billy.core.services.builders.GenericInvoiceEntryBuilder;
import com.premiumminds.billy.core.services.entities.Product;
import com.premiumminds.billy.core.services.entities.Tax;
import com.premiumminds.billy.core.services.exceptions.DocumentIssuingException;
import com.premiumminds.billy.core.services.exceptions.DocumentSeriesDoesNotExistException;
import com.premiumminds.billy.gin.services.exceptions.ExportServiceException;
import com.premiumminds.billy.persistence.entities.jpa.JPAInvoiceSeriesEntity;
import com.premiumminds.billy.portugal.BillyPortugal;
import com.premiumminds.billy.portugal.services.documents.util.PTIssuingParams;
import com.premiumminds.billy.portugal.services.entities.PTAddress;
import com.premiumminds.billy.portugal.services.entities.PTApplication;
import com.premiumminds.billy.portugal.services.entities.PTBusiness;
import com.premiumminds.billy.portugal.services.entities.PTContact;
import com.premiumminds.billy.portugal.services.entities.PTCreditNote;
import com.premiumminds.billy.portugal.services.entities.PTCreditNoteEntry;
import com.premiumminds.billy.portugal.services.entities.PTCustomer;
import com.premiumminds.billy.portugal.services.entities.PTGenericInvoice;
import com.premiumminds.billy.portugal.services.entities.PTInvoice;
import com.premiumminds.billy.portugal.services.entities.PTInvoiceEntry;
import com.premiumminds.billy.portugal.services.entities.PTProduct;
import com.premiumminds.billy.portugal.services.entities.PTTax;
import com.premiumminds.billy.portugal.services.export.exceptions.SAFTPTExportException;
import com.premiumminds.billy.portugal.services.export.pdf.creditnote.PTCreditNotePDFExportRequest;
import com.premiumminds.billy.portugal.services.export.pdf.creditnote.PTCreditNoteTemplateBundle;
import com.premiumminds.billy.portugal.services.export.pdf.invoice.PTInvoicePDFExportRequest;
import com.premiumminds.billy.portugal.services.export.pdf.invoice.PTInvoiceTemplateBundle;
import com.premiumminds.billy.portugal.services.export.saftpt.PTSAFTFileGenerator;
import com.premiumminds.billy.portugal.util.KeyGenerator;
import dummyApp.persistence.Billy;

public class PortugalDemoApp {
	private final Injector injector;

	public PortugalDemoApp(Injector injector) {
		this.injector = injector;
	}

	public void run()
			throws IOException,
			ParseException,
			DocumentIssuingException,
			SAFTPTExportException,
			ExportServiceException, SeriesUniqueCodeNotFilled, DocumentSeriesDoesNotExistException {
		SimpleDateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");

		BillyPortugal billyPortugal = new BillyPortugal(injector);
		PTIssuingParams invoiceParameters = getPtInvoiceIssuingParams();
		PTIssuingParams creditNoteParameters = getPtCreditNoteIssuingParams();

		PTApplication.Builder applicationBuilder = getPTApplicationBuilder(billyPortugal);
		PTApplication application = createPtApplication(billyPortugal, applicationBuilder);
		PTBusiness business = createPtBusiness(billyPortugal, applicationBuilder);
		PTCustomer customer = createPtCustomer(billyPortugal);

		createSeries(invoiceParameters.getInvoiceSeries(), business, "CCCC2345");
		createSeries(creditNoteParameters.getInvoiceSeries(), business, "CCCC2346");

		final PTTax flatTax = createFlatTax(billyPortugal);

		PTProduct product = createPtProduct(billyPortugal);
		PTProduct productExempt = createPtProductExempt(billyPortugal);
		PTProduct productFlat = createPtProductFlat(billyPortugal, flatTax);
		PTInvoice invoice = createPtInvoice(dateFormat, billyPortugal, business, invoiceParameters, customer, product, productExempt, productFlat);
		PTCreditNote creditNote = createPtCreditNote(dateFormat,
                billyPortugal,
                business,
                creditNoteParameters,
                customer,
                product,
                invoice);

		Date startDate = dateFormat.parse("01-01-2013");
		Date endDate = dateFormat.parse("01-01-2014");

		exportSAFT(billyPortugal, application, business, startDate, endDate);

		exportInvoicePDF(billyPortugal, application, invoice);
		exportCreditNotePDF(billyPortugal, application, creditNote);
	}

	private void createSeries(String series, PTBusiness business, String uniqueCode) {
		InvoiceSeriesEntity entity = new JPAInvoiceSeriesEntity();
		entity.setBusiness(business);
		entity.setSeries(series);
		entity.setSeriesUniqueCode(uniqueCode);
		DAOInvoiceSeries daoInvoiceSeries = injector.getInstance(DAOInvoiceSeries.class);
		daoInvoiceSeries.create(entity);
	}

	private PTTax createFlatTax(BillyPortugal billyPortugal) {
		final PTTax.Builder taxBuilder = this.injector.getInstance(PTTax.Builder.class);
		taxBuilder.setTaxRate(Tax.TaxRateType.FLAT, new BigDecimal("3.14"))
				.setContextUID(billyPortugal.contexts().continent().allContinentRegions().getUID())
				.setCode("code1")
				.setDescription("description 1")
				.setValidFrom(new Date(0))
				.setCurrency(Currency.getInstance("EUR"))
				.setValue(new BigDecimal("5.55"));

		return billyPortugal.taxes().persistence().create(taxBuilder);
	}

	private PTApplication.Builder getPTApplicationBuilder(BillyPortugal billyPortugal) throws MalformedURLException {
		PTContact.Builder contactBuilder = billyPortugal.contacts().builder();
		contactBuilder.setName("Developer 1")
				.setTelephone("2000000001");

		PTApplication.Builder applicationBuilder = billyPortugal.applications().builder();
		applicationBuilder.setDeveloperCompanyName("Developer Company Name")
				.setDeveloperCompanyTaxIdentifier("1000000001","PT")
				.setSoftwareCertificationNumber(Billy.SOFTWARE_CERTIFICATION)
				.setName("Billy")
				.setVersion("1.0")
				.setApplicationKeysPath(new URL("http://www.keys.path"))
				.addContact(contactBuilder)
				.setMainContact(contactBuilder);
		return applicationBuilder;
	}

	private PTApplication createPtApplication(BillyPortugal billyPortugal, PTApplication.Builder applicationBuilder) {
		return billyPortugal.applications().persistence().create(applicationBuilder);
	}

	private PTBusiness createPtBusiness(BillyPortugal billyPortugal, PTApplication.Builder applicationBuilder) {
		PTContact.Builder contactBuilder = billyPortugal.contacts().builder();
		contactBuilder.setName("CEO 1")
				.setTelephone("200000002");

		PTAddress.Builder addressBuilder = billyPortugal.addresses().builder();
		addressBuilder.setStreetName("Street name 2")
				.setNumber("2")
				.setPostalCode("1000-100")
				.setCity("Lisbon")
				.setISOCountry("PT")
				.setDetails("Av. 5 de Outubro Nº 2 1000-100 Lisboa");

		PTBusiness.Builder businessBuilder = billyPortugal.businesses().builder();
		businessBuilder.addApplication(applicationBuilder)
				.addContact(contactBuilder, true)
				.setMainContactUID(contactBuilder.build().getUID())
				.setName("Business 1")
				.setCommercialName("Business, INC")
				.setFinancialID("500003564", "PT")
				.setAddress(addressBuilder)
				.setBillingAddress(addressBuilder);

		return billyPortugal.businesses().persistence().create(businessBuilder);
	}

	private PTInvoice createPtInvoice(SimpleDateFormat dateFormat,
                                      BillyPortugal billyPortugal,
                                      PTBusiness business,
                                      PTIssuingParams invoiceParameters,
                                      PTCustomer customer,
                                      PTProduct product,
                                      PTProduct productExempt,
									  PTProduct productTax) throws ParseException, DocumentIssuingException, SeriesUniqueCodeNotFilled, DocumentSeriesDoesNotExistException {
		PTInvoice.Builder invoiceBuilder = billyPortugal.invoices().builder();

		Date invoiceDate = dateFormat.parse("01-03-2013");
		invoiceBuilder.setSelfBilled(false)
				.setCancelled(false)
				.setBilled(false)
				.setDate(invoiceDate)
				.setSourceId("User 1")
				.setSourceBilling(PTGenericInvoice.SourceBilling.P)
				.setBusinessUID(business.getUID())
				.setCustomerUID(customer.getUID());

		PTInvoiceEntry.Builder entryBuilder = billyPortugal.invoices().entryBuilder();
		entryBuilder
				.setCurrency(Currency.getInstance("EUR"))
				.setQuantity(new BigDecimal("10"))
				.setTaxPointDate(dateFormat.parse("01-02-2013"))
				.setUnitAmount(GenericInvoiceEntryBuilder.AmountType.WITH_TAX, new BigDecimal("100"))
				.setContextUID(billyPortugal.contexts().continent().allContinentRegions().getUID())
				.setProductUID(product.getUID())
				.setDescription(product.getDescription())
				.setUnitOfMeasure(product.getUnitOfMeasure());

		invoiceBuilder.addEntry(entryBuilder);

		entryBuilder = billyPortugal.invoices().entryBuilder();
		entryBuilder
				.setCurrency(Currency.getInstance("EUR"))
				.setQuantity(new BigDecimal("10.0"))
				.setTaxPointDate(dateFormat.parse("01-02-2013"))
				.setUnitAmount(GenericInvoiceEntryBuilder.AmountType.WITH_TAX, new BigDecimal("100"))
				.setContextUID(billyPortugal.contexts().continent().allContinentRegions().getUID())
				.setProductUID(productExempt.getUID())
				.setDescription(productExempt.getDescription())
				.setUnitOfMeasure(productExempt.getUnitOfMeasure())
				.setTaxExemptionCode("M99")
				.setTaxExemptionReason("reason 1");

		invoiceBuilder.addEntry(entryBuilder);

		entryBuilder = billyPortugal.invoices().entryBuilder();
		entryBuilder
				.setCurrency(Currency.getInstance("EUR"))
				.setQuantity(new BigDecimal("10.0"))
				.setTaxPointDate(dateFormat.parse("01-02-2013"))
				.setUnitAmount(GenericInvoiceEntryBuilder.AmountType.WITH_TAX, new BigDecimal("100"))
				.setContextUID(billyPortugal.contexts().continent().allContinentRegions().getUID())
				.setProductUID(productTax.getUID())
				.setDescription(productTax.getDescription())
				.setUnitOfMeasure(productTax.getUnitOfMeasure())
				.setTaxExemptionCode("M99")
				.setTaxExemptionReason("reason 1");

		invoiceBuilder.addEntry(entryBuilder);

		return billyPortugal.invoices().issue(invoiceBuilder, invoiceParameters);
	}

	private PTCreditNote createPtCreditNote(SimpleDateFormat dateFormat,
                                            BillyPortugal billyPortugal,
                                            PTBusiness business,
                                            PTIssuingParams invoiceParameters,
                                            PTCustomer customer,
                                            PTProduct product,
                                            PTInvoice invoice) throws ParseException, DocumentIssuingException, SeriesUniqueCodeNotFilled, DocumentSeriesDoesNotExistException {
		PTCreditNote.Builder creditNoteBuilder = billyPortugal.creditNotes().builder();

		Date creditNoteDate = dateFormat.parse("01-03-2013");
		creditNoteBuilder.setSelfBilled(false)
				.setCancelled(false)
				.setBilled(false)
				.setDate(creditNoteDate)
				.setSourceId("User 2")
				.setSourceBilling(PTGenericInvoice.SourceBilling.P)
				.setBusinessUID(business.getUID())
				.setCustomerUID(customer.getUID());

		PTCreditNoteEntry.Builder entryBuilder = billyPortugal.creditNotes().entryBuilder();
		entryBuilder.setAmountType(GenericInvoiceEntryBuilder.AmountType.WITH_TAX)
				.setCurrency(Currency.getInstance("EUR"))
				.setQuantity(new BigDecimal("10"))
				.setTaxPointDate(dateFormat.parse("01-02-2013"))
				.setUnitAmount(GenericInvoiceEntryBuilder.AmountType.WITH_TAX, new BigDecimal("100"))
				.setContextUID(billyPortugal.contexts().continent().allContinentRegions().getUID())
				.setProductUID(product.getUID())
				.setDescription(product.getDescription())
				.setUnitOfMeasure(product.getUnitOfMeasure())
				.setReferenceUID(invoice.getUID())
				.setReason("some reason 1");

		creditNoteBuilder.addEntry(entryBuilder);

		return billyPortugal.creditNotes().issue(creditNoteBuilder, invoiceParameters);
	}

	private PTProduct createPtProduct(BillyPortugal billyPortugal) {
		PTProduct.Builder productBuilder = billyPortugal.products().builder();
		productBuilder.setDescription("product 1")
				.setNumberCode("1")
				.setProductCode("1")
				.setType(Product.ProductType.GOODS)
				.setUnitOfMeasure("kg")
				.setProductGroup("group 1")
				.addTaxUID(billyPortugal.taxes().continent().normal().getUID());

		return billyPortugal.products().persistence().create(productBuilder);
	}

	private PTProduct createPtProductExempt(BillyPortugal billyPortugal) {
		PTProduct.Builder productBuilder = billyPortugal.products().builder();
		productBuilder.setDescription("product 2")
				.setNumberCode("2")
				.setProductCode("2")
				.setType(Product.ProductType.GOODS)
				.setUnitOfMeasure("kg")
				.setProductGroup("group 1")
				.addTaxUID(billyPortugal.taxes().exempt().getUID());

		return billyPortugal.products().persistence().create(productBuilder);
	}

	private PTProduct createPtProductFlat(BillyPortugal billyPortugal, PTTax flatTax) {
		PTProduct.Builder productBuilder = billyPortugal.products().builder();
		productBuilder.setDescription("product 3")
				.setNumberCode("3")
				.setProductCode("3")
				.setType(Product.ProductType.GOODS)
				.setUnitOfMeasure("kg")
				.setProductGroup("group 1")
				.addTaxUID(flatTax.getUID());

		return billyPortugal.products().persistence().create(productBuilder);
	}

	private PTIssuingParams getPtInvoiceIssuingParams() {
		PTIssuingParams invoiceParameters = PTIssuingParams.Util.newInstance();

		KeyGenerator gen = new KeyGenerator(getClass().getResource(App.PRIVATE_KEY_DIR));

		invoiceParameters.setPrivateKey(gen.getPrivateKey());
		invoiceParameters.setPublicKey(gen.getPublicKey());
		invoiceParameters.setInvoiceSeries("A");
		invoiceParameters.setPrivateKeyVersion("1");
		return invoiceParameters;
	}

	private PTIssuingParams getPtCreditNoteIssuingParams() {
		PTIssuingParams invoiceParameters = PTIssuingParams.Util.newInstance();

		KeyGenerator gen = new KeyGenerator(getClass().getResource(App.PRIVATE_KEY_DIR));

		invoiceParameters.setPrivateKey(gen.getPrivateKey());
		invoiceParameters.setPublicKey(gen.getPublicKey());
		invoiceParameters.setInvoiceSeries("B");
		invoiceParameters.setPrivateKeyVersion("1");
		return invoiceParameters;
	}

	private PTCustomer createPtCustomer(BillyPortugal billyPortugal) {
		PTContact.Builder contactBuilder = billyPortugal.contacts().builder();
		contactBuilder.setName("Customer 1")
				.setTelephone("telephone 1");

		PTAddress.Builder addressBuilder = billyPortugal.addresses().builder();
		addressBuilder.setStreetName("Customer 1 street name 1")
				.setNumber("2")
				.setPostalCode("1000-100")
				.setCity("Lisbon")
				.setISOCountry("PT")
				.setDetails("Av. 5 de Outubro Nº 2 1000-100 Lisboa");

		PTCustomer.Builder customerBuilder = billyPortugal.customers().builder();
		customerBuilder.setName("Customer name 1")
				.setTaxRegistrationNumber("123456789", "PT")
				.addAddress(addressBuilder, true)
				.setBillingAddress(addressBuilder)
				.setShippingAddress(addressBuilder)
				.setHasSelfBillingAgreement(false)
				.addContact(contactBuilder);

		return billyPortugal.customers().persistence().create(customerBuilder);
	}

	private void exportSAFT(BillyPortugal billyPortugal,
                            PTApplication application,
                            PTBusiness business,
                            Date startDate,
                            Date endDate)
			throws SAFTPTExportException, IOException {
		billyPortugal.saft().export(application.getUID(),
				business.getUID(),
				startDate,
				endDate,
				"saft.xml",
				PTSAFTFileGenerator.SAFTVersion.CURRENT, true);
	}

	private void exportInvoicePDF(BillyPortugal billyPortugal,
                                  PTApplication application,
                                  PTInvoice invoice)
            throws IOException, ExportServiceException {
		InputStream xslInputStream = this.getClass().getResourceAsStream(Billy.INVOICE_XSL_PATH);

		InputStream logoStream = this.getClass().getResourceAsStream("/logoBig.png");
		Files.copy(logoStream, Paths.get(Billy.LOGO_PATH), StandardCopyOption.REPLACE_EXISTING);
		PTInvoiceTemplateBundle templateBundle = new PTInvoiceTemplateBundle(Billy.LOGO_PATH, xslInputStream,
				application.getSoftwareCertificationNumber().toString());

		String resultPath = new SimpleDateFormat("'pt_invoice_'yyyy-MM-dd'T'HH:mm:ss'.pdf'").format(new Date());
		billyPortugal.invoices().pdfExport(
				new PTInvoicePDFExportRequest(invoice.getUID(), templateBundle, resultPath));
	}

	private void exportCreditNotePDF(BillyPortugal billyPortugal,
                                     PTApplication application,
                                     PTCreditNote creditNote)
            throws IOException, ExportServiceException {
		InputStream xslInputStream = this.getClass().getResourceAsStream(Billy.CREDIT_NOTE_XSL_PATH);

		InputStream logoStream = this.getClass().getResourceAsStream("/logoBig.png");
		Files.copy(logoStream, Paths.get(Billy.LOGO_PATH), StandardCopyOption.REPLACE_EXISTING);
		PTCreditNoteTemplateBundle templateBundle = new PTCreditNoteTemplateBundle(Billy.LOGO_PATH, xslInputStream,
				application.getSoftwareCertificationNumber().toString());

		String resultPath = new SimpleDateFormat("'pt_creditNote_'yyyy-MM-dd'T'HH:mm:ss'.pdf'").format(new Date());
		billyPortugal.creditNotes().pdfExport(
				new PTCreditNotePDFExportRequest(creditNote.getUID(), templateBundle, resultPath));
	}

}


text:
require 'spec_helper'
require 'yaml'

describe SCV::Config do

  let(:file_store) { double(VCSToolkit::FileStore)            }
  subject(:config) { SCV::Config.new file_store, 'config.yml' }

  let(:content) do
    {
      'test' => {
        'inside' => 'one',
        'list-test' => [
          'item 1',
          'item 2',
        ]
      }
    }
  end

  before(:each) do
    allow(file_store).to receive(:file?).with('config.yml').and_return(true)
    allow(file_store).to receive(:fetch).with('config.yml').and_return(content.to_yaml)
  end

  it 'loads the configuration file in initialize' do
    expect(file_store).to receive(:file?).with('config.yml').and_return(true)
    expect(file_store).to receive(:fetch).with('config.yml').and_return(content.to_yaml)

    expect(config.data).to eq content
  end

  describe '#load' do
    it 'loads the config file' do
      expect(file_store).to receive(:fetch).with('config.yml').and_return(content.to_yaml)
      config.load

      expect(config.data).to eq content
    end

    it 'creates the config file if it does not exitst' do
      expect(file_store).to receive(:file?).with('config.yml').and_return(false)

      expect(config.data).to eq({})
    end
  end

  describe '#save' do
    it 'saves the configuration file' do
      config.data = content
      expect(file_store).to receive(:store).with('config.yml', content.to_yaml)

      config.save
    end
  end

  describe '#[]' do
    it 'proxies data#[]' do
      expect(config.data).to receive(:[]).with('key').and_return('value')
      expect(config['key']).to eq 'value'
    end

    it 'splits the key by dots and resolves deep structures' do
      expect(config['test.list-test']).to match_array [
        'item 1',
        'item 2'
      ]
    end
  end

  describe '#[]=' do
    it 'proxies data#[]=' do
      expect(config.data).to receive(:[]=).with('key', 'value').and_return('value')
      expect(config['key'] = 'value').to eq 'value'
    end

    it 'splits the key by dots and resolves deep structures' do
      config['test.inside'] = 'changed item'

      expect(config['test']['inside']).to eq 'changed item'
    end

    it 'sets the keys along the path to hashes' do
      config['this.should.create.hashes'] = 'yay!'

      expect(config['this']['should']['create']['hashes']).to eq 'yay!'
    end
  end

  describe '#delete' do
    it 'deletes direct keys' do
      config.delete 'test'
      expect(config.data.key? 'test').to be_false
    end

    it 'deletes nested keys' do
      config.delete 'test.inside'
      expect(config.data.key? 'test'     ).to be_true
      expect(config['test'].key? 'inside').to be_false
    end
  end
end


text:
namespace Microsoft.Azure.Management.VideoAnalyzer
{
    using Microsoft.Rest;
    using Microsoft.Rest.Azure;
    using Models;
    using System.Collections;
    using System.Collections.Generic;
    using System.Threading;
    using System.Threading.Tasks;

    /// <summary>
    /// PipelineJobOperationStatusesOperations operations.
    /// </summary>
    public partial interface IPipelineJobOperationStatusesOperations
    {
        /// <summary>
        /// Get the operation statuses.
        /// </summary>
        /// <remarks>
        /// Get the operation status of a pipeline job with the given
        /// operationId.
        /// </remarks>
        /// <param name='resourceGroupName'>
        /// The name of the resource group. The name is case insensitive.
        /// </param>
        /// <param name='accountName'>
        /// The Azure Video Analyzer account name.
        /// </param>
        /// <param name='pipelineJobName'>
        /// The pipeline job name.
        /// </param>
        /// <param name='operationId'>
        /// The operation ID.
        /// </param>
        /// <param name='customHeaders'>
        /// The headers that will be added to request.
        /// </param>
        /// <param name='cancellationToken'>
        /// The cancellation token.
        /// </param>
        /// <exception cref="ErrorResponseException">
        /// Thrown when the operation returned an invalid status code
        /// </exception>
        /// <exception cref="Microsoft.Rest.SerializationException">
        /// Thrown when unable to deserialize the response
        /// </exception>
        /// <exception cref="Microsoft.Rest.ValidationException">
        /// Thrown when a required parameter is null
        /// </exception>
        Task<AzureOperationResponse<PipelineJobOperationStatus>> GetWithHttpMessagesAsync(string resourceGroupName, string accountName, string pipelineJobName, string operationId, Dictionary<string, List<string>> customHeaders = null, CancellationToken cancellationToken = default(CancellationToken));
    }
}


text:
Based on a routing schema:
- A state object can be derived from the current pathname
- A state object can be used to generate a path name

## API Usage

Define the routes for your application and how they should map to application state
```js
import {route} from '@sanity/state-router'

const router = route('/', [
  route('/products/:productId'),
  route('/users/:userId'),
  route('/:page'),
])

router.encode({})
// => '/'
router.decode('/')
// => {}

router.encode({productId: 54})
// => '/products/54'

router.decode('/products/54')
// => {productId: 54}

router.encode({userId: 22})
// => '/users/22'

router.decode('/users/54')
// => {userId: 54}

router.encode({page: 'about'})
// => '/about'

router.decode('/about')
// => {page: about}

```

## React usage
### Setup routes and provider

```jsx
import {route} from '@sanity/state-router'
import {RouterProvider, withRouter} from '@sanity/state-router/components'

const router = route('/', [
  route('/bikes/:bikeId')
])

const history = createHistory()

function handleNavigate(nextUrl, {replace} = {}) {
  if (replace) {
    history.replace(nextUrl)
  } else {
    history.push(nextUrl)
  }
}

const App = withRouter(function App({router}) {
  if (router.state.bikeId) {
    return <BikePage id={router.state.bikeId} />
  }
  return (
    <div>
      <h1>Welcome</h1>
      <StateLink state={{bikeId: 22}}>Go to bike 22</StateLink>
    </div>
  )
})

function render(location) {
  ReactDOM.render((
    <RouterProvider
     router={router}
     onNavigate={handleNavigate}
     state={router.decode(location.pathname)}>
      <App />
    </RouterProvider>
  ), document.getElementById('container'))
}
history.listen(() => render(document.location))

```

## API

- `route(path : string, ?options : Options, ?children : ) : Router`
- `route.scope(name : string, path : string, ?options : Options, ?children : ) : Router`
- `Router`:
  - `encode(state : object) : string`
  - `decode(path : string) : object`
  - `isRoot(path : string) : boolean`
  - `getBasePath() : string`,
  - `isNotFound(pathname: string): boolean`
  - `getRedirectBase(pathname : string) : ?string`

- `RouteChildren`:
  ```
  Router | [Router] | ((state) => Router | [Router])
  ```
- `Options`:
  ```
  {
    path?: string,
    children?: RouteChildren,
    transform?: {[key: string] : Transform<*>},
    scope?: string
  }
  ```

  - `children` can be either another router returned from another `route()-call`, an array of routers or a function that gets passed the matched parameters, and conditionally returns child routes

## Limitations
- Parameterized paths *only*. Each route must have at least one unique parameter. If not, there's no way of unambiguously resolve a path from an empty state.

Consider the following routes:
```js
const router = route('/', [
  route('/about'),
  route('/contact')
])
```

What route should be resolved from an empty state? Since both `/about` and `/contact` above resolves to an empty state object, there's no way to encode an empty state unambiguously back to either of them. The solution to this would be to introduce the page name as a parameter instead:
```js
const router = route('/', route('/:page'))
```

Now, `/about` would resolve to the state `{page: 'about'}` which unambiguously can map back to `/page`, and an empty state can map to `/`. To figure out if you are on the index page, you can check for `state.page == null`, (and set the state.page to null to navigate back to the index)

### No query params
Query parameters doesn't work too well with router scopes as they operate in a global namespace. A possible workaround is to "fake" query params in a path segment using transforms:
```js
function decodeParams(pathsegment) {
  return pathsegment.split(';')
    .reduce((params, pair) => {
      const [key, value] = pair.split('=')
      params[key] = value
      return params
    }, {})
}
function encodeParams(params) {
  return Object.keys(params)
    .map(key => `${key}=${params[key]}`)
    .join(';')
}

const router = route('/some/:section/:settings', {
  transform: {
    settings: {
      toState: decodeParams,
      toPath: encodeParams
    }
  }
}, route('/other/:page'))
```
This call...
```js
router.decode('/some/bar/width=full;view=details')
```
...will return the following state
```js
{
  section: 'bar',
  settings: {
    width: 'full',
    view: 'details',
  }
}
```
Conversely calling
```js
router.encode({
  section: 'bar',
  settings: {
    width: 'full',
    view: 'details',
  }
})
```
will return
```
/some/bar/width=full;view=details
```

## Scopes
A scope is a separate router state space, allowing different parts of an application to be completely agnostic about the overall routing schema is like. Let's illustrate:

```js
import {route} from './src'
function findAppByName(name) {
  return name === 'pokemon' && {
    name: 'pokemon',
    router: route('/:section', route('/:pokemonName'))
  }
}

const router = route('/', [
  route('/users/:username'),
  route('/apps/:appName', params => {
    const app = findAppByName(params.appName)
    return app && route.scope(app.name, '/', app.router)
  })
])
```
Decoding the following path...
```js
router.decode('/apps/pokemon/stats/bulbasaur')
```
...will give us the state:
```js
{
  appName: 'pokemon',
  pokemon: {
    section: 'stats',
    pokemonName: 'bulbasaur'
  }
}
```

## Intents

An _intent_ is a kind of global route that can be used for dispatching user actions. The intent route can be mounted with
```js
route.intents(<basePath>)
```
Intent links bypasses scoping, and will always be mapped to the configured `basePath`.

An intent consists of a name, e.g. `open` and a set of parameters, e.g. `{id: 'abc33'}` and the easiest way to make a link to an intent is using the `IntentLink` React component:

```jsx
<IntentLink intent="open" params={{id: abc33}}>Open document</IntentLink>
```
This will generate an `<a` tag with a href like `/<base path>/open/id=abc33` depending on where the intent handler is mounted

State router comes with a built in intent-route parser that decodes an intent route to route state.

Full example:
```
const router = route('/', [
  route('/users/:username'),
  route.intents('/intents') // <-- sets up intent routes at the /intents base path
])
```

Decoding the url `/intents/open/id=abc33` will produce the following state:

```js
{
  intent: 'open',
  params: {id: 'abc33'}
}
```

It is now up to your application logic to translate this intent into an action, and redirect accordingly.

## 404s

To check whether a path name matches, you can use the isNotFound method on the returned router instance:

```js
const router = route('/pages/:page')

router.isNotFound('/some/invalid/path')
// => true

```

## Base paths

Using a base path is as simple as adding a toplevel route with no params:
```js
const router = route('/some/basepath', [
 route('/:foo'),
 route('/:bar')
])
```
Any empty router state will resolve to `/some/basepath`. To check if you should redirect to the base path on app init, you can use the `router.isRoot(path)` and `router.getBasePath()` method:

```js
if (router.isRoot(location.pathname)) {
  const basePath = router.getBasePath()
  if (basePath !== location.pathname) {
    history.replaceState(null, null, basePath)
  }
}
```

For convenience, this check is combined in the method `router.getRedirectBase()`, that if a redirect is needed, will return the base path, otherwise `null`

```js
const redirectTo = router.getRedirectBase(location.pathname)
if (redirectTo) {
  history.replaceState(null, null, redirectTo)
}
```

## License

MIT-licensed


text:
package cn.remex.db.sql;

import cn.remex.core.exception.ServiceCode;
import cn.remex.core.reflect.ReflectUtil;
import cn.remex.core.util.Assert;
import cn.remex.core.util.Judgment;
import cn.remex.core.util.Param;
import cn.remex.core.util.StringHelper;
import cn.remex.db.DbCvo;
import cn.remex.db.exception.RsqlConnectionException;
import cn.remex.db.rsql.RsqlConstants;
import cn.remex.db.rsql.RsqlUtils;
import cn.remex.db.rsql.connection.RDBManager;
import cn.remex.db.rsql.connection.dialect.Dialect;
import cn.remex.db.rsql.model.Modelable;

import javax.persistence.ManyToMany;
import javax.persistence.OneToMany;
import java.lang.reflect.Method;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

/**
 * 名称:
 * 缩写:
 * 用途:
 * Created by yangy on 2016/1/12 0012.
 */
public class Select<T extends Modelable> {
    private Dialect dialect;
    private DbCvo<T> dbCvo;
    private String beanName;
    private String tableName;
    private String aliasName;

    private Param<Integer> tableIndex;// 用于控制数据库虚拟表明的序号。
    private Param<Integer> paramIndex;// 用于控制数据库参数的序号。

    //存储相应的变量
    private List<NamedParam> namedParams;
    // 定义对象类型数据的查询sql
    private String prefix = "SELECT \r\n\t";
    private String part_from;// = "\r\nFROM "+dialect.quoteKey(beanName)+" "+_tableAliasName;
    private String part_where;// = "\r\n  " + cvo.getFilter().toSQL(_tableAliasName, namedParams, 0);
    // 动态排序必须放到dao
    // String part_order = "\r\n"+obtainSQLOrder(Option.getOrder());
    private StringBuilder part_column_sb = new StringBuilder();
    private StringBuilder part_jion_sb = new StringBuilder();
    private StringBuilder part_groupby_sb = new StringBuilder("GROUP BY \r\n\t");
    private StringBuilder sqlString = new StringBuilder();

    public Select(DbCvo<T> dbCvo) {
        if(dbCvo._getTableIndex()==null){
            dbCvo._setTableIndex(new Param(0));
        }
        if(dbCvo._getParamIndex()==null){
            dbCvo._setParamIndex(new Param(0));
        }
        if (dbCvo._getTableAliasName() == null) {
            dbCvo._setTableAliasName((dbCvo._getTableAliasName()==null?"T":dbCvo._getTableAliasName())+(dbCvo._getTableIndex().param++));
        }
        if (dbCvo._getNamedParams() == null) {
            dbCvo._setNamedParams(new ArrayList<>());
        }

        this.dbCvo = dbCvo;
        this.namedParams = dbCvo._getNamedParams();
        this.dialect = RDBManager.getLocalSpaceConfig(dbCvo._getSpaceName()).getDialect();
        this.tableIndex = dbCvo._getTableIndex();
        this.paramIndex = dbCvo._getParamIndex();
        this.beanName = this.dbCvo.getBeanName();
        this.tableName = this.dbCvo.getBeanName();
        this.aliasName = dbCvo._getTableAliasName()==null?("T"+(tableIndex.param++)):dbCvo._getTableAliasName();
        part_from = "\r\nFROM "+dialect.aliasTableName(tableName, aliasName);
        part_where = "\r\n  WHERE 1=1 \r\n";
    }

    /**
     * 向select中添加需要查询本表的基本列;
     * <p>
     * 如果dataColumns完全放弃后,这个方法将被{@link #appendColumn(SqlColumn)}替代
     */
    @Deprecated
    public void appendBaseColumn(String fieldName, String fieldAlias) {
        part_column_sb.append(dialect.aliasFullName(aliasName, fieldName, fieldAlias)).append(",\r\n\t");
    }

    public void appendColumn(SqlColumn column) {
        String curAliasName = null == column.getSupColumn() || null == column.getSupColumn().getFieldAliasName() ? aliasName : column.getSupColumn().getAliasName();
        if (dbCvo._isHasAggregateBy() && column.getAggregateFunction() != null) {
            part_column_sb.append(dialect.aliasAggrFun(useAggregateFunction(curAliasName, column.getAggregateFunction(), column.getFieldAliasName()), aliasAggregateFunction(column.getAggregateFunction(), column.getFieldName()))).append(",\r\n\t");
        } else {
            part_column_sb.append(dialect.aliasFullName(curAliasName, column.getFieldName(), column.getFieldAliasName())).append(",\r\n\t");
        }
    }

    public void leftJoinModel(SqlColumn column) {
        Class fieldClass = column.getNodeClass();
        String jioned_Tb = StringHelper.getClassSimpleName(fieldClass);
        String jioned_Tb_alias = StringHelper.getAbbreviation(fieldClass) + tableIndex.param++;

        column.setTableName(jioned_Tb);
        column.setAliasName(jioned_Tb_alias);//虚拟表名只能再select实例化并调用去生成sql时才能产生

        part_jion_sb.append("\r\nLEFT JOIN ").append(dialect.aliasTableName(jioned_Tb, jioned_Tb_alias))
                .append("\r\n\tON ")
                .append(dialect.quoteFullName(jioned_Tb_alias, RsqlConstants.SYS_id))
                .append("=")
                .append(dialect.quoteFullName((null == column.getSupColumn() || null == column.getSupColumn().getFieldAliasName() ? aliasName : column.getSupColumn().getAliasName()), column.getFieldName()))
                .append(" ")
        ;
        if (null != column.getFilters())
            part_where += ((" AND ") + (column.getFilters().toSQL(false, column.getAliasName(), namedParams, paramIndex, tableIndex, dbCvo)));

        //系统关键字段默认加载
        if (!dbCvo._isHasGroupBy() && !dbCvo._isSubStatment())
            part_column_sb.append(dialect.aliasFullName(jioned_Tb_alias, RsqlConstants.SYS_id, column.obtainChain().toString() + RsqlConstants.SYS_id)).append(",\r\n\t");
    }

    public void leftJoinList(SqlColumn column) {
        Class fieldClass = column.getNodeClass();
        String jioned_Tb = StringHelper.getClassSimpleName(fieldClass);
        String jioned_Tb_alias = StringHelper.getAbbreviation(fieldClass) + tableIndex.param++;

        column.setTableName(jioned_Tb);
        column.setAliasName(jioned_Tb_alias);//虚拟表名只能再select实例化并调用去生成sql时才能产生

        //这里要考虑OneToMany / ManyToMany 的不同情况;OneToMany无论是否级联关系都是在多方维护;ManyToMany无论是否级联关系都是在中间表中维护
        Class parentClass = column.getSupColumn() == null ? dbCvo.getBeanClass() : (Class) column.getSupColumn().getNodeClass();
        String parentAlias = column.getSupColumn() == null || Judgment.nullOrBlank(column.getSupColumn().getAliasName()) ? aliasName : column.getSupColumn().getAliasName();
        OneToMany otm = ReflectUtil.getAnnotation(parentClass, column.getFieldName(), OneToMany.class);
        ManyToMany curMtm = ReflectUtil.getAnnotation(parentClass, column.getFieldName(), ManyToMany.class);
        if (otm != null) {
            part_jion_sb.append("\r\nLEFT JOIN ").append(dialect.aliasTableName(jioned_Tb, jioned_Tb_alias))
                    .append("\r\n\tON ")
                    .append(dialect.quoteFullName(jioned_Tb_alias, otm.mappedBy()))
                    .append("=")
                    .append(dialect.quoteFullName((null == column.getSupColumn() || null == column.getSupColumn().getFieldAliasName() ? aliasName : column.getSupColumn().getAliasName()), RsqlConstants.SYS_id))
                    .append(" ")
            ;

        } else {
            String primaryBeanName = parentClass.getSimpleName();//主方类名,默认为当前,根据多多关系检查后,修改
            Class<?> primaryBeanClass = parentClass;//主方类,默认为当前,根据多多关系检查后,修改
            String primaryFieldName = column.getFieldName();//主方多对多属性名,默认为当前,根据多多关系检查后,修改
            Type primaryFieldType = column.getNodeType();//主方多对多属性类型,默认为当前,根据多多关系检查后,修改
            Class<?> tgtBeanClass = ReflectUtil.getListActualType(column.getNodeType());

            /**********************判断是多对多***********************************/
            // 如果是多对多,则检查多对多关系
            boolean meIsManyToManyPrimary = true;
            String tgtMappedBy = null, mappedBy = null;//多对多中对方属性名称
            // 对方表的集合字段
            Map<String, Method> tcGetters = SqlType.getGetters(tgtBeanClass, FieldType.TCollection);

            // mb必须主从双方都配置,主方还需配置targetEntity。从方无需配置targetEntity属性。
            if (curMtm != null && !Judgment.nullOrBlank(curMtm.mappedBy())) {
                mappedBy = curMtm.mappedBy();

                Assert.notNull(mappedBy, ServiceCode.FAIL, new StringBuilder("显式指定为双方维护的多对多映射中本类").append(dbCvo.getBeanClass().getName()).append("没有指定对方类多对多属性:").append(mappedBy).toString(), RsqlConnectionException.class);
                Assert.isTrue(tcGetters.containsKey(mappedBy), ServiceCode.FAIL, new StringBuilder(" 显式指定为双方维护的多对多映射中对方类").append(tgtBeanClass.getName()).append("不存在该多对多属性:").append(mappedBy).toString(), RsqlConnectionException.class);

                // 获取对方类多对多声明
                ManyToMany tgtMtm = ReflectUtil.getAnnotation(tgtBeanClass, mappedBy, ManyToMany.class);
                Assert.notNull(tgtMtm, ServiceCode.FAIL, "显示声明多对多时,需要双方指定ManyToMany声明,此处对方类的ManyToMany声明为空!请更正!", RsqlConnectionException.class);
                tgtMappedBy = tgtMtm.mappedBy();// 对方类的多对多属性

                Assert.notNull(tgtMappedBy, ServiceCode.FAIL, new StringBuilder("显式指定为双方维护的多对多映射中对方类").append(tgtBeanClass.getName()).append("没有指定本类的多对多属性:").append(mappedBy).toString(), RsqlConnectionException.class);

                Class<?> curTe = curMtm.targetEntity(); // 检查维护关系 本类中的targetEntity应该等于对方类型
                Class<?> tgtTe = tgtMtm.targetEntity(); // 检查维护关系 对方类中的targetEntity应该等于本类型

                Assert.isTrue((null == curTe && null == tgtTe) || (null != curTe && null != tgtTe), ServiceCode.FAIL, new StringBuilder("显式指定为双方维护的多对多映射中,ManyToMany声明不能同时设置targetEntity或者targetEntity同时为空!设置targetEntity的为主维护方。").append(mappedBy).toString(),
                        RsqlConnectionException.class);

                meIsManyToManyPrimary = !"void".equals(curTe.toString());//targetEntity不为空的一方为主维护方
            }

            //如果不是主表,则修改对应的表名和字段。
            if (!meIsManyToManyPrimary) {
                primaryBeanName = tgtBeanClass.getSimpleName();
                primaryBeanClass = tgtBeanClass;
                primaryFieldName = mappedBy;//自己mappedby的值刚好是对方的属性
                primaryFieldType = SqlType.getFields(primaryBeanClass, FieldType.TCollection).get(primaryFieldName);
            }
            // 多对多 end

            Map<String, ColumnType> ctColumns = RsqlUtils.obtainSKeyCollectionTableColumns(primaryBeanName, primaryFieldName, primaryFieldType);
            Iterator<Map.Entry<String, ColumnType>> ctkeys = ctColumns.entrySet().iterator();
            String Mid_Foreign_column = ctkeys.next().getKey();
            String Mid_Primary_column = ctkeys.next().getKey();

            String Mid_table_name = RsqlUtils.obtainSKeyCollectionTableName(dialect,primaryBeanName,primaryFieldName);
            String Mid_table_alias = StringHelper.getAbbreviation(parentClass)+"_"+StringHelper.getAbbreviation(tgtBeanClass)+"_"+tableIndex.param++;
            part_jion_sb
                    .append("\n\rLEFT JOIN ").append(dialect.aliasTableName(Mid_table_name, Mid_table_alias))
                    .append("\n\r\tON ")
                    .append(dialect.quoteFullName(Mid_table_alias, meIsManyToManyPrimary ? Mid_Primary_column : Mid_Foreign_column)).append("=")
                    .append(dialect.quoteFullName(parentAlias, RsqlConstants.SYS_id))
                    .append("\n\rLEFT JOIN ").append(dialect.aliasTableName(jioned_Tb, jioned_Tb_alias))
                    .append("\n\r\tON ")
                    .append(dialect.quoteFullName(Mid_table_alias, meIsManyToManyPrimary ? Mid_Foreign_column : Mid_Primary_column)).append("=")
                    .append(dialect.quoteFullName(jioned_Tb_alias, RsqlConstants.SYS_id))
            ;
        }

        if (null != column.getFilters())
            part_where += ((" AND ") + (column.getFilters().toSQL(false, column.getAliasName(), namedParams, paramIndex, tableIndex, dbCvo)));

        //系统关键字段默认加载 //TODO LHY取消id的加载,如果需要则需显式调用withColumn
        if (!dbCvo._isHasGroupBy() && !dbCvo._isSubStatment())
            part_column_sb.append(dialect.aliasFullName(jioned_Tb_alias, RsqlConstants.SYS_id, column.obtainChain().toString() + RsqlConstants.SYS_id)).append(",\r\n\t");
    }

    public String sqlString() {
        String where = dbCvo.getFilter().toSQL(false, aliasName, namedParams, paramIndex, tableIndex, dbCvo);
        if (!Judgment.nullOrBlank(where.trim()))
            part_where += (" AND ") + where;

        // 代码与代码名称映射功能添加。
        RsqlUtils.dealWithCodeRef(dbCvo._getSpaceName(), dbCvo.getBeanClass(), aliasName, part_column_sb);

        String part_column = part_column_sb.delete(part_column_sb.length() - 4, part_column_sb.length()).toString();
        String part_groupBy = dbCvo._isHasGroupBy() ? part_groupby_sb.delete(part_groupby_sb.length() - 4, part_groupby_sb.length()).toString() : "";

        sqlString.append(prefix).append(part_column).append(part_from).append(
                part_jion_sb.toString()).append(part_where).append(part_groupBy)
        // .append(part_order)
        ;

        // 删除多余空间
        ((ArrayList) namedParams).trimToSize();

        return sqlString.toString();
    }

    public void groupBy(SqlColumn column) {
        String curAliasName = null == column.getSupColumn() || null == column.getSupColumn().getFieldAliasName() ? aliasName : column.getSupColumn().getAliasName();
        part_groupby_sb.append(curAliasName).append(".").append(dialect.openQuote()).append(column.getFieldName()).append(dialect.closeQuote()).append(",\r\n\t");
    }

    private String useAggregateFunction(String curAliasName, AggregateFunction fun, String fieldName) {
        return fun.toString() + "(" + curAliasName + "." + dialect.openQuote() + fieldName + dialect.closeQuote() + ")";
    }

    public static String aliasAggregateFunction(AggregateFunction fun, String fieldName) {
        if (fun.toString().startsWith("Field_")) {
            return fieldName;
        } else {
            return fieldName + "$" + fun.toString();
        }
    }
}


text:
layout: post
title: Predicting Data Scientist Salaries Using Logistic Regression
---

This project began as a web scraping exercise for our Data Science Immersive course which turned into a project in which we use Logistic Regression to predict whether a Data Science job listing in Boston will pay over $90,000.

The class brainstormed what predictors might be useful for a model and decided on four:

- Years of experience
- Whether the term 'scientist' is in the job title
- Whether the term 'start-up' appears in the job description
- Whether the term 'Phd' appears in the job description

The class split into teams to perform the web scraping, each team responsible for one of the four features.  Once the data was gathered we individually cleaned and merged the data and built models to attempt to predict whether listings had a salary of over $90K.

## Importing and Cleaning the CSVs

Each job's feature was stored in seperate CSV files with a job ID and the datapoint.  Each CSV had to be imported and cleaned, then they were merged using the job ID in order to conduct further analysis using a single dataframe.

See Appendix A below to for the code that performed the clean and merge.

## Importing necessary libraries

```python
import pandas as pd
import random
from sklearn.metrics import accuracy_score
from sklearn.cross_validation import cross_val_score
```

# Establish baselines to compare model against

To determine how well our model is performing I will need to compare how it does vs a random guess whether a job listing pays over 90K.

I also want to see how accurate taking the assumption that every job is listed over 90K, or that every job is listed under 90K is:

```python
random_guess = []
all_over_90K = []
all_under_90K = []
#y is the prediction target set
for i in y:
    random_guess.append(random.randint(0,1))
    all_over_90K.append(1)
    all_under_90K.append(0)
print("Random Guess: {0:.2%}".format(accuracy_score(y, random_guess)))
print("All Over90K: {0:.2%}".format(accuracy_score(y, all_over_90K)))
print("All Under90K: {0:.2%}".format(accuracy_score(y, all_under_90K)))
#{percent:.2%}'.format(percent=1.0/3.0)
```
### Baseline Accuracies:
- Random Guess: 51.88%
- All Over90K: 53.90%
- All Under90K: 46.10%

## Making a Logistic RegressionCV Model

Now that the data is ready and the baseline performance is established it's time to build a model and see if it accurately predicts whether a job pays over 90K.

Here I chose sci-kit learn's Logistic RegressionCV and train and evaluate based on the four features the team originally brainstormed for predictors:
    - Years of experience
    - Whether the term 'scientist' is in the job title
    - Whether the term 'start-up' appears in the job description
    - Whether the term 'Phd' appears in the job description

```python
from sklearn.linear_model import LogisticRegressionCV
X = df[[
        'years',
        'scientist_in_title',
        'has_startup',
        'has_phd'
    ]]
y = df.over_90k

model = LogisticRegressionCV(cv=3)
model.fit(X, y) # This trains the model using 3 cross validated sets from the data

for metric in ['accuracy', 'precision', 'recall', 'roc_auc']:
    scores = cross_val_score(model, X, y, scoring=metric)
    print("mean {}: {}, all: {}".format(metric, scores.mean(), scores))
```

    mean accuracy: 0.5671291451994911, all: [ 0.59437751  0.56854839  0.53846154]
    mean precision: 0.6269932389335374, all: [ 0.64102564  0.70149254  0.53846154]
    mean recall: 0.6368159203980099, all: [ 0.55970149  0.35074627  1.        ]
    mean roc_auc: 0.6059093151666236, all: [ 0.62255029  0.53698612  0.65819153]

### Model Accuracy: 56.7%

The model's results are better than random guessing, but fall quite short of inspiring confidence.

While this model is doing better than a coin flip it edges out the best baseline - assuming all jobs listings pay over 90K - by less than 3%.

## Thoughts and Conclusions

At the project outset, features the team thought would help determine whether a position had a salary over 90K turned out to be poor predictors when put into a model.  Significant accuracy improvements would likely be made by gathering more data points from the job descriptions, for example if 'SQL' is in the description, or if the title contains the term 'Senior' or 'Sr' or 'intern' etc.

As an overarching caveat, the data used was incomplete and inaccurate at best.  The web scraping effort put forth for data collection did not lend itself to a clean dataset provided us with less than 1000 job listings to use as data.

It's also worth mentioning that any job that did not provide a salary range on its listing was not used in this prediction which biases the dataset. For example, it's possible that listings that disclose a salary range are inherently higher paying.

#### Appendix A: Importing and Merging data

```python
#   This code imports each csv and creates a dataframe with the job ID as the index
#   which is then user later to merge the datapoints across jobs
python = pd.read_csv("./id_and_Python.csv",index_col='id')
del python['Unnamed: 0']
python = python.dropna()
python = python.reset_index().drop_duplicates(subset='id', keep='last').set_index('id')
years = pd.read_csv("./id_years - removed_dupes.csv", index_col='id')
del years['Unnamed: 0']
years = years.reset_index().drop_duplicates(subset='id', keep='last').set_index('id')
salary = pd.read_csv("./id_salary.csv", index_col='id')
del salary['Unnamed: 0']
phd = pd.read_csv("./phd_df.csv",index_col='job_id',encoding='ISO-8859-1')
del phd['Unnamed: 0']
phd = phd.reset_index().drop_duplicates(subset='job_id', keep='last').set_index('job_id')
startup = pd.read_csv("./startup_df.csv",index_col='id')
del startup['Unnamed: 0']
startup = startup.reset_index().drop_duplicates(subset='id', keep='last').set_index('id')
```

```python
#   This code merges each dataframe
df = pd.merge(python,years,how='outer', left_index=True, right_index=True, indicator='py_year')
df = pd.merge(df,scientist,how='outer', left_index=True, right_index=True, indicator='scientist_merge')
df.rename(columns={'Classifier':'scientist_in_title'},inplace=True)
df = pd.merge(df,salary,how='left',left_index=True,right_index=True,indicator='salary_merge')
df = pd.merge(df,startup,how='left',left_index=True,right_index=True,indicator='startup_merge')
df = pd.merge(df,phd,how='left',left_index=True,right_index=True,indicator='phd_merge')
```

```python
#   Fills any 'NaN' values with 0s
df['Python'] = df['Python'].fillna(0)
df['years'] = df['years'].fillna(0)
df['has_phd'] = df['has_phd'].fillna(0)
df['has_startup'] = df['has_startup'].fillna(0)
df['scientist_in_title'] = df['scientist_in_title'].fillna(0)
```

```python
df.head()
```

      Python
      years
      py_year
      Title
      scientist_in_title
      scientist_merge
      over_90k
      salary_merge
      has_startup
      startup_merge
      has_phd
      title
      phd_merge

      jl_f43cd8061406b3d7
      1.0
      0.0
      left_only
      NaN
      0.0
      left_only
      1.0
      both
      0.0
      left_only
      0.0
      NaN
      left_only

      jl_b59a64d3298e7fe8
      1.0
      0.0
      left_only
      NaN
      0.0
      left_only
      1.0
      both
      0.0
      left_only
      0.0
      NaN
      left_only

      jl_fbadb59b5f73dc18
      1.0
      0.0
      both
      NaN
      0.0
      left_only
      1.0
      both
      0.0
      left_only
      0.0
      NaN
      left_only

      jl_90603c7f1f0af480
      1.0
      0.0
      both
      NaN
      0.0
      left_only
      1.0
      both
      0.0
      left_only
      0.0
      NaN
      left_only

      jl_70351b5092814475
      1.0
      0.0
      left_only
      biomedical data scientist
      1.0
      both
      1.0
      both
      0.0
      left_only
      1.0
      Biomedical Data Scientist
      both

```python
def examine_coefficients(model, df):
    df = pd.DataFrame(
        { 'Coefficient' : model.coef_[0] , 'Feature' : df.columns}
    ).sort_values(by='Coefficient')
    return df[df.Coefficient !=0 ]

examine_coefficients(model, X)
```

      Coefficient
      Feature

      1
      -0.458136
      scientist_in_title

      0
      0.154580
      years

      3
      0.708252
      has_phd

      2
      1.580908
      has_startup


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">

<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
<TITLE>
StaticProcessMonitor (comm 0.2.0.0 API)
</TITLE>

<META NAME="date" CONTENT="2010-11-05">

<LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">

      Overview
      Package
    Class
      Use
      Tree
      Deprecated
      Index
      Help

 PREV CLASS
 NEXT CLASS

  FRAMES
 NO FRAMES

  All Classes

  SUMMARY: NESTED | FIELD | CONSTR | METHOD

DETAIL: FIELD | CONSTR | METHOD

io.s4.comm.file

Class StaticProcessMonitor

java.lang.Object
  io.s4.comm.file.StaticProcessMonitor

All Implemented Interfaces:
ProcessMonitor

public class StaticProcessMonitor
extends java.lang.Object
implements ProcessMonitor

Constructor Summary

StaticProcessMonitor(java.lang.String address,
                     java.lang.String root)

Method Summary

 java.util.List<java.lang.Object>
getDestinationList()

 java.util.Map<java.lang.Integer,java.lang.Object>
getDestinationMap()

 int
getTaskCount()

 void
monitor()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

StaticProcessMonitor

public StaticProcessMonitor(java.lang.String address,
                            java.lang.String root)

Method Detail

monitor

public void monitor()

Specified by:
monitor in interface ProcessMonitor

getDestinationList

public java.util.List<java.lang.Object> getDestinationList()

Specified by:
getDestinationList in interface ProcessMonitor

getDestinationMap

public java.util.Map<java.lang.Integer,java.lang.Object> getDestinationMap()

Specified by:
getDestinationMap in interface ProcessMonitor

getTaskCount

public int getTaskCount()

Specified by:
getTaskCount in interface ProcessMonitor

      Overview
      Package
    Class
      Use
      Tree
      Deprecated
      Index
      Help

 PREV CLASS
 NEXT CLASS

  FRAMES
 NO FRAMES

  All Classes

  SUMMARY: NESTED | FIELD | CONSTR | METHOD

DETAIL: FIELD | CONSTR | METHOD

Copyright © 2010. All Rights Reserved.