application_vnd.jupyter.stderr_output_2.png text_plain_output_1.png
from google.cloud import bigquery

import os

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import time

import numpy as np
import pandas as pd
import os

PROJECT_ID = 'YOUR_OWN_PROJECT_ID'
from google.cloud import bigquery
client = bigquery.Client(project=PROJECT_ID, location='US')

path = '../input/'
dataset_id = 'titanic'
dataset_ref = client.dataset(dataset_id)
job_config = bigquery.LoadJobConfig()
job_config.source_format = bigquery.SourceFormat.CSV
job_config.skip_leading_rows = 1
job_config.autodetect = True

existing_datasets = [dataset.dataset_id for dataset in client.list_datasets()]
if not dataset_id in existing_datasets:
    client.create_dataset(dataset_id)

dataset_ref = client.dataset(dataset_id)

existing_tables = [table.table_id for table in client.list_tables(dataset_id)]
for filename in os.listdir(path):
    table_id = filename.rstrip('.csv')
    if not table_id in existing_tables:
        table_ref = dataset_ref.table(table_id)
        with open(path + filename, 'rb') as source_file:
            job = client.load_table_from_file(source_file, table_ref, job_config=job_config)
        job.result()

table_ref = dataset_ref.table('train')
train = client.get_table(table_ref)

train.schema

client.list_rows(train, max_results=5).to_dataframe()

train_query = "\nCREATE OR REPLACE MODEL `titanic.logistic_reg_model`\nOPTIONS (\n  model_type = 'logistic_reg'\n  ) AS (\nSELECT\n  Pclass, Name, Sex, Age, SibSp, Parch, Ticket, Fare, Cabin, Embarked,  Survived AS label\nFROM\n  `titanic.train`)\n        "

train_query_job = client.query(train_query)

while 1:
    if client.get_job(train_query_job.job_id).state == 'DONE':
        break
    time.sleep(5)

predict_query = '\nSELECT\n * \nFROM\n  ML.PREDICT(MODEL `titanic.logistic_reg_model`, (\nSELECT\n  Pclass, Name, Sex, Age, SibSp, Parch, Ticket, Fare, Cabin, Embarked, PassengerId\n\nFROM\n  `titanic.test`\n))\n      '

predict_query_job = client.query(predict_query)

predicted = predict_query_job.to_dataframe()

sub = pd.read_csv('../input/test.csv', header=0)

sub = pd.merge(sub, predicted, on='PassengerId', how='right')

sub = sub[['PassengerId', 'predicted_label']].rename(index=str, columns={'predicted_label': 'Survived'})

sub.head()