python调用hive脚本_python 中写hive 脚本

1、直接执行.sql脚本

import numpy as np

import pandas as pd

import lightgbm as lgb

from pandas import DataFrame

from sklearn.model_selection import train_test_split

from io import StringIO

import gc

import sys

import os

hive_cmd = "hive -f ./sql/sql.sql"

output = os.popen(hive_cmd)

data_cart_prop = pd.read_csv(StringIO(unicode(output.read(),'utf-8')), sep="\t",header=0)

2、Hive语句执行

假如有如下hive sql：

hive_cmd = 'hive -e "select count(*) from hbase.routermac_sort_10;"'

一般在python中按照如下方式执行该hive sql：

os.system(hive_cmd)

---------------------

hive_cmd1 = "hive -f ./user.sql"

output1 = os.popen(hive_cmd1)

test_user = pd.read_csv(StringIO(unicode(output1.read(),'utf-8')), sep="\t",header=0)

hive_cmd2 = "hive -f ./action.sql"

output2 = os.popen(hive_cmd2)

test_action = pd.read_csv(StringIO(unicode(output2.read(),'utf-8')), sep="\t",header=0)

hive_cmd3 = "hive -f ./click.sql"

output3 = os.popen(hive_cmd3)

test_click = pd.read_csv(StringIO(unicode(output3.read(),'utf-8')), sep="\t",header=0)

为了显示表头，在脚本中加上一句：set hive.cli.print.header=true;

或者，使用如下语句：

hive_cmd = 'hive -e "set hive.cli.print.header=true;SELECT * FROM dev.temp_dev_jypt_decor_user_label_phase_one_view_feature WHERE(dt = "2018-09-17");"'

output = os.popen(hive_cmd)

data_cart_prop = pd.read_csv(StringIO(unicode(output.read(),'utf-8')), sep="\t",header=0)

3、tf 显存占用

import tensorflow as tf

tf.enable_eager_execution()

x = tf.get_variable('x', shape=[1], initializer=tf.constant_initializer(3.))

with tf.GradientTape() as tape:

y = tf.square(x)

y_grad = tape.gradient(y, x)

print([y.numpy(), y_grad.numpy()])