kylin-cube: 使用REST api来build, 自动构建cube
文章目录
- 1，执行cube的相关任务
- a, build/rebuild cube
- b, 获取job的状态
- 2, 元数据备份、恢复
- 3, 自动构建cube
| api | 返回值 | 描述 |
|---|---|---|
| http://ip:7070/kylin/api/models | json数组 | 查看所有model元数据 |
| http://ip:7070/kylin/api/cubes | json数组 | 查看所有cube元数据 |
| http://ip:7070/kylin/api/jobs/xxx | json数组 | 查看单个cube提交后的任务 |
获取用户名密码： 被base64加密后的格式：QURNSU46S1lMSU4=
UserName=ADMINPassword=KYLINbaseUsernamePasswd=`python -c "import base64; print base64.standard_b64encode('$UserName:$Password')"`
1，执行cube的相关任务
a, build/rebuild cube
curl -X PUT \-H "Authorization: Basic $baseUsernamePasswd" \-H "Content-Type: application/json" \-d '{ "startTime": 0, "endTime": 1388563200000, "buildType": "BUILD" }' \http://localhost:7070/kylin/api/cubes/ys_demo/build#http://localhost:7070/kylin/api/cubes/ys_demo/rebuild##数据返回{ "uuid":"2eb3c342-daa2-4fab-abb8-c622beb46d9b", ...}
b, 获取job的状态
curl -X GET \-H "Authorization: Basic QURNSU46S1lMSU4=" \-H "Content-Type: application/json" \http://localhost:7070/kylin/api/jobs/cbfaf44f-c7bb-4b5a-9da4-671f6c84b1bb##返回的数据{ ...,"submitter":"ADMIN","job_status":"RUNNING","progress":53.84615384615385...}
2, 元数据备份、恢复
[hdfs@node1 kylin]$ metastore.shusage: metastore.sh backupmetastore.sh fetch DATAmetastore.sh resetmetastore.sh refresh-cube-signaturemetastore.sh restore PATH_TO_LOCAL_METAmetastore.sh list RESOURCE_PATHmetastore.sh cat RESOURCE_PATHmetastore.sh remove RESOURCE_PATHmetastore.sh clean [--delete true][hdfs@node1 kylin]$ metastore.sh list /Starting list /Retrieving hadoop conf dir......[/UUID, /acl, /cube, /cube_desc, /cube_statistics, /dict, /execute, /execute_output, /model_desc, /project, /query, /table, /table_exd, /user][hdfs@node1 kylin]$ metastore.sh backupStarting backup to /opt/kylin/meta_backups/meta_2019_04_19_12_11_46.....[hdfs@node1 kylin]$ ls meta_backups/meta_2019_04_19_10_29_28 meta_2019_04_19_12_11_46[hdfs@node1 kylin]$ ls meta_backups/meta_2019_04_19_10_29_28/acl cube_desc dict execute_output project table usercube cube_statistics execute model_desc query table_exd UUID[hdfs@node1 kylin]$ ls meta_backups/meta_2019_04_19_10_29_28/project/CDR_ADMISSION_INFO.json kylin_demo.json Ys_kylin_Demo.json[hdfs@node1 kylin]$ ls meta_backups/meta_2019_04_19_10_29_28/cubeaa2423.json aa2.json kylin.json kylinspark.json ys_demo.json
3, 自动构建cube
文档：http://kylin.apache.org/cn/docs/howto/howto_build_cube_with_restapi.html
kylin_cube_cron.sh
!/bin/bash
set -x
error_file=/tmp/kylin_error.log
cubes=(
cube1
cube2
)status_check(){
jobid=$1;error_count=0; try_count=0while truedosleep 20let try_count++# look error jobs, resume it!response=$(curl -s -X GET -H 'Authorization: Basic QURNSU46S1lMSU4=' \ "http://192.168.56.6:7070/kylin/api/jobs/$jobid" )echo $response |grep -oE job_status.* |grep "FINISHED"[ $? -eq 0 ] && breakecho $response |grep -oE job_status.* |grep "ERROR"# try 60 times: 1200 second =20minif [ $? -eq 0 ] && [ $error_count -le 60 ] ;thenlet error_count++curl -X PUT -H 'Authorization: Basic QURNSU46S1lMSU4=' -H 'Content-Type: application/json' \-d '{"buildType":"BUILD"}' \"http://192.168.56.6:7070/kylin/api/jobs/$jobid/resume"# wait 6 times: 120 second =2 min to finishelif [ $try_count -ge 6 ] ; thenbreakelsecontinuefidone
}
curl_build(){
cubeid=$1#1,build cmdresponse=`curl -X PUT -H "Authorization: Basic QURNSU46S1lMSU4=" -H 'Content-Type: application/json' -d '{"buildType":"BUILD"}' \ http://192.168.56.6:7070/kylin/api/cubes/${ cubeid}/rebuild`#2,resume cmdjobid=`echo $response |awk -F',' '{print $1}' |grep 'uuid' |awk -F':' '{print $2}' |xargs`#check if job is errorif [ x$jobid != "x" ] ; thenstatus_check $jobidelseecho "$(date '%F %T'), error== $cubeid ==> $response" >> $error_filefi
}
for cube in ${cubes[*]}docurl_build $cubedone
女爷i
怼烎@
╰+攻爆jí腚メ
左手的ㄟ右手
还没有评论，来说两句吧...