Hadoop常用配置

hadoop 配置

  • HDFS 远程访问 dfs.namenode.rpc-address=0.0.0.0:8020

HIVE 空间数据库支持

参考:https://community.hortonworks.com/articles/44319/geo-spatial-queries-with-hive-using-esri-geometry.html
https://blog.csdn.net/qianshangding0708/article/details/50381966

  1. 获取空间数据自定义插件:
    共三个jar包,esri-geometry-api-2.1.0.jar spatial-sdk-hive-2.0.0.jar spatial-sdk-json-2.0.0.jar
    • https://github.com/Esri/spatial-framework-for-hadoop
    • https://github.com/Esri/geometry-api-java/wiki
  2. Jar放入${HIVE_HOME}/auxlib目录,所有节点
    在${HIVE_HOME}中创建文件夹auxlib,然后将自定义jar文件放入该文件夹中。 scp -r /usr/hdp/current/hive-server2/auxlib cluster01.node02:/usr/hdp/current/hive-server2/ 此方法添加shell链接不需要重启Hive。而且比较便捷。Hive Server需要重启生效。
  3. 或直接将Jar加入${HIVE_HOME}/lib目录
    此方法 shell链接 和 Hive Server需要重启生效。
  4. 重启hive服务
  5. shell连接hive 以系统hive用户登录,执行命令创建自定义空间函数 UDF
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73
    74
    75
    76
    77
    78
    79
    80
    81
    82
    83
    84
    85
    86
     create function ST_Aggr_ConvexHull as 'com.esri.hadoop.hive.ST_Aggr_ConvexHull';
     create function ST_Aggr_Intersection as 'com.esri.hadoop.hive.ST_Aggr_Intersection';
     create function ST_Aggr_Union as 'com.esri.hadoop.hive.ST_Aggr_Union';
     create function ST_Area as 'com.esri.hadoop.hive.ST_Area';
     create function ST_AsBinary as 'com.esri.hadoop.hive.ST_AsBinary';
     create function ST_AsGeoJson as 'com.esri.hadoop.hive.ST_AsGeoJson';
     create function ST_AsJson as 'com.esri.hadoop.hive.ST_AsJson';
     create function ST_AsShape as 'com.esri.hadoop.hive.ST_AsShape';
     create function ST_AsText as 'com.esri.hadoop.hive.ST_AsText';
     create function ST_Bin as 'com.esri.hadoop.hive.ST_Bin';
     create function ST_BinEnvelope as 'com.esri.hadoop.hive.ST_BinEnvelope';
     create function ST_Boundary as 'com.esri.hadoop.hive.ST_Boundary';
     create function ST_Buffer as 'com.esri.hadoop.hive.ST_Buffer';
     create function ST_Centroid as 'com.esri.hadoop.hive.ST_Centroid';
     create function ST_Contains as 'com.esri.hadoop.hive.ST_Contains';
     create function ST_ConvexHull as 'com.esri.hadoop.hive.ST_ConvexHull';
     create function ST_CoordDim as 'com.esri.hadoop.hive.ST_CoordDim';
     create function ST_Crosses as 'com.esri.hadoop.hive.ST_Crosses';
     create function ST_Difference as 'com.esri.hadoop.hive.ST_Difference';
     create function ST_Dimension as 'com.esri.hadoop.hive.ST_Dimension';
     create function ST_Disjoint as 'com.esri.hadoop.hive.ST_Disjoint';
     create function ST_Distance as 'com.esri.hadoop.hive.ST_Distance';
     create function ST_EndPoint as 'com.esri.hadoop.hive.ST_EndPoint';
     create function ST_Envelope as 'com.esri.hadoop.hive.ST_Envelope';
     create function ST_EnvIntersects as 'com.esri.hadoop.hive.ST_EnvIntersects';
     create function ST_Equals as 'com.esri.hadoop.hive.ST_Equals';
     create function ST_ExteriorRing as 'com.esri.hadoop.hive.ST_ExteriorRing';
     create function ST_GeodesicLengthWGS84 as 'com.esri.hadoop.hive.ST_GeodesicLengthWGS84';
     create function ST_GeomCollection as 'com.esri.hadoop.hive.ST_GeomCollection';
     create function ST_Geometry as 'com.esri.hadoop.hive.ST_Geometry';
     create function ST_GeometryAccessor as 'com.esri.hadoop.hive.ST_GeometryAccessor';
     create function ST_GeometryN as 'com.esri.hadoop.hive.ST_GeometryN';
     create function ST_GeometryProcessing as 'com.esri.hadoop.hive.ST_GeometryProcessing';
     create function ST_GeometryRelational as 'com.esri.hadoop.hive.ST_GeometryRelational';
     create function ST_GeometryType as 'com.esri.hadoop.hive.ST_GeometryType';
     create function ST_GeomFromGeoJson as 'com.esri.hadoop.hive.ST_GeomFromGeoJson';
     create function ST_GeomFromJson as 'com.esri.hadoop.hive.ST_GeomFromJson';
     create function ST_GeomFromShape as 'com.esri.hadoop.hive.ST_GeomFromShape';
     create function ST_GeomFromText as 'com.esri.hadoop.hive.ST_GeomFromText';
     create function ST_GeomFromWKB as 'com.esri.hadoop.hive.ST_GeomFromWKB';
     create function ST_InteriorRingN as 'com.esri.hadoop.hive.ST_InteriorRingN';
     create function ST_Intersection as 'com.esri.hadoop.hive.ST_Intersection';
     create function ST_Intersects as 'com.esri.hadoop.hive.ST_Intersects';
     create function ST_Is3D as 'com.esri.hadoop.hive.ST_Is3D';
     create function ST_IsClosed as 'com.esri.hadoop.hive.ST_IsClosed';
     create function ST_IsEmpty as 'com.esri.hadoop.hive.ST_IsEmpty';
     create function ST_IsMeasured as 'com.esri.hadoop.hive.ST_IsMeasured';
     create function ST_IsRing as 'com.esri.hadoop.hive.ST_IsRing';
     create function ST_IsSimple as 'com.esri.hadoop.hive.ST_IsSimple';
     create function ST_Length as 'com.esri.hadoop.hive.ST_Length';
     create function ST_LineFromWKB as 'com.esri.hadoop.hive.ST_LineFromWKB';
     create function ST_LineString as 'com.esri.hadoop.hive.ST_LineString';
     create function ST_M as 'com.esri.hadoop.hive.ST_M';
     create function ST_MaxM as 'com.esri.hadoop.hive.ST_MaxM';
     create function ST_MaxX as 'com.esri.hadoop.hive.ST_MaxX';
     create function ST_MaxY as 'com.esri.hadoop.hive.ST_MaxY';
     create function ST_MaxZ as 'com.esri.hadoop.hive.ST_MaxZ';
     create function ST_MinM as 'com.esri.hadoop.hive.ST_MinM';
     create function ST_MinX as 'com.esri.hadoop.hive.ST_MinX';
     create function ST_MinY as 'com.esri.hadoop.hive.ST_MinY';
     create function ST_MinZ as 'com.esri.hadoop.hive.ST_MinZ';
     create function ST_MLineFromWKB as 'com.esri.hadoop.hive.ST_MLineFromWKB';
     create function ST_MPointFromWKB as 'com.esri.hadoop.hive.ST_MPointFromWKB';
     create function ST_MPolyFromWKB as 'com.esri.hadoop.hive.ST_MPolyFromWKB';
     create function ST_MultiLineString as 'com.esri.hadoop.hive.ST_MultiLineString';
     create function ST_MultiPoint as 'com.esri.hadoop.hive.ST_MultiPoint';
     create function ST_MultiPolygon as 'com.esri.hadoop.hive.ST_MultiPolygon';
     create function ST_NumGeometries as 'com.esri.hadoop.hive.ST_NumGeometries';
     create function ST_NumInteriorRing as 'com.esri.hadoop.hive.ST_NumInteriorRing';
     create function ST_NumPoints as 'com.esri.hadoop.hive.ST_NumPoints';
     create function ST_Overlaps as 'com.esri.hadoop.hive.ST_Overlaps';
     create function ST_Point as 'com.esri.hadoop.hive.ST_Point';
     create function ST_PointFromWKB as 'com.esri.hadoop.hive.ST_PointFromWKB';
     create function ST_PointN as 'com.esri.hadoop.hive.ST_PointN';
     create function ST_PointZ as 'com.esri.hadoop.hive.ST_PointZ';
     create function ST_PolyFromWKB as 'com.esri.hadoop.hive.ST_PolyFromWKB';
     create function ST_Polygon as 'com.esri.hadoop.hive.ST_Polygon';
     create function ST_Relate as 'com.esri.hadoop.hive.ST_Relate';
     create function ST_SetSRID as 'com.esri.hadoop.hive.ST_SetSRID';
     create function ST_SRID as 'com.esri.hadoop.hive.ST_SRID';
     create function ST_StartPoint as 'com.esri.hadoop.hive.ST_StartPoint';
     create function ST_SymmetricDiff as 'com.esri.hadoop.hive.ST_SymmetricDiff';
     create function ST_Touches as 'com.esri.hadoop.hive.ST_Touches';
     create function ST_Union as 'com.esri.hadoop.hive.ST_Union';
     create function ST_Within as 'com.esri.hadoop.hive.ST_Within';
     show functions;