Hive探索之路

  • bde的docker镜像:hadoop2.7+hive2.3+nifi1.6

    • 相关配置

    • HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver
      HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive
      HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive
      HIVE_SITE_CONF_datanucleus_autoCreateSchema=false
      HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083
      # HIVE_SITE_CONF_hive_metastore_uris=thrift://192.168.16.15:9083
      # HIVE_SITE_CONF_hive_enforce_bucketing=true
      HIVE_SITE_CONF_hive_exec_dynamic_partition_mode=nonstrict
      HIVE_SITE_CONF_hive_txn_manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager
      HIVE_SITE_CONF_hive_compactor_initiator_on=true
      HIVE_SITE_CONF_hive_compactor_worker_threads=5
      HIVE_SITE_CONF_hive_support_concurrency=true
      # HIVE_SITE_CONF_hive_txn_strict_locking_mode=false
      HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false

      # CORE_CONF_fs_defaultFS=hdfs://namenode:8020
      CORE_CONF_fs_defaultFS=hdfs://192.168.16.15:8020
      CORE_CONF_dfs_replication=1
      CORE_CONF_hadoop_http_staticuser_user=root
      CORE_CONF_hadoop_proxyuser_hue_hosts=*
      CORE_CONF_hadoop_proxyuser_hue_groups=*

      HDFS_CONF_dfs_webhdfs_enabled=true
      HDFS_CONF_dfs_permissions_enabled=false

      MAPRED_CONF_mapreduce_framework_name=yarn
      MAPRED_CONF_mapreduce_application_classpath=$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*
      MAPRED_CONF_mapred_job_tracker=192.168.16.15:8021

      YARN_CONF_yarn_nodemanager_aux___services=mapreduce_shuffle
      YARN_CONF_yarn_log___aggregation___enable=true
      YARN_CONF_yarn_resourcemanager_recovery_enabled=true
      YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
      YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate
      YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
      YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/
      YARN_CONF_yarn_timeline___service_enabled=true
      YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true
      YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true
      YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
      YARN_CONF_yarn_timeline___service_hostname=historyserver
      YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032
      YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030
      YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031