Hadoop2.7实战v1.0之Flume1.6.0搭建(Http Source-->Memory Chanel --> Hdfs Sink)

  • hackeruncle

    2016-05-22 23:29:16
  • Hadoop
  • 原创

Hadoop2.7实战v1.0之Flume1.6.0搭建(Http Source-->Memory Chanel --> Hdfs Sink)

1.查看系统是否已经配置jdk1.7.0

点击(此处)折叠或打开

  1. [root@xxx-01 jdk1.7.0_25]# bin/java -version
  2. java version "1.7.0_25"
  3. Java(TM) SE Runtime Environment (build 1.7.0_25-b15)
  4. Java HotSpot(TM) 64-Bit Server VM (build 23.25-b01, mixed mode)
  5. [root@xxx-01 jdk1.7.0_25]# pwd
  6. /usr/java/jdk1.7.0_25
  7. [root@xxx-01 jdk1.7.0_25]


##假如没有配置jdk1.7以上的版本,请参考 http://blog.itpub.net/30089851/viewspace-1994585/ 的 "安装JDK"

2.下载和解压flume1.6.0

点击(此处)折叠或打开

  1. [root@xxx-01 local]# wget http://ftp.cuhk.edu.hk/pub/packages/apache.org/flume/1.6.0/apache-flume-1.6.0-bin.tar.gz
  2. [root@xxx-01 local]# tar zxvf apache-flume-1.6.0-bin.tar.gz
  3. [root@xxx-01 local]# cd apache-flume-1.6.0-bin
  4. [root@xxx-01 apache-flume-1.6.0-bin]# ll
  5. total 140
  6. drwxr-xr-x 2 template games 4096 May 21 17:32 bin
  7. -rw-r--r-- 1 template games 69856 May 9 2015 CHANGELOG
  8. drwxr-xr-x 2 template games 4096 May 21 17:32 conf
  9. -rw-r--r-- 1 template games 6172 May 9 2015 DEVNOTES
  10. drwxr-xr-x 10 template games 4096 May 12 2015 docs
  11. drwxr-xr-x 2 root root 4096 May 21 17:32 lib
  12. -rw-r--r-- 1 template games 25903 May 9 2015 LICENSE
  13. -rw-r--r-- 1 template games 249 May 9 2015 NOTICE
  14. -rw-r--r-- 1 template games 1779 May 9 2015 README
  15. -rw-r--r-- 1 template games 1585 May 9 2015 RELEASE-NOTES
  16. drwxr-xr-x 2 root root 4096 May 21 17:32 tools
  17. [root@xxx-01 apache-flume-1.6.0-bin]#
  18. [root@xxx-01 apache-flume-1.6.0-bin]# cd conf
  19. [root@xxx-01 conf]# ls -l
  20. total 16
  21. -rw-r--r-- 1 template games 1661 May 9 2015 flume-conf.properties.template
  22. -rw-r--r-- 1 template games 1110 May 9 2015 flume-env.ps1.template
  23. -rw-r--r-- 1 template games 1214 May 9 2015 flume-env.sh.template
  24. -rw-r--r-- 1 template games 3107 May 9 2015 log4j.properties
  25. [root@xxx-01 conf]#
  26. [root@xxx-01 conf]# cp flume-env.sh.template flume-env.sh
  27. [root@xxx-01 conf]# cp flume-conf.properties.template flume-conf.properties
  28. [root@xxx-01 conf]#

 3.配置flume-env.sh 和环境变量

点击(此处)折叠或打开

  1. [root@xxx-01 conf]# vi flume-env.sh
  2. export JAVA_HOME=/usr/java/jdk1.7.0_25
  3. export HADOOP_HOME=/opt/cloudera/parcels/CDH/lib/hadoop
  4. [root@xxx-01 ~]# vi /etc/profile
  5. export JAVA_HOME="/usr/java/jdk1.7.0_25"
  6. export FLUME_HOME=/usr/local/apache-flume-1.6.0-bin
  7. export FLUME_CONF_DIR=$FLUME_HOME/conf

  9. export PATH=$FLUME_HOME/bin:$JAVA_HOME/bin:$PATH

  11. [root@xxx-01 ~]# source /etc/profile
  12. [root@xxx-01 ~]# echo $FLUME_HOME
  13. /usr/local/apache-flume-1.6.0-bin

4.配置flume-conf.properties

点击(此处)折叠或打开

  1. [root@xxx-01 conf]# vi flume-conf.properties
  2. # Name the components on this agent
  3. a1.sources = r1
  4. a1.sinks = k1
  5. a1.channels = c1

  7. # Describe/configure the source
    ###默认http handle的格式是json
  8. a1.sources.r1.type = http
  9. a1.sources.r1.bind = 本机ip
  10. a1.sources.r1.port = 5140

  12. a1.sources.r1.fileHeader = false
  13. #a1.sources.r1.deserializer.outputCharset=UTF-8

  15. a1.sources.r1.interceptors =i1
  16. a1.sources.r1.interceptors.i1.type = timestamp

  18. # Describe the sink
  19. a1.sinks.k1.type = hdfs
  20. a1.sinks.k1.channel = c1
  21. # 可以指定hdfs ha的fs.defaultFS配置信息,而不是指定其中一台master的,关键是当前flume机器要有hadoop环境(因为要加载hadoop jar包)
  22. #和在flume机器上这三个hadoop-env.sh hdfs-site.xml core-site.xml文件要与 日志存储的hdfs配置一致.
  23. a1.sinks.k1.hdfs.path = hdfs://nameservice1/testwjp/%Y-%m-%d/%H
  24. #a1.sinks.k1.hdfs.path = hdfs://xxx-01:8022/testwjp/%Y-%m-%d/%H
  25. a1.sinks.k1.hdfs.filePrefix = logs
  26. a1.sinks.k1.hdfs.inUsePrefix = .

  28. a1.sinks.k1.hdfs.rollInterval = 0
  29. ### roll 16 m = 16777216 bytes
  30. a1.sinks.k1.hdfs.rollSize = 16777216
  31. a1.sinks.k1.hdfs.rollCount = 0
  32. a1.sinks.k1.hdfs.batchSize = 1000
  33. a1.sinks.k1.hdfs.writeFormat = text

  35. ###A. plain text format 普通文本格式
  36. a1.sinks.k1.hdfs.fileType = DataStream

  38. ###B. compressed plain text to zip format 压缩格式
  39. #a1.sinks.k1.hdfs.fileType = CompressedStream
  40. #a1.sinks.k1.hdfs.codeC = bzip2

  42. # Use a channel which buffers events in memory
  43. a1.channels.c1.type = memory
  44. a1.channels.c1.keep-alive = 30
  45. a1.channels.c1.capacity = 100000
  46. a1.channels.c1.transactionCapacity = 1000
  48. # Bind the source and sink to the channel
  49. a1.sources.r1.channels = c1
  50. a1.sinks.k1.channel = c1

5.启动agent
[root@xxx-01 apache-flume-1.6.0-bin]#bin/flume-ng agent -c conf -f conf/http-memory-hdfs.properties -n a1 -Dflume.root.logger=INFO,console


###后台启动
nohup bin/flume-ng agent -c conf -f conf/http-memory-hdfs.properties -n a1 -Dflume.root.logger=INFO,console &


6.测试端A
[root@xxx-01 bin]# curl -X POST -d'[{"headers":{"h1":"v1","h2":"v2"},"body":"hello body"}]'  http://10.168.11.13:5140


7.测试端B Fox浏览器+HttpRequester工具
URL:http://本机ip:5140
Type:POST
Content Type: application/json
Content:

点击(此处)折叠或打开

  1. [{
  2.    "headers" : {
  3.        "timestamp" : "1",
  4.        "host" : "random_host1.example.com"
  5.        },
  6.    "body" : "random_body1"
  7.    },
  8.    {
  9.    "headers" : {
  10.         "timestamp" : "2",
  11.        "host" : "random_host2.example.com"
  12.        },
  13.    "body" : "random_body2"
  14.   }]

 ###单击"Submit"按钮,返回状态 200,标识内容post成功.
  
 
8.验证数据是否sink到hdfs上
命令行:hadoop fs -ls hdfs://nameservice1/testwjp/
Web:http://x.x.x.x:50070/

官方文档: http://flume.apache.org/FlumeUserGuide.html

请使用浏览器的分享功能分享到微信等