storm简单实例+将storm任务迁移至flink处理研究
基础storm程序示例
Storm的流处理主要就是通过Spout和Bolt节点进行处理，可以继承这些类写自己的逻辑
public class FlinkStormDemo {
public static void main(String[] args) {
//1.创建执行环境
LocalCluster stormCluster = new LocalCluster();
TopologyBuilder builder = new TopologyBuilder();
//2.创建一个初始的数据源
builder.setSpout("word", new WordSpout());
//3.对数据源进行第一次加工
builder.setBolt("word-1",new WordBolt1(), 1).shuffleGrouping("word");
//4.对数据源进行第二次加工
builder.setBolt("word-2",new WordBolt2(), 1).shuffleGrouping("word-1");
//5.配置一些参数
Config config = new Config();
config.setDebug(true);
//6.提交storm任务并处理
stormCluster.submitTopology("storm-task", config, builder.createTopology());
}
static class WordSpout extends BaseRichSpout {
private SpoutOutputCollector spoutOutputCollector;
@Override
public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
this.spoutOutputCollector = spoutOutputCollector;
}
@Override
public void nextTuple() {
try {
Thread.sleep(10000);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("数据初始化中....");
String initData = "abc";
spoutOutputCollector.emit(new Values(initData));
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("word"));
}
}
static class WordBolt1 extends BaseRichBolt {
private OutputCollector collector;
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector = outputCollector;
}
@Override
public void execute(Tuple tuple) {
System.out.println("数据第1次处理中....");
//给上次获取的单词拼接上def
collector.emit(tuple, new Values(tuple.getString(0) + "def"));
collector.ack(tuple);
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields("word"));
}
}
static class WordBolt2 extends BaseRichBolt {
private OutputCollector collector;
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
this.collector = outputCollector;
}
@Override
public void execute(Tuple tuple) {
System.out.println("数据第2次处理中....");
//输出处理结果
System.out.println("处理结果：" + tuple.getString(0));
collector.ack(tuple);
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
}
}
} 执行结果：
利用flink-storm程序实现类似功能
需要更改flink相关依赖的版本到1.7.0，主要依赖了flink-storm的jar包
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>1.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_2.11</artifactId>
<version>1.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.11</artifactId>
<version>1.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-storm_2.11</artifactId>
<version>1.7.0</version>
</dependency> 只需要修改两处：
LocalCluster替换为FlinkLocalCluster，处理的任务从TopologyBuilder
.createTopology替换为FlinkTopology.createTopology(TopologyBuilder)
执行结果：
利用flink程序实现类似功能
利用Kafka发送初始消息“测试数据”。
public class FlinkProducer {
public static void main(String[] args) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
Properties properties = new Properties();
properties.put("bootstrap.servers", "10.225.173.107:9092,10.225.173.108:9092,10.225.173.109:9092");
FlinkKafkaProducer<String> flinkKafkaProducer = new FlinkKafkaProducer<>("flink", new SimpleStringSchema(), properties);
DataStreamSource<String> source = env.fromElements("测试数据");
source.addSink(flinkKafkaProducer);
env.execute();
}
} 接收Kafka的初始消息“测试数据”并加工处理。
public class FlinkConsumer {
public static void main(String[] args) throws Exception {
//1.创建执行环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//2.配置并创建初始数据源
Properties properties = new Properties();
properties.put("bootstrap.servers", "10.225.173.107:9092,10.225.173.108:9092,10.225.173.109:9092");
FlinkKafkaConsumer<String> flinkKafkaConsumer = new FlinkKafkaConsumer<>("flink", new SimpleStringSchema(), properties);
DataStreamSource<String> source = env.addSource(flinkKafkaConsumer);
//3.对数据源进行连续处理
source.process(new FlinkBolt1()).
process(new FlinkBolt2()).
process(new FlinkBolt3());
//4.执行flink程序
env.execute();
}
static class FlinkBolt1 extends ProcessFunction<String, Object> {
@Override
public void open(Configuration parameters) {
//开始第1次处理....
}
@Override
public void processElement(String s, ProcessFunction<String, Object>.Context context, Collector<Object> collector) throws Exception {
System.out.println("第1次处理前的值是：" + s);
s += "abc";
System.out.println("第1次处理后的值是：" + s);
collector.collect(s);
}
@Override
public void close() throws Exception {
//结束第1次处理
}
}
static class FlinkBolt2 extends ProcessFunction<Object, Object> {
@Override
public void open(Configuration parameters) {
//开始第2次处理....
}
@Override
public void processElement(Object s, ProcessFunction<Object, Object>.Context context, Collector<Object> collector) throws Exception {
s = s.toString();
System.out.println("第2次处理前的值是：" + s);
s += "def";
System.out.println("第2次处理后的值是：" + s);
collector.collect(s);
}
@Override
public void close() throws Exception {
//结束第2次处理....
}
}
static class FlinkBolt3 extends ProcessFunction<Object, Object> {
@Override
public void open(Configuration parameters) {
//开始第3次处理....
}
@Override
public void processElement(Object s, ProcessFunction<Object, Object>.Context context, Collector<Object> collector) throws Exception {
s = s.toString();
System.out.println("第3次处理前的值是：" + s);
s += "ghi";
System.out.println("第3次处理后的值是：" + s);
collector.collect(s);
}
@Override
public void close() throws Exception {
//结束第3次处理....
}
}
} 处理结果：
查看10道真题和解析