STEP1:单条数据处理，试探处理规律
核心问题：
从类似"8.35.201.144 - - [30/May/2013:17:38:20 +0800] "GET /uc_server/avatar.php?uid=29331&size=middle HTTP/1.1" 301 -"的数据中提取 ip-访问时间-访问的url
代码：（用到了/opt/hadoop-3.1.0/share/hadoop中common和mapreduce下的一些jar包，java project）
import java.io.FileNotFoundException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
public class SingalData {
public static void main(String[] args) throws ParseException,FileNotFoundException {
String data = "8.35.201.144 - - [30/May/2013:17:38:20 +0800] \"GET /uc_server/"
+ "avatar.php?uid=29331&size=middle HTTP/1.1\" 301 -";
System.out.println("待处理的原数据为:"+data);
//解析ip
String ip = data.substring(0, data.indexOf("- -"));
System.out.println("ip的解析结果:"+ip);
// 解析time
String tmpTime = data.substring(data.indexOf("[") + 1);
tmpTime = tmpTime.substring(0, tmpTime.indexOf(" +0800"));
SimpleDateFormat dateFormat1 = new SimpleDateFormat(
"dd/MM/yyyy:HH:mm:ss");
Date date = dateFormat1.parse(tmpTime.replace("May", "05"));
SimpleDateFormat dateFormat2 = new SimpleDateFormat(
"yyyy-MM-dd|HH:mm:ss");
String time = dateFormat2.format(date);
System.out.println("time的解析结果:"+time);
// 解析url
String tmpUrl = data.substring(data.indexOf("\"") + 1);
if (tmpUrl.contains("HTTP")) {
tmpUrl = tmpUrl.substring(0, tmpUrl.indexOf(" HTTP"));
tmpUrl = tmpUrl.split(" ")[1];
} else {
tmpUrl = tmpUrl.substring(0, tmpUrl.indexOf("\""));
}
String url = tmpUrl;
System.out.println("url的解析结果:"+url);
}
}
运行结果：
待处理的原数据为:8.35.201.144 - - [30/May/2013:17:38:20 +0800] "GET /uc_server/
avatar.php?uid=29331&size=middle HTTP/1.1" 301 -
ip的解析结果:8.35.201.144
time的解析结果:2013-05-30|17:38:20
url的解析结果:/uc_server/avatar.php?uid=29331&size=middle
STEP2:写MyMapper
步骤分析：
①确定Map的输入为<LongWritable, Tex>
②将Map传入的Text转化为String类型进行处理
③确定Map的输出为<LongWritable, NullWritable>
④将String处理的结果即“ip+time+url"转为text并写入context
源码：
mport java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class MyMapper extends Mapper<LongWritable, Text, Text,NullWritable> {
@Override
protected void map(LongWritable key, Text value,Context context)throws
IOException, InterruptedException {
/*
*key:输入的数据
*value:数据 即句子
*Context:Map上下文 上文HDFS 下文Reducer
*/
String data=value.toString();
if(data.contains("/uc_server")||data.contains("/data")||
data.contains("/static")||data.contains("/template")||
data.contains("/source")||data.contains("/favicon.ico")||
data.contains("/images"))
return;
//获取ip
String ip=data.substring(0, data.indexOf("- -"));
//获取并格式化time
String tmpTime=data.substring(data.indexOf("[")+1);
tmpTime=tmpTime.substring(0,tmpTime.indexOf(" +0800"));
SimpleDateFormat dateFormat1=new SimpleDateFormat("dd/MM/yyyy:HH:mm:ss");
Date date=null;
try {
date = dateFormat1.parse(tmpTime.replace("May", "05"));
} catch (ParseException e) {
e.printStackTrace();
}
SimpleDateFormat dateFormat2=new SimpleDateFormat("yyyy-MM-dd|HH:mm:ss");
String time=dateFormat2.format(date);
//获取访问的url
String tmpUrl=data.substring(data.indexOf("\"")+1);
if(tmpUrl.contains("HTTP")){
tmpUrl = tmpUrl.substring(0, tmpUrl.indexOf(" HTTP"));
tmpUrl = tmpUrl.split(" ")[1];
}else{
tmpUrl = tmpUrl.substring(0, tmpUrl.indexOf("\""));
}
String url=tmpUrl;
//写入上下文
context.write(new Text(ip+" "+time+" "+url),NullWritable.get());
}
}
STEP3:分析是否需要些Reduce
发现map中已处理完毕，不需要写reduce
STEP4:写Main:
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class Main {
public static void main(String[] args) throws
IOException, ClassNotFoundException, InterruptedException {
//创建一个job=map+reduce
Configuration conf=new Configuration();
//创建一个job
Job job=Job.getInstance(conf);
//指定任务的入口
job.setJarByClass(Main.class);
//指定map
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(NullWritable.class);
//指定任务的输入输出
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
//提交任务
job.waitForCompletion(true);//true表示打印日志信息
}
}
STEP5:打jar包放到Hadoop上运行查看结果，需要准备好数据（日志）在HDFS上
运行结果示例：
1.169.170.214 2013-05-30|22:49:01 /api.php?mod=js&bid=65
1.169.170.214 2013-05-30|22:49:14 /api.php?mod=js&bid=65
1.170.183.87 2013-05-30|18:39:33 /api.php?mod=js&bid=65
1.170.183.87 2013-05-30|18:39:59 /api.php?mod=js&bid=94
1.170.6.222 2013-05-30|20:22:36 /api.php?mod=js&bid=65
1.170.6.222 2013-05-30|20:23:01 /api.php?mod=js&bid=94
1.171.165.64 2013-05-30|23:20:59 /api.php?mod=js&bid=65
1.171.165.64 2013-05-30|23:21:17 /thread-11220-1-1.html
1.171.52.130 2013-05-30|22:04:23 /api.php?mod=js&bid=65
1.171.52.242 2013-05-30|19:38:13 /api.php?mod=js&bid=65
1.171.62.241 2013-05-30|18:26:36 /api.php?mod=js&bid=66
1.173.227.222 2013-05-30|21:21:51 /api.php?mod=js&bid=65
1.173.227.222 2013-05-30|21:21:59 /api.php?mod=js&bid=66
1.173.227.222 2013-05-30|21:40:08 /api.php?mod=js&bid=94
