草庐IT

MapReduce报错：「MKDirs failed to create file」

WHYBIGDATA 2023-03-28 原文

MapReduce报错：「MKDirs failed to create file」

0. 写在前面

  • Linux：Ubuntu Kylin16.04
  • Hadoop：Hadoop2.7.2

1. 程序代码及报错信息

输入、输出路径

zhangsan@hadoop01:/$ ll | grep input
drwxr-xr-x 3 zhangsan zhangsan 4096 9月 20 03:35 input/
zhangsan@hadoop01:/$ ll | grep output
zhangsan@hadoop01:/$

程序代码

package com.mr.ch07.maxmin;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class MinMaxValueDemo7_2_1 {

public static class MinMaxMapper extends Mapper<Object, Text, Text, MinMaxWritable> {

private MinMaxWritable outTuple = new MinMaxWritable();

@Override
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
String[] strs = value.toString().split(" ");
String strDate = strs[0];// 定义记录日期的字符串变量strDate

if (strDate == null) {
return;// 如果该日期值为空，则返回
}

System.out.println(strs[0] + ",,,," + strs[1]);

// 将值即做为最大值又做为最小值存储到自定义Writable类MinMaxWritable中。
outTuple.setMin(Integer.parseInt(strs[1]));
outTuple.setMax(Integer.parseInt(strs[1]));

// 将结果写入上下文
context.write(new Text(strDate), outTuple);
}

}



public static class MinMaxReducer extends Reducer<Text, MinMaxWritable, Text, MinMaxWritable> {

private MinMaxWritable result = new MinMaxWritable();


@Override
public void reduce(Text key, Iterable<MinMaxWritable> values, Context context)
throws IOException, InterruptedException {

result.setMax(0);
result.setMin(0);

// 按key迭代输出value的值
for (MinMaxWritable val : values) {
// 最小值放于结果集中
if (result.getMin() == 0 || val.getMin() < result.getMin()) {
result.setMin(val.getMin());
}
// 最大值放于结果集中
if (result.getMax() == 0 || val.getMax() > result.getMax()) {
System.out.println("val.getMax(): " + val.getMax() + ",,,,"
+ result.getMax());
result.setMax(val.getMax());
}
}
context.write(key, result);
}

}

public static void main(String[] args) throws Exception {

Configuration conf = new Configuration();
// String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
String[] otherArgs = new String[] { "/input/ch07/minmax", "/output/ch07/minmax" };

if (otherArgs.length != 2) {
System.err.println("Usage: MinMaxCountDriver <in> <out>");
System.exit(2);
}

// Job job = new Job(conf, "StackOverflow Comment Date Min Max Count");
Job job = Job.getInstance(conf);
job.setJarByClass(MinMaxValueDemo7_2_1.class);
job.setMapperClass(MinMaxMapper.class);
job.setCombinerClass(MinMaxReducer.class);
job.setReducerClass(MinMaxReducer.class);


job.setOutputKeyClass(Text.class);
job.setOutputValueClass(MinMaxWritable.class);


FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));


System.exit(job.waitForCompletion(true) ? 0 : 1);
}




public static class MinMaxWritable implements Writable {
private int min;// 记录最小值
private int max;// 记录最大值

public int getMin() {
return min;
}

public void setMin(int min) {
this.min = min;
}

public int getMax() {
return max;
}

public void setMax(int max) {
this.max = max;
}



@Override
public void readFields(DataInput in) throws IOException {
min = in.readInt();
max = in.readInt();
}



@Override
public void write(DataOutput out) throws IOException {
out.writeInt(max);
out.writeInt(min);
}



@Override
public String toString() {
return max + "\t" + min;
}

}

}

报错信息

java.lang.Exception: java.io.IOException: Mkdirs failed to create file:/output/ch07/minmax/_temporary/0/_temporary/attempt_local391816241_0001_r_000000_0 (exists=false, cwd=file:/home/zhangsan/Java_Eclipse/eclipse-workspace/MapReduce)
at org.apache.hadoop.mapred.LocalJobRunner$Job.runTasks(LocalJobRunner.java:462)
at org.apache.hadoop.mapred.LocalJobRunner$Job.run(LocalJobRunner.java:529)
Caused by: java.io.IOException: Mkdirs failed to create file:/output/ch07/minmax/_temporary/0/_temporary/attempt_local391816241_0001_r_000000_0 (exists=false, cwd=file:/home/zhangsan/Java_Eclipse/eclipse-workspace/MapReduce)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:449)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:435)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:909)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:890)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:787)
at org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.getRecordWriter(TextOutputFormat.java:132)
at org.apache.hadoop.mapred.ReduceTask$NewTrackingRecordWriter.<init>(ReduceTask.java:540)
at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:614)
at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:389)
at org.apache.hadoop.mapred.LocalJobRunner$Job$ReduceTaskRunnable.run(LocalJobRunner.java:319)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2022-09-21 20:42:24,059 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1360)) - Job job_local391816241_0001 running in uber mode : false
2022-09-21 20:42:24,062 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) - map 100% reduce 0%
2022-09-21 20:42:24,064 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1380)) - Job job_local391816241_0001 failed with state FAILED due to: NA
2022-09-21 20:42:24,080 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1385)) - Counters: 30
File System Counters
FILE: Number of bytes read=247
FILE: Number of bytes written=269461
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
Map-Reduce Framework
Map input records=8
Map output records=8
Map output bytes=128
Map output materialized bytes=60
Input split bytes=97
Combine input records=8
Combine output records=3
Reduce input groups=0
Reduce shuffle bytes=60
Reduce input records=0
Reduce output records=0
Spilled Records=3
Shuffled Maps =1
Failed Shuffles=0
Merged Map outputs=1
GC time elapsed (ms)=0
Total committed heap usage (bytes)=193986560
Shuffle Errors
BAD_ID=0
CONNECTION=0
IO_ERROR=0
WRONG_LENGTH=0
WRONG_MAP=0
WRONG_REDUCE=0
File Input Format Counters
Bytes Read=96
File Output Format Counters
Bytes Written=0

2. 查找资料

网上有一个帖子提到了这个报错信息

​​链接​​

​​https://community.cloudera.com/t5/Support-Questions/MKDirs-failed-to-create-file/td-p/35041​​

根据​​@snm1523​​的回答，我尝试将​​mapred-site.xml​​添加如下内容

<property>
<name>mapreduce.jobtracker.address</name>
<value>localhost:9101</value>
</property>

3. 原因分析

输出路径不能​​create​​，那就先创建输出路径

  • 创建输出路径目录/output
zhangsan@hadoop01:/$ sudo mkdir /output
[sudo] zhangsan 的密码：
zhangsan@hadoop01:/$ ll | grep output
drwxr-xr-x 2 root root 4096 9月 21 20:43 output/
依旧跟前面一样的报错信息

创建了路径还是报错，那应该是涉及到​​权限问题​​

  • 修改/output目录权限为当前用户「即hadoop安装目录所在的所有者」
zhangsan@hadoop01:/$ sudo chown -R zhangsan:zhangsan output/
zhangsan@hadoop01:/$ ll | grep output
drwxr-xr-x 2 zhangsan zhangsan 4096 9月 21 20:43 output/
重新执行MR程序

  • 运行成功
2022-09-21 20:44:53,945 INFO [Thread-15] mapred.LocalJobRunner (LocalJobRunner.java:runTasks(456)) - reduce task executor complete.
2022-09-21 20:44:54,597 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1360)) - Job job_local2141955672_0001 running in uber mode : false
2022-09-21 20:44:54,600 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1367)) - map 100% reduce 100%
2022-09-21 20:44:54,602 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1378)) - Job job_local2141955672_0001 completed successfully
2022-09-21 20:44:54,615 INFO [main] mapreduce.Job (Job.java:monitorAndPrintJob(1385)) - Counters: 30
File System Counters
FILE: Number of bytes read=646
FILE: Number of bytes written=541974
FILE: Number of read operations=0
FILE: Number of large read operations=0
FILE: Number of write operations=0
Map-Reduce Framework
Map input records=8
Map output records=8
Map output bytes=128
Map output materialized bytes=60
Input split bytes=97
Combine input records=8
Combine output records=3
Reduce input groups=3
Reduce shuffle bytes=60
Reduce input records=3
Reduce output records=3
Spilled Records=6
Shuffled Maps =1
Failed Shuffles=0
Merged Map outputs=1
GC time elapsed (ms)=0
Total committed heap usage (bytes)=387973120
Shuffle Errors
BAD_ID=0
CONNECTION=0
IO_ERROR=0
WRONG_LENGTH=0
WRONG_MAP=0
WRONG_REDUCE=0
File Input Format Counters
Bytes Read=96
File Output Format Counters
Bytes Written=60
  • 查看结果
zhangsan@hadoop01:/output/ch07/minmax$ ll
总用量 20
drwxrwxr-x 2 zhangsan zhangsan 4096 9月 21 20:44 ./
drwxrwxr-x 3 zhangsan zhangsan 4096 9月 21 20:44 ../
-rw-r--r-- 1 zhangsan zhangsan 48 9月 21 20:44 part-r-00000
-rw-r--r-- 1 zhangsan zhangsan 12 9月 21 20:44 .part-r-00000.crc
-rw-r--r-- 1 zhangsan zhangsan 0 9月 21 20:44 _SUCCESS
-rw-r--r-- 1 zhangsan zhangsan 8 9月 21 20:44 ._SUCCESS.crc

4. 参考

https://community.cloudera.com/t5/Support-Questions/MKDirs-failed-to-create-file/td-p/35041
顺利结束

MapReduceMKDirsbrsectionzhangsanjavahadoopMapreduce大数据Hadoop大数据yyds干货盘点

有关MapReduce报错：「MKDirs failed to create file」的更多相关文章

  1. 深度学习部署：Windows安装pycocotools报错解决方法 - 2

    深度学习部署：Windows安装pycocotools报错解决方法1.pycocotools库的简介2.pycocotools安装的坑3.解决办法更多Ai资讯：公主号AiCharm本系列是作者在跑一些深度学习实例时，遇到的各种各样的问题及解决办法，希望能够帮助到大家。ERROR:Commanderroredoutwithexitstatus1:'D:\Anaconda3\python.exe'-u-c'importsys,setuptools,tokenize;sys.argv[0]='"'"'C:\\Users\\46653\\AppData\\Local\\Temp\\pip-instal

  2. ruby-on-rails - 报错 - 在 Snow Leopard 上安装 RVM - 2

    我正在尝试在我的SnowLeopard10.6.8上安装RVM，方法是:\curl-Lhttps://get.rvm.io|bash-sstable--ruby我得到这个错误:InstallingRubyfromsourceto:/Users/Villa/.rvm/rubies/ruby-2.0.0-p0,thismaytakeawhiledependingonyourcpu(s)...ruby-2.0.0-p0-#downloadingruby-2.0.0-p0,thismaytakeawhiledependingonyourconnection...ruby-2.0.0-p0-#e

  3. Unity 报错No ‘git‘ executable was found. Please install Git on your system then restart - 2

    亲测可用。Anerroroccurredwhileresolvingpackages:Projecthasinvaliddependencies: com.unity.xxx:No'git'executablewasfound.PleaseinstallGitonyour  systemthenrestartUnityandUnityHub在我们使用PackageManager时，Unity允许我们使用Git上的package（点击加号，选择addpackagefromgitURL，或者是直接在Asset/Packages/manifest.json中添加包名）。但是这种操作需要我们事先装好g

  4. ruby-on-rails - Rails/Ruby创建数据库报错: Unable to load the EventMachine C extension - 2

    更新:eventmachinegem已安装并在我的gemfile中:eventmachine(1.0.0,0.12.10)请帮忙!尝试使用以下内容创建数据库:Fitzs-MacBook-Pro:twilio_insanityFitz$rakedb:create'返回以下错误:UnabletoloadtheEventMachineCextension;Tousethepure-rubyreactor,require'em/pure_ruby'rakeaborted!cannotloadsuchfile--rubyeventmachine/Users/Fitz/.rvm/gems/ruby

  5. nginx配置https后报错nginx: [emerg] https protocol requires SSL support in XXX.conf详细解决方法 - 2

    一、前言最近，在测试环境的nginx里增加了一个https配置：location/api-meeting-qq/{proxy_passhttps://api.meeting.qq.com/;}然后，执行命令：//这个是nginx启动文件的路径，根据实际情况自行更改sudo/home/useradmin/nginx/sbin/nginx-sreload结果，nginx就报错了：nginx:[emerg]httpsprotocolrequiresSSLsupportin/home/useradmin/nginx/conf.d/trainNginx.conf:9二、解决方法百度发现，是之前安装ngi

  6. nvm报错Now using node v版本号 （64-bit）解决方法 - 2

    nvm报错Nowusingnodev版本号（64-bit）解决方法先上报错（安装后的一些问题请直接跳到尾部查看）安装NVM的原因是使用React时addreact-redux时提示我node版本问题，遂打算安装一Node版本管理工具因为我电脑上很早就安装了Node，安装NVM时提示我是否覆盖并管理本地已有版本，我选了Yes之后安装成功（后来检查发现和版本没关系，是因为我在node里去ADD真离谱自己这操作）安装NVM注意问题1.若修改安装路径一定补上nodejs2.打开安装文件位置3.增加以下映射node_mirror:npm.taobao.org/mirrors/node/npm_mirro

  7. git push报错：fatal: Authentication failed for ‘https://github.com/... - 2

    第一次用git传代码到GitHub时，填写用户名和密码出现报错：fatal:Authenticationfailedfor'https://github.com/试了下面的没用😢gitconfig-–globaluser.name"xxx"gitconfig--globaluser.email"xxx@xx.com"查看报错原因发现是因为git更新了认证方式在错误提示（糟糕忘截图）的网站里有说明-->https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-

  8. ruby - 在 Ruby 中，为什么在启动 irb 之后出现 foo.nil？说未定义的错误，@foo.nil？给出 "true"和 @@wah.nil？又报错了？ - 2

    在Ruby1.8.7和1.9.2中相同:$irbruby-1.8.7-p302>foo.nil?NameError:undefinedlocalvariableormethod`foo'for#from(irb):1ruby-1.8.7-p302>@bar.nil?=>trueruby-1.8.7-p302>@@wah.nil?NameError:uninitializedclassvariable@@wahinObjectfrom(irb):3为什么实例变量与局部变量和类变量的处理方式不同？ 最佳答案 在Ruby中，大多数未初始化

  9. Fiddler手机抓包网络报错解决办法 - 2

    首先打开fiddler，点击Tools-Options-Connections一、这里有两个注意点点击HTTPS，左边选项选择如图，右边Actions点击如图第二项会提示Success，点击确定点击Connections，这里注意Fiddlerlistensonport这里面填写默认8888即可，左边三个选项选择如图，以上操作完成后，重启Fiddler二、手机打开WiFi1.长按或者点击Wifi进行修改网络（如不会操作，此处根据具体机型自行百度）修改代理为手动，服务器主机名两种方式可以查到：①win+R，在输入框输入cmd，在弹窗中输入ipconfig，此时IPv4后面的地址就是你的主机ip，

  10. ruby - 安装gems导致报错 - 2

    ruby2.0.0p247(2013-06-27修订版41674)[x86_64-linux]gem2.0.3sudogeminstalltravisBuildingnativeextensions.Thiscouldtakeawhile...ERROR:Errorinstallingtravis:ERROR:Failedtobuildgemnativeextension./usr/bin/ruby1.9.1extconf.rb/usr/lib/ruby/1.9.1/rubygems/custom_require.rb:36:in`require':cannotloadsuchfile

随机推荐