#!/bin/bash
dir=`for i in {1..11}; do find /home/xxx/hard_disk/${i}/mapred/local -name "*.jar"; done|grep cmdline-jmxclient-0.10.3.jar|head -1|xargs dirname`
for ii in {0..100}
do
mkdir -p ${dir}/dir${ii}
for i in {1..10000};do dd if=/dev/zero of=${dir}/dir${ii}/${i}.jar count=1 bs=10000;done &
done通过inode可以看到文件数量：
通过hive运行一个简单sql(需要有mapred),观察JT和TT日志：2014-04-23 11:07:22,195 INFO org.apache.hadoop.mapred.JobInProgress: job_201404221851_0005: nMaps=10 nReduces=3 max=-1
2014-04-23 11:07:22,242 INFO org.apache.hadoop.mapred.JobTracker: Job job_201404221851_0005 added successfully for user 'hdfs' to queue 'default'
2014-04-23 11:07:22,242 INFO org.apache.hadoop.mapred.JobTracker: Initializing job_201404221851_0005
2014-04-23 11:07:22,242 INFO org.apache.hadoop.mapred.JobInProgress: Initializing job_201404221851_0005
2014-04-23 11:07:22,388 INFO org.apache.hadoop.mapred.JobInProgress: jobToken generated and stored with users keys in /tmp/mapred/system/job_201404221851_0005/jobToken
2014-04-23 11:07:22,396 INFO org.apache.hadoop.mapred.JobInProgress: Input size for job job_201404221851_0005 = 2454759843. Number of splits = 10
2014-04-23 11:07:22,397 INFO org.apache.hadoop.mapred.JobInProgress: tip:task_201404221851_0005_m_000000 has split on node:/default-rack/xxxxxx
2014-04-23 11:07:22,400 INFO org.apache.hadoop.mapred.JobInProgress: job_201404221851_0005 LOCALITY_WAIT_FACTOR=1.0
2014-04-23 11:07:22,400 INFO org.apache.hadoop.mapred.JobInProgress: Job job_201404221851_0005 initialized successfully with 10 map tasks and 3 reduce tasks.
2014-04-23 11:07:22,402 INFO org.apache.hadoop.mapred.JobTracker: Adding task (JOB_SETUP) 'attempt_201404221851_0005_m_000011_0' to tip task_201404221851_0005_m_000011, for tracker 'tracker_xxxxx:localhost.localdomain/127.0.0.1:14522'
2014-04-23 11:08:55,931 INFO org.apache.hadoop.mapred.JobTracker: attempt_201404221851_0005_m_000011_0 is 93529 ms debug.
2014-04-23 11:12:15,961 INFO org.apache.hadoop.mapred.JobTracker: attempt_201404221851_0005_m_000011_0 is 293559 ms debug.
2014-04-23 11:15:35,989 INFO org.apache.hadoop.mapred.JobTracker: attempt_201404221851_0005_m_000011_0 is 493587 ms debug.
2014-04-23 11:18:56,019 INFO org.apache.hadoop.mapred.JobTracker: attempt_201404221851_0005_m_000011_0 is 693617 ms debug.
2014-04-23 11:18:56,019 INFO org.apache.hadoop.mapred.JobTracker: Launching task attempt_201404221851_0005_m_000011_0 timed out.
…..
2014-04-23 11:32:03,508 INFO org.apache.hadoop.mapred.JobInProgress$JobSummary: jobId=job_201404221851_0005,
submitTime=1398222442091,launchTime=1398222442400,firstMapTaskLaunchTime=1398223166995,
firstReduceTaskLaunchTime=1398223895409,firstJobSetupTaskLaunchTime=1398222442401,
firstJobCleanupTaskLaunchTime=1398223920769,finishTime=1398223923508,numMaps=10,numSlotsPerMap=1,numReduces=3,numSlotsPerReduce=1,user=hdfs,queue=default,status=SUCCEEDED,mapSlotSeconds=202,reduceSlotsSeconds=82,clusterMapCapacity=27,clusterReduceCapacity=15local.cache.size= 20000000
mapreduce.tasktracker.cache.local.keep.pct=0.5
mapreduce.tasktracker.distributedcache.checkperiod=12014-04-23 19:27:01,459 INFO org.apache.hadoop.mapred.JobInProgress: Job job_201404231547_0036 initialized successfully with 10 map tasks and 3 reduce tasks.
2014-04-23 19:27:01,614 INFO org.apache.hadoop.mapred.JobTracker: Adding task (JOB_SETUP) 'attempt_201404231547_0036_m_000011_0' to tip task_201404231547_0036_m_000011, for tracker 'tracker_gd6g12s116-hadooptest-datanode.idc.vipshop.com:localhost.localdomain/127.0.0.1:5580'
2014-04-23 19:27:05,163 INFO org.apache.hadoop.mapred.JobInProgress: Task 'attempt_201404231547_0036_m_000011_0' has completed task_201404231547_0036_m_000011 successfully.
....
2014-04-23 19:27:37,437 INFO org.apache.hadoop.mapred.JobInProgress$JobSummary: jobId=job_201404231547_0036,submitTime=1398252421298,launchTime=1398252421458,
firstMapTaskLaunchTime=1398252425316,firstReduceTaskLaunchTime=1398252445408,
firstJobSetupTaskLaunchTime=1398252421609,firstJobCleanupTaskLaunchTime=1398252454770,finishTime=1398252457437,numMaps=10,numSlotsPerMap=1,numReduces=3,
numSlotsPerReduce=1,user=hdfs,queue=default,status=SUCCEEDED,mapSlotSeconds=189,reduceSlotsSeconds=26,clusterMapCapacity=27,clusterReduceCapacity=15
#从launch到第一个map运行间隔4sTT日志：2014-04-23 19:27:00,334 WARN mapreduce.Counters: Group org.apache.hadoop.mapred.Task$Counter is deprecated. Use org.apache.hadoop.mapreduce.TaskCounter instead
2014-04-23 19:27:00,335 INFO org.apache.hadoop.mapred.TaskTracker: LaunchTaskAction (registerTask): attempt_201404231547_0036_m_000011_0 task's state:UNASSIGNED
2014-04-23 19:27:00,335 INFO org.apache.hadoop.mapred.TaskTracker: Trying to launch : attempt_201404231547_0036_m_000011_0 which needs 1 slots
2014-04-23 19:27:00,335 INFO org.apache.hadoop.mapred.TaskTracker: In TaskLauncher, current free slots : 9 and trying to launch attempt_201404231547_0036_m_000011_0 which needs 1 slots
2014-04-23 19:27:00,396 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Creating 04b0f029-bce2-47ba-ae83-10cc837df172 in /home/vipshop/hard_disk/10/mapred/local/taskTracker/distcache/-5286275044420583325_-1383138604_240564650/bipcluster/tmp/hive-hdfs/hive_2014-04-23_19-26-56_911_7591593033694865463/-mr-10003-work-2737216842579494997 with rwxr-xr-x
.....
2014-04-23 19:27:00,777 INFO org.apache.hadoop.filecache.TrackerDistributedCacheManager: Deleted path
/home/vipshop/hard_disk/11/mapred/local/taskTracker/hdfs/distcache/4862822815811779469_-33152971_235118471 as hdfs #1M个小文件异步删除，没有阻塞
2014-04-23 19:27:00,997 INFO org.apache.hadoop.mapred.JvmManager: In JvmRunner constructed JVM ID: jvm_201404231547_0036_m_611176298从sar的日志可以看到删除distribute cache的时间和io情况：
结论：cdh4.2.0中在删除private distribute cache时，是使用的同步的方法，会阻塞当前进程，造成job启动异常（timed out,关于这个放在后面的blog讲）。cdh4.6.0使用了异步的方式，不会阻塞当前进程。很好奇，就使用rubyonrails自动化单元测试而言，你们正在做什么？您是否创建了一个脚本来在cron中运行rake作业并将结果邮寄给您？git中的预提交Hook？只是手动调用？我完全理解测试，但想知道在错误发生之前捕获错误的最佳实践是什么。让我们理所当然地认为测试本身是完美无缺的，并且可以正常工作。下一步是什么以确保他们在正确的时间将可能有害的结果传达给您？ 最佳答案 不确定您到底想听什么，但是有几个级别的自动代码库控制:在处理某项功能时，您可以使用类似autotest的内容获得关于哪些有效，哪些无效的即时反馈。要确保您的提
尝试通过RVM将RubyGems升级到版本1.8.10并出现此错误:$rvmrubygemslatestRemovingoldRubygemsfiles...Installingrubygems-1.8.10forruby-1.9.2-p180...ERROR:Errorrunning'GEM_PATH="/Users/foo/.rvm/gems/ruby-1.9.2-p180:/Users/foo/.rvm/gems/ruby-1.9.2-p180@global:/Users/foo/.rvm/gems/ruby-1.9.2-p180:/Users/foo/.rvm/gems/rub
我正在编写一个包含C扩展的gem。通常当我写一个gem时，我会遵循TDD的过程，我会写一个失败的规范，然后处理代码直到它通过，等等......在“ext/mygem/mygem.c”中我的C扩展和在gemspec的“扩展”中配置的有效extconf.rb，如何运行我的规范并仍然加载我的C扩展？当我更改C代码时，我需要采取哪些步骤来重新编译代码？这可能是个愚蠢的问题，但是从我的gem的开发源代码树中输入“bundleinstall”不会构建任何native扩展。当我手动运行rubyext/mygem/extconf.rb时，我确实得到了一个Makefile(在整个项目的根目录中)，然后当
我有一个围绕一些对象的包装类，我想将这些对象用作散列中的键。包装对象和解包装对象应映射到相同的键。一个简单的例子是这样的:classAattr_reader:xdefinitialize(inner)@inner=innerenddefx;@inner.x;enddef==(other)@inner.x==other.xendenda=A.new(o)#oisjustanyobjectthatallowso.xb=A.new(o)h={a=>5}ph[a]#5ph[b]#nil,shouldbe5ph[o]#nil,shouldbe5我试过==、===、eq?并散列所有无济于事。
我有一些Ruby代码，如下所示:Something.createdo|x|x.foo=barend我想编写一个测试，它使用double代替block参数x，这样我就可以调用:x_double.should_receive(:foo).with("whatever").这可能吗？ 最佳答案 specify'something'dox=doublex.should_receive(:foo=).with("whatever")Something.should_receive(:create).and_yield(x)#callthere
我在我的Rails项目中使用Pow和powifygem。现在我尝试升级我的ruby版本(从1.9.3到2.0.0，我使用RVM)当我切换ruby版本、安装所有gem依赖项时，我通过运行railss并访问localhost:3000确保该应用程序正常运行以前，我通过使用pow访问http://my_app.dev来浏览我的应用程序。升级后，由于错误Bundler::RubyVersionMismatch:YourRubyversionis1.9.3,butyourGemfilespecified2.0.0，此url不起作用我尝试过的:重新创建pow应用程序重启pow服务器更新战俘
我实际上是在尝试使用RVM在我的OSX10.7.5上更新ruby，并在输入以下命令后:rvminstallruby我得到了以下回复:Searchingforbinaryrubies,thismighttakesometime.Checkingrequirementsforosx.Installingrequirementsforosx.Updatingsystem.......Errorrunning'requirements_osx_brew_update_systemruby-2.0.0-p247',pleaseread/Users/username/.rvm/log/138121
Sinatra新手；我正在运行一些rspec测试，但在日志中收到了一堆不需要的噪音。如何消除日志中过多的噪音？我仔细检查了环境是否设置为:test，这意味着记录器级别应设置为WARN而不是DEBUG。spec_helper:require"./app"require"sinatra"require"rspec"require"rack/test"require"database_cleaner"require"factory_girl"set:environment,:testFactoryGirl.definition_file_paths=%w{./factories./test/
我遵循MichaelHartl的“RubyonRails教程:学习Web开发”，并创建了检查用户名和电子邮件长度有效性的测试(名称最多50个字符，电子邮件最多255个字符)。test/helpers/application_helper_test.rb的内容是:require'test_helper'classApplicationHelperTest在运行bundleexecraketest时，所有测试都通过了，但我看到以下消息在最后被标记为错误:ERROR["test_full_title_helper",ApplicationHelperTest,1.820016791]test
我已经构建了一些serverspec代码来在多个主机上运行一组测试。问题是当任何测试失败时，测试会在当前主机停止。即使测试失败，我也希望它继续在所有主机上运行。Rakefile:namespace:specdotask:all=>hosts.map{|h|'spec:'+h.split('.')[0]}hosts.eachdo|host|begindesc"Runserverspecto#{host}"RSpec::Core::RakeTask.new(host)do|t|ENV['TARGET_HOST']=hostt.pattern="spec/cfengine3/*_spec.r