hbase RegionServer节点启动失败

半夜收到告警,hbase一台主机服务断掉了,起来查看确实服务进程不挂了,这时候我手动重启,但是用jps查看到进程刚出现,有掉了,只好看日记到底是因为上面,日志如下: 2016-02-27 05:50:09,201 FATAL regionserver.HRegionServer: Master rejected startup because clock is out of sync org.apache.hadoop.hbase.ClockOutOfSyncException: org.apache.hadoop.hbase.ClockOutOfSyncException: Server hadnode3,60020,1456523407857 has been rejected; Reported time is too far out of sync with master. Time difference of 195659ms > max allowed of 30000ms at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:345) at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:238) at org.apache.hadoop.hbase.master.HMaster.regionServerStartup(HMaster.java:1277) at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:7910) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2027) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:74) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:526) at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106) at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:284) at org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:2104) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:857) at java.lang.Thread.run(Thread.java:744) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.ClockOutOfSyncException): org.apache.hadoop.hbase.ClockOutOfSyncException: Server hadnode3,60020,1456523407857 has been rejected; Reported time is too far out of sync with master. Time difference of 195659ms > max allowed of 30000ms at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:345) at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:238) at org.apache.hadoop.hbase.master.HMaster.regionServerStartup(HMaster.java:1277) at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:7910) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2027) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:74) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) at org.apache.hadoop.hbase.ipc.RpcClient.call(RpcClient.java:1457) at org.apache.hadoop.hbase.ipc.RpcClient.callBlockingMethod(RpcClient.java:1661) at org.apache.hadoop.hbase.ipc.RpcClient$BlockingRpcChannelImplementation.callBlockingMethod(RpcClient.java:1719) at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.regionServerStartup(RegionServerStatusProtos.java:8277) at org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:2102) ... 2 more 2016-02-27 05:50:09,203 FATAL regionserver.HRegionServer: ABORTING region server hadnode3,60020,1456523407857: Unhandled: org.apache.hadoop.hbase.ClockOutOfSyncException: Server hadnode3,60020,1456523407857 has been rejected; Reported time is too far out of sync with master. Time difference of 195659ms > max allowed of 30000ms at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:345) at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:238) at org.apache.hadoop.hbase.master.HMaster.regionServerStartup(HMaster.java:1277) at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:7910) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2027) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:74) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) org.apache.hadoop.hbase.ClockOutOfSyncException: org.apache.hadoop.hbase.ClockOutOfSyncException: Server hadnode3,60020,1456523407857 has been rejected; Reported time is too far out of sync with master. Time difference of 195659ms > max allowed of 30000ms at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:345) at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:238) at org.apache.hadoop.hbase.master.HMaster.regionServerStartup(HMaster.java:1277) at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:7910) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2027) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:74) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:526) at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106) at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:284) at org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:2104) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:857) at java.lang.Thread.run(Thread.java:744) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.ClockOutOfSyncException): org.apache.hadoop.hbase.ClockOutOfSyncException: Server hadnode3,60020,1456523407857 has been rejected; Reported time is too far out of sync with master. Time difference of 195659ms > max allowed of 30000ms at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:345) at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:238) at org.apache.hadoop.hbase.master.HMaster.regionServerStartup(HMaster.java:1277) at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:7910) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2027) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:74) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) at org.apache.hadoop.hbase.ipc.RpcClient.call(RpcClient.java:1457) at org.apache.hadoop.hbase.ipc.RpcClient.callBlockingMethod(RpcClient.java:1661) at org.apache.hadoop.hbase.ipc.RpcClient$BlockingRpcChannelImplementation.callBlockingMethod(RpcClient.java:1719) at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.regionServerStartup(RegionServerStatusProtos.java:8277) at org.apache.hadoop.hbase.regionserver.HRegionServer.reportForDuty(HRegionServer.java:2102) ... 2 more 2016-02-27 05:50:09,205 FATAL regionserver.HRegionServer: RegionServer abort: loaded coprocessors are: [] 2016-02-27 05:50:09,205 INFO regionserver.HRegionServer: STOPPED: Unhandled: org.apache.hadoop.hbase.ClockOutOfSyncException: Server hadnode3,60020,1456523407857 has been rejected; Reported time is too far out of sync with master. Time difference of 195659ms > max allowed of 30000ms at org.apache.hadoop.hbase.master.ServerManager.checkClockSkew(ServerManager.java:345) at org.apache.hadoop.hbase.master.ServerManager.regionServerStartup(ServerManager.java:238) at org.apache.hadoop.hbase.master.HMaster.regionServerStartup(HMaster.java:1277) at org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:7910) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2027) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:108) at org.apache.hadoop.hbase.ipc.FifoRpcScheduler$1.run(FifoRpcScheduler.java:74) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.run(FutureTask.java:262) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:744) 2016-02-27 05:50:09,206 INFO ipc.RpcServer: Stopping server on 60020 2016-02-27 05:50:09,207 INFO regionserver.HRegionServer: Stopping infoServer 2016-02-27 05:50:09,208 INFO mortbay.log: Stopped SelectChannelConnector@0.0.0.0:60030 2016-02-27 05:50:09,309 INFO snapshot.RegionServerSnapshotManager: Stopping RegionServerSnapshotManager abruptly. 2016-02-27 05:50:09,309 INFO regionserver.HRegionServer: aborting server null 2016-02-27 05:50:09,309 DEBUG catalog.CatalogTracker: Stopping catalog tracker org.apache.hadoop.hbase.catalog.CatalogTracker@483ca727 2016-02-27 05:50:09,310 INFO client.HConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x35315fd22183a56 2016-02-27 05:50:09,313 INFO zookeeper.ZooKeeper: Session: 0x35315fd22183a56 closed 2016-02-27 05:50:09,313 INFO zookeeper.ClientCnxn: EventThread shut down 2016-02-27 05:50:09,314 INFO regionserver.HRegionServer: stopping server null; all regions closed. 2016-02-27 05:50:09,414 INFO regionserver.Leases: regionserver60020 closing leases 2016-02-27 05:50:09,415 INFO regionserver.Leases: regionserver60020 closed leases 2016-02-27 05:50:09,415 INFO regionserver.CompactSplitThread: Waiting for Split Thread to finish... 2016-02-27 05:50:09,415 INFO regionserver.CompactSplitThread: Waiting for Merge Thread to finish... 2016-02-27 05:50:09,415 INFO regionserver.CompactSplitThread: Waiting for Large Compaction Thread to finish... 2016-02-27 05:50:09,415 INFO regionserver.CompactSplitThread: Waiting for Small Compaction Thread to finish... 2016-02-27 05:50:09,419 WARN zookeeper.RecoverableZooKeeper: Node /hbase/rs/hadnode3,60020,1456523407857 already deleted, retry=false 2016-02-27 05:50:09,419 WARN regionserver.HRegionServer: Failed deleting my ephemeral node org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase/rs/hadnode3,60020,1456523407857 at org.apache.zookeeper.KeeperException.create(KeeperException.java:111) at org.apache.zookeeper.KeeperException.create(KeeperException.java:51) at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873) at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.delete(RecoverableZooKeeper.java:179) at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:1273) at org.apache.hadoop.hbase.zookeeper.ZKUtil.deleteNode(ZKUtil.java:1262) at org.apache.hadoop.hbase.regionserver.HRegionServer.deleteMyEphemeralNode(HRegionServer.java:1315) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1027) at java.lang.Thread.run(Thread.java:744) 2016-02-27 05:50:09,423 INFO zookeeper.ZooKeeper: Session: 0x25315fd2dfa41b1 closed 2016-02-27 05:50:09,423 INFO zookeeper.ClientCnxn: EventThread shut down 2016-02-27 05:50:09,423 INFO regionserver.HRegionServer: stopping server null; zookeeper connection closed. 2016-02-27 05:50:09,423 INFO regionserver.HRegionServer: regionserver60020 exiting 2016-02-27 05:50:09,423 ERROR regionserver.HRegionServerCommandLine: Region server exiting java.lang.RuntimeException: HRegionServer Aborted at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.start(HRegionServerCommandLine.java:66) at org.apache.hadoop.hbase.regionserver.HRegionServerCommandLine.run(HRegionServerCommandLine.java:85) at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:70) at org.apache.hadoop.hbase.util.ServerCommandLine.doMain(ServerCommandLine.java:126) at org.apache.hadoop.hbase.regionserver.HRegionServer.main(HRegionServer.java:2489) 2016-02-27 05:50:09,425 INFO regionserver.ShutdownHook: Shutdown hook starting; hbase.shutdown.hook=true; fsShutdownHook=org.apache.hadoop.fs.FileSystem$Cache$ClientFinalizer@42b2818f 2016-02-27 05:50:09,425 INFO regionserver.ShutdownHook: Starting fs shutdown hook thread. 2016-02-27 05:50:09,428 INFO regionserver.ShutdownHook: Shutdown hook finished.
已邀请:

koyo - 网站管理 致力做全栈工程师 爱ui 爱前端

从log中可以清晰的看到“Master rejected startup because clock is out of sync”,master拒绝启动,因为时间不同步和master服务器的时间。只好同步网络时间,问题解决!

空心菜 - 心向阳光,茁壮成长

时间问题!

要回复问题请先登录注册