001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.mapred;
020
021 import java.io.DataInput;
022 import java.io.DataOutput;
023 import java.io.IOException;
024 import java.util.ArrayList;
025 import java.util.Collection;
026
027 import org.apache.hadoop.classification.InterfaceAudience;
028 import org.apache.hadoop.classification.InterfaceStability;
029 import org.apache.hadoop.io.Text;
030 import org.apache.hadoop.io.Writable;
031 import org.apache.hadoop.io.WritableUtils;
032 import org.apache.hadoop.mapreduce.ClusterMetrics;
033 import org.apache.hadoop.mapreduce.TaskTrackerInfo;
034 import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
035
036 /**
037 * Status information on the current state of the Map-Reduce cluster.
038 *
039 * <p><code>ClusterStatus</code> provides clients with information such as:
040 * <ol>
041 * <li>
042 * Size of the cluster.
043 * </li>
044 * <li>
045 * Name of the trackers.
046 * </li>
047 * <li>
048 * Task capacity of the cluster.
049 * </li>
050 * <li>
051 * The number of currently running map & reduce tasks.
052 * </li>
053 * <li>
054 * State of the <code>JobTracker</code>.
055 * </li>
056 * <li>
057 * Details regarding black listed trackers.
058 * </li>
059 * </ol></p>
060 *
061 * <p>Clients can query for the latest <code>ClusterStatus</code>, via
062 * {@link JobClient#getClusterStatus()}.</p>
063 *
064 * @see JobClient
065 */
066 @InterfaceAudience.Public
067 @InterfaceStability.Stable
068 public class ClusterStatus implements Writable {
069 /**
070 * Class which encapsulates information about a blacklisted tasktracker.
071 *
072 * The information includes the tasktracker's name and reasons for
073 * getting blacklisted. The toString method of the class will print
074 * the information in a whitespace separated fashion to enable parsing.
075 */
076 public static class BlackListInfo implements Writable {
077
078 private String trackerName;
079
080 private String reasonForBlackListing;
081
082 private String blackListReport;
083
084 BlackListInfo() {
085 }
086
087
088 /**
089 * Gets the blacklisted tasktracker's name.
090 *
091 * @return tracker's name.
092 */
093 public String getTrackerName() {
094 return trackerName;
095 }
096
097 /**
098 * Gets the reason for which the tasktracker was blacklisted.
099 *
100 * @return reason which tracker was blacklisted
101 */
102 public String getReasonForBlackListing() {
103 return reasonForBlackListing;
104 }
105
106 /**
107 * Sets the blacklisted tasktracker's name.
108 *
109 * @param trackerName of the tracker.
110 */
111 void setTrackerName(String trackerName) {
112 this.trackerName = trackerName;
113 }
114
115 /**
116 * Sets the reason for which the tasktracker was blacklisted.
117 *
118 * @param reasonForBlackListing
119 */
120 void setReasonForBlackListing(String reasonForBlackListing) {
121 this.reasonForBlackListing = reasonForBlackListing;
122 }
123
124 /**
125 * Gets a descriptive report about why the tasktracker was blacklisted.
126 *
127 * @return report describing why the tasktracker was blacklisted.
128 */
129 public String getBlackListReport() {
130 return blackListReport;
131 }
132
133 /**
134 * Sets a descriptive report about why the tasktracker was blacklisted.
135 * @param blackListReport report describing why the tasktracker
136 * was blacklisted.
137 */
138 void setBlackListReport(String blackListReport) {
139 this.blackListReport = blackListReport;
140 }
141
142 @Override
143 public void readFields(DataInput in) throws IOException {
144 trackerName = Text.readString(in);
145 reasonForBlackListing = Text.readString(in);
146 blackListReport = Text.readString(in);
147 }
148
149 @Override
150 public void write(DataOutput out) throws IOException {
151 Text.writeString(out, trackerName);
152 Text.writeString(out, reasonForBlackListing);
153 Text.writeString(out, blackListReport);
154 }
155
156 @Override
157 /**
158 * Print information related to the blacklisted tasktracker in a
159 * whitespace separated fashion.
160 *
161 * The method changes any newlines in the report describing why
162 * the tasktracker was blacklisted to a ':' for enabling better
163 * parsing.
164 */
165 public String toString() {
166 StringBuilder sb = new StringBuilder();
167 sb.append(trackerName);
168 sb.append("\t");
169 sb.append(reasonForBlackListing);
170 sb.append("\t");
171 sb.append(blackListReport.replace("\n", ":"));
172 return sb.toString();
173 }
174
175 }
176
177 private int numActiveTrackers;
178 private Collection<String> activeTrackers = new ArrayList<String>();
179 private int numBlacklistedTrackers;
180 private int numExcludedNodes;
181 private long ttExpiryInterval;
182 private int map_tasks;
183 private int reduce_tasks;
184 private int max_map_tasks;
185 private int max_reduce_tasks;
186 private JobTrackerStatus status;
187 private Collection<BlackListInfo> blacklistedTrackersInfo =
188 new ArrayList<BlackListInfo>();
189
190 ClusterStatus() {}
191
192 /**
193 * Construct a new cluster status.
194 *
195 * @param trackers no. of tasktrackers in the cluster
196 * @param blacklists no of blacklisted task trackers in the cluster
197 * @param ttExpiryInterval the tasktracker expiry interval
198 * @param maps no. of currently running map-tasks in the cluster
199 * @param reduces no. of currently running reduce-tasks in the cluster
200 * @param maxMaps the maximum no. of map tasks in the cluster
201 * @param maxReduces the maximum no. of reduce tasks in the cluster
202 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
203 */
204 ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
205 int maps, int reduces,
206 int maxMaps, int maxReduces, JobTrackerStatus status) {
207 this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps,
208 maxReduces, status, 0);
209 }
210
211 /**
212 * Construct a new cluster status.
213 *
214 * @param trackers no. of tasktrackers in the cluster
215 * @param blacklists no of blacklisted task trackers in the cluster
216 * @param ttExpiryInterval the tasktracker expiry interval
217 * @param maps no. of currently running map-tasks in the cluster
218 * @param reduces no. of currently running reduce-tasks in the cluster
219 * @param maxMaps the maximum no. of map tasks in the cluster
220 * @param maxReduces the maximum no. of reduce tasks in the cluster
221 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
222 * @param numDecommissionedNodes number of decommission trackers
223 */
224 ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
225 int maps, int reduces, int maxMaps, int maxReduces,
226 JobTrackerStatus status, int numDecommissionedNodes) {
227 numActiveTrackers = trackers;
228 numBlacklistedTrackers = blacklists;
229 this.numExcludedNodes = numDecommissionedNodes;
230 this.ttExpiryInterval = ttExpiryInterval;
231 map_tasks = maps;
232 reduce_tasks = reduces;
233 max_map_tasks = maxMaps;
234 max_reduce_tasks = maxReduces;
235 this.status = status;
236 }
237
238 /**
239 * Construct a new cluster status.
240 *
241 * @param activeTrackers active tasktrackers in the cluster
242 * @param blacklistedTrackers blacklisted tasktrackers in the cluster
243 * @param ttExpiryInterval the tasktracker expiry interval
244 * @param maps no. of currently running map-tasks in the cluster
245 * @param reduces no. of currently running reduce-tasks in the cluster
246 * @param maxMaps the maximum no. of map tasks in the cluster
247 * @param maxReduces the maximum no. of reduce tasks in the cluster
248 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
249 */
250 ClusterStatus(Collection<String> activeTrackers,
251 Collection<BlackListInfo> blacklistedTrackers,
252 long ttExpiryInterval,
253 int maps, int reduces, int maxMaps, int maxReduces,
254 JobTrackerStatus status) {
255 this(activeTrackers, blacklistedTrackers, ttExpiryInterval, maps, reduces,
256 maxMaps, maxReduces, status, 0);
257 }
258
259
260 /**
261 * Construct a new cluster status.
262 *
263 * @param activeTrackers active tasktrackers in the cluster
264 * @param blackListedTrackerInfo blacklisted tasktrackers information
265 * in the cluster
266 * @param ttExpiryInterval the tasktracker expiry interval
267 * @param maps no. of currently running map-tasks in the cluster
268 * @param reduces no. of currently running reduce-tasks in the cluster
269 * @param maxMaps the maximum no. of map tasks in the cluster
270 * @param maxReduces the maximum no. of reduce tasks in the cluster
271 * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
272 * @param numDecommissionNodes number of decommission trackers
273 */
274
275 ClusterStatus(Collection<String> activeTrackers,
276 Collection<BlackListInfo> blackListedTrackerInfo, long ttExpiryInterval,
277 int maps, int reduces, int maxMaps, int maxReduces,
278 JobTrackerStatus status, int numDecommissionNodes) {
279 this(activeTrackers.size(), blackListedTrackerInfo.size(),
280 ttExpiryInterval, maps, reduces, maxMaps, maxReduces, status,
281 numDecommissionNodes);
282 this.activeTrackers = activeTrackers;
283 this.blacklistedTrackersInfo = blackListedTrackerInfo;
284 }
285
286 /**
287 * Get the number of task trackers in the cluster.
288 *
289 * @return the number of task trackers in the cluster.
290 */
291 public int getTaskTrackers() {
292 return numActiveTrackers;
293 }
294
295 /**
296 * Get the names of task trackers in the cluster.
297 *
298 * @return the active task trackers in the cluster.
299 */
300 public Collection<String> getActiveTrackerNames() {
301 return activeTrackers;
302 }
303
304 /**
305 * Get the names of task trackers in the cluster.
306 *
307 * @return the blacklisted task trackers in the cluster.
308 */
309 public Collection<String> getBlacklistedTrackerNames() {
310 ArrayList<String> blacklistedTrackers = new ArrayList<String>();
311 for(BlackListInfo bi : blacklistedTrackersInfo) {
312 blacklistedTrackers.add(bi.getTrackerName());
313 }
314 return blacklistedTrackers;
315 }
316
317 /**
318 * Get the number of blacklisted task trackers in the cluster.
319 *
320 * @return the number of blacklisted task trackers in the cluster.
321 */
322 public int getBlacklistedTrackers() {
323 return numBlacklistedTrackers;
324 }
325
326 /**
327 * Get the number of excluded hosts in the cluster.
328 * @return the number of excluded hosts in the cluster.
329 */
330 public int getNumExcludedNodes() {
331 return numExcludedNodes;
332 }
333
334 /**
335 * Get the tasktracker expiry interval for the cluster
336 * @return the expiry interval in msec
337 */
338 public long getTTExpiryInterval() {
339 return ttExpiryInterval;
340 }
341
342 /**
343 * Get the number of currently running map tasks in the cluster.
344 *
345 * @return the number of currently running map tasks in the cluster.
346 */
347 public int getMapTasks() {
348 return map_tasks;
349 }
350
351 /**
352 * Get the number of currently running reduce tasks in the cluster.
353 *
354 * @return the number of currently running reduce tasks in the cluster.
355 */
356 public int getReduceTasks() {
357 return reduce_tasks;
358 }
359
360 /**
361 * Get the maximum capacity for running map tasks in the cluster.
362 *
363 * @return the maximum capacity for running map tasks in the cluster.
364 */
365 public int getMaxMapTasks() {
366 return max_map_tasks;
367 }
368
369 /**
370 * Get the maximum capacity for running reduce tasks in the cluster.
371 *
372 * @return the maximum capacity for running reduce tasks in the cluster.
373 */
374 public int getMaxReduceTasks() {
375 return max_reduce_tasks;
376 }
377
378 /**
379 * Get the JobTracker's status.
380 *
381 * @return {@link JobTrackerStatus} of the JobTracker
382 */
383 public JobTrackerStatus getJobTrackerStatus() {
384 return status;
385 }
386
387 /**
388 * Gets the list of blacklisted trackers along with reasons for blacklisting.
389 *
390 * @return the collection of {@link BlackListInfo} objects.
391 *
392 */
393 public Collection<BlackListInfo> getBlackListedTrackersInfo() {
394 return blacklistedTrackersInfo;
395 }
396
397 public void write(DataOutput out) throws IOException {
398 if (activeTrackers.size() == 0) {
399 out.writeInt(numActiveTrackers);
400 out.writeInt(0);
401 } else {
402 out.writeInt(activeTrackers.size());
403 out.writeInt(activeTrackers.size());
404 for (String tracker : activeTrackers) {
405 Text.writeString(out, tracker);
406 }
407 }
408 if (blacklistedTrackersInfo.size() == 0) {
409 out.writeInt(numBlacklistedTrackers);
410 out.writeInt(blacklistedTrackersInfo.size());
411 } else {
412 out.writeInt(blacklistedTrackersInfo.size());
413 out.writeInt(blacklistedTrackersInfo.size());
414 for (BlackListInfo tracker : blacklistedTrackersInfo) {
415 tracker.write(out);
416 }
417 }
418 out.writeInt(numExcludedNodes);
419 out.writeLong(ttExpiryInterval);
420 out.writeInt(map_tasks);
421 out.writeInt(reduce_tasks);
422 out.writeInt(max_map_tasks);
423 out.writeInt(max_reduce_tasks);
424 WritableUtils.writeEnum(out, status);
425 }
426
427 public void readFields(DataInput in) throws IOException {
428 numActiveTrackers = in.readInt();
429 int numTrackerNames = in.readInt();
430 if (numTrackerNames > 0) {
431 for (int i = 0; i < numTrackerNames; i++) {
432 String name = Text.readString(in);
433 activeTrackers.add(name);
434 }
435 }
436 numBlacklistedTrackers = in.readInt();
437 int blackListTrackerInfoSize = in.readInt();
438 if(blackListTrackerInfoSize > 0) {
439 for (int i = 0; i < blackListTrackerInfoSize; i++) {
440 BlackListInfo info = new BlackListInfo();
441 info.readFields(in);
442 blacklistedTrackersInfo.add(info);
443 }
444 }
445 numExcludedNodes = in.readInt();
446 ttExpiryInterval = in.readLong();
447 map_tasks = in.readInt();
448 reduce_tasks = in.readInt();
449 max_map_tasks = in.readInt();
450 max_reduce_tasks = in.readInt();
451 status = WritableUtils.readEnum(in, JobTrackerStatus.class);
452 }
453 }