001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.mapred;
020
021 import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString;
022 import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString;
023
024 import java.io.DataInput;
025 import java.io.DataOutput;
026 import java.io.IOException;
027 import java.text.ParseException;
028 import java.util.Collection;
029 import java.util.Iterator;
030
031 import org.apache.commons.collections.IteratorUtils;
032 import org.apache.commons.logging.Log;
033 import org.apache.hadoop.classification.InterfaceAudience;
034 import org.apache.hadoop.classification.InterfaceStability;
035 import org.apache.hadoop.mapreduce.FileSystemCounter;
036 import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup;
037 import org.apache.hadoop.mapreduce.counters.AbstractCounters;
038 import org.apache.hadoop.mapreduce.counters.CounterGroupBase;
039 import org.apache.hadoop.mapreduce.counters.CounterGroupFactory;
040 import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup;
041 import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup;
042 import org.apache.hadoop.mapreduce.counters.GenericCounter;
043 import org.apache.hadoop.mapreduce.counters.Limits;
044 import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
045 import org.apache.hadoop.mapreduce.util.CountersStrings;
046
047 import com.google.common.collect.Iterators;
048
049 /**
050 * A set of named counters.
051 *
052 * <p><code>Counters</code> represent global counters, defined either by the
053 * Map-Reduce framework or applications. Each <code>Counter</code> can be of
054 * any {@link Enum} type.</p>
055 *
056 * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
057 * counters from a particular <code>Enum</code> class.
058 */
059 @InterfaceAudience.Public
060 @InterfaceStability.Stable
061 public class Counters
062 extends AbstractCounters<Counters.Counter, Counters.Group> {
063
064 public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
065
066 public Counters() {
067 super(groupFactory);
068 }
069
070 public Counters(org.apache.hadoop.mapreduce.Counters newCounters) {
071 super(newCounters, groupFactory);
072 }
073
074 /**
075 * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters
076 * @param newCounters new Counters
077 * @return old Counters instance corresponding to newCounters
078 */
079 static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) {
080 return new Counters(newCounters);
081 }
082
083 public synchronized Group getGroup(String groupName) {
084 return super.getGroup(groupName);
085 }
086
087 @SuppressWarnings("unchecked")
088 public synchronized Collection<String> getGroupNames() {
089 return IteratorUtils.toList(super.getGroupNames().iterator());
090 }
091
092 public synchronized String makeCompactString() {
093 return CountersStrings.toEscapedCompactString(this);
094 }
095
096 /**
097 * A counter record, comprising its name and value.
098 */
099 @InterfaceAudience.Public
100 @InterfaceStability.Stable
101 public static class Counter implements org.apache.hadoop.mapreduce.Counter {
102 org.apache.hadoop.mapreduce.Counter realCounter;
103
104 Counter(org.apache.hadoop.mapreduce.Counter counter) {
105 this.realCounter = counter;
106 }
107
108 public Counter() {
109 this(new GenericCounter());
110 }
111
112 @SuppressWarnings("deprecation")
113 @Override
114 public void setDisplayName(String displayName) {
115 realCounter.setDisplayName(displayName);
116 }
117
118 @Override
119 public String getName() {
120 return realCounter.getName();
121 }
122
123 @Override
124 public String getDisplayName() {
125 return realCounter.getDisplayName();
126 }
127
128 @Override
129 public long getValue() {
130 return realCounter.getValue();
131 }
132
133 @Override
134 public void setValue(long value) {
135 realCounter.setValue(value);
136 }
137
138 @Override
139 public void increment(long incr) {
140 realCounter.increment(incr);
141 }
142
143 @Override
144 public void write(DataOutput out) throws IOException {
145 realCounter.write(out);
146 }
147
148 @Override
149 public void readFields(DataInput in) throws IOException {
150 realCounter.readFields(in);
151 }
152
153 /**
154 * Returns the compact stringified version of the counter in the format
155 * [(actual-name)(display-name)(value)]
156 * @return the stringified result
157 */
158 public String makeEscapedCompactString() {
159 return toEscapedCompactString(realCounter);
160 }
161
162 /**
163 * Checks for (content) equality of two (basic) counters
164 * @param counter to compare
165 * @return true if content equals
166 * @deprecated
167 */
168 @Deprecated
169 public boolean contentEquals(Counter counter) {
170 return realCounter.equals(counter.getUnderlyingCounter());
171 }
172
173 /**
174 * @return the value of the counter
175 */
176 public long getCounter() {
177 return realCounter.getValue();
178 }
179
180 @Override
181 public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() {
182 return realCounter;
183 }
184
185 @Override
186 public synchronized boolean equals(Object genericRight) {
187 if (genericRight instanceof Counter) {
188 synchronized (genericRight) {
189 Counter right = (Counter) genericRight;
190 return getName().equals(right.getName()) &&
191 getDisplayName().equals(right.getDisplayName()) &&
192 getValue() == right.getValue();
193 }
194 }
195 return false;
196 }
197
198 @Override
199 public int hashCode() {
200 return realCounter.hashCode();
201 }
202 }
203
204
205 /**
206 * <code>Group</code> of counters, comprising of counters from a particular
207 * counter {@link Enum} class.
208 *
209 * <p><code>Group</code>handles localization of the class name and the
210 * counter names.</p>
211 */
212 @InterfaceAudience.Public
213 @InterfaceStability.Stable
214 public static class Group implements CounterGroupBase<Counter> {
215 private CounterGroupBase<Counter> realGroup;
216
217 Group(GenericGroup group) {
218 this.realGroup = group;
219 }
220 Group(FSGroupImpl group) {
221 this.realGroup = group;
222 }
223
224 @SuppressWarnings({ "unchecked", "rawtypes" })
225 Group(FrameworkGroupImpl group) {
226 this.realGroup = group;
227 }
228
229 /**
230 * @param counterName the name of the counter
231 * @return the value of the specified counter, or 0 if the counter does
232 * not exist.
233 */
234 public long getCounter(String counterName) {
235 return getCounterValue(realGroup, counterName);
236 }
237
238 /**
239 * @return the compact stringified version of the group in the format
240 * {(actual-name)(display-name)(value)[][][]} where [] are compact strings
241 * for the counters within.
242 */
243 public String makeEscapedCompactString() {
244 return toEscapedCompactString(realGroup);
245 }
246
247 /**
248 * Get the counter for the given id and create it if it doesn't exist.
249 * @param id the numeric id of the counter within the group
250 * @param name the internal counter name
251 * @return the counter
252 * @deprecated use {@link #findCounter(String)} instead
253 */
254 @Deprecated
255 public Counter getCounter(int id, String name) {
256 return findCounter(name);
257 }
258
259 /**
260 * Get the counter for the given name and create it if it doesn't exist.
261 * @param name the internal counter name
262 * @return the counter
263 */
264 public Counter getCounterForName(String name) {
265 return findCounter(name);
266 }
267
268 @Override
269 public void write(DataOutput out) throws IOException {
270 realGroup.write(out);
271 }
272
273 @Override
274 public void readFields(DataInput in) throws IOException {
275 realGroup.readFields(in);
276 }
277
278 @Override
279 public Iterator<Counter> iterator() {
280 return realGroup.iterator();
281 }
282
283 @Override
284 public String getName() {
285 return realGroup.getName();
286 }
287
288 @Override
289 public String getDisplayName() {
290 return realGroup.getDisplayName();
291 }
292
293 @Override
294 public void setDisplayName(String displayName) {
295 realGroup.setDisplayName(displayName);
296 }
297
298 @Override
299 public void addCounter(Counter counter) {
300 realGroup.addCounter(counter);
301 }
302
303 @Override
304 public Counter addCounter(String name, String displayName, long value) {
305 return realGroup.addCounter(name, displayName, value);
306 }
307
308 @Override
309 public Counter findCounter(String counterName, String displayName) {
310 return realGroup.findCounter(counterName, displayName);
311 }
312
313 @Override
314 public Counter findCounter(String counterName, boolean create) {
315 return realGroup.findCounter(counterName, create);
316 }
317
318 @Override
319 public Counter findCounter(String counterName) {
320 return realGroup.findCounter(counterName);
321 }
322
323 @Override
324 public int size() {
325 return realGroup.size();
326 }
327
328 @Override
329 public void incrAllCounters(CounterGroupBase<Counter> rightGroup) {
330 realGroup.incrAllCounters(rightGroup);
331 }
332
333 @Override
334 public CounterGroupBase<Counter> getUnderlyingGroup() {
335 return realGroup;
336 }
337
338 @Override
339 public synchronized boolean equals(Object genericRight) {
340 if (genericRight instanceof CounterGroupBase<?>) {
341 @SuppressWarnings("unchecked")
342 CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>)
343 genericRight).getUnderlyingGroup();
344 return Iterators.elementsEqual(iterator(), right.iterator());
345 }
346 return false;
347 }
348
349 @Override
350 public int hashCode() {
351 return realGroup.hashCode();
352 }
353 }
354
355 // All the group impls need this for legacy group interface
356 static long getCounterValue(CounterGroupBase<Counter> group, String counterName) {
357 Counter counter = group.findCounter(counterName, false);
358 if (counter != null) return counter.getValue();
359 return 0L;
360 }
361
362 // Mix the generic group implementation into the Group interface
363 private static class GenericGroup extends AbstractCounterGroup<Counter> {
364
365 GenericGroup(String name, String displayName, Limits limits) {
366 super(name, displayName, limits);
367 }
368
369 @Override
370 protected Counter newCounter(String counterName, String displayName,
371 long value) {
372 return new Counter(new GenericCounter(counterName, displayName, value));
373 }
374
375 @Override
376 protected Counter newCounter() {
377 return new Counter();
378 }
379
380 @Override
381 public CounterGroupBase<Counter> getUnderlyingGroup() {
382 return this;
383 }
384 }
385
386 // Mix the framework group implementation into the Group interface
387 private static class FrameworkGroupImpl<T extends Enum<T>>
388 extends FrameworkCounterGroup<T, Counter> {
389
390 FrameworkGroupImpl(Class<T> cls) {
391 super(cls);
392 }
393
394 @Override
395 protected Counter newCounter(T key) {
396 return new Counter(new FrameworkCounter<T>(key, getName()));
397 }
398
399 @Override
400 public CounterGroupBase<Counter> getUnderlyingGroup() {
401 return this;
402 }
403 }
404
405 // Mix the file system counter group implementation into the Group interface
406 private static class FSGroupImpl extends FileSystemCounterGroup<Counter> {
407
408 @Override
409 protected Counter newCounter(String scheme, FileSystemCounter key) {
410 return new Counter(new FSCounter(scheme, key));
411 }
412
413 @Override
414 public CounterGroupBase<Counter> getUnderlyingGroup() {
415 return this;
416 }
417 }
418
419 public synchronized Counter findCounter(String group, String name) {
420 if (name.equals("MAP_INPUT_BYTES")) {
421 LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
422 "Use FileInputFormatCounters as group name and " +
423 " BYTES_READ as counter name instead");
424 return findCounter(FileInputFormatCounter.BYTES_READ);
425 }
426 return getGroup(group).getCounterForName(name);
427 }
428
429 /**
430 * Provide factory methods for counter group factory implementation.
431 * See also the GroupFactory in
432 * {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters}
433 */
434 static class GroupFactory extends CounterGroupFactory<Counter, Group> {
435
436 @Override
437 protected <T extends Enum<T>>
438 FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) {
439 return new FrameworkGroupFactory<Group>() {
440 @Override public Group newGroup(String name) {
441 return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package
442 }
443 };
444 }
445
446 @Override
447 protected Group newGenericGroup(String name, String displayName,
448 Limits limits) {
449 return new Group(new GenericGroup(name, displayName, limits));
450 }
451
452 @Override
453 protected Group newFileSystemGroup() {
454 return new Group(new FSGroupImpl());
455 }
456 }
457
458 private static final GroupFactory groupFactory = new GroupFactory();
459
460 /**
461 * Find a counter by using strings
462 * @param group the name of the group
463 * @param id the id of the counter within the group (0 to N-1)
464 * @param name the internal name of the counter
465 * @return the counter for that name
466 * @deprecated use {@link #findCounter(String, String)} instead
467 */
468 @Deprecated
469 public Counter findCounter(String group, int id, String name) {
470 return findCounter(group, name);
471 }
472
473 /**
474 * Increments the specified counter by the specified amount, creating it if
475 * it didn't already exist.
476 * @param key identifies a counter
477 * @param amount amount by which counter is to be incremented
478 */
479 public void incrCounter(Enum<?> key, long amount) {
480 findCounter(key).increment(amount);
481 }
482
483 /**
484 * Increments the specified counter by the specified amount, creating it if
485 * it didn't already exist.
486 * @param group the name of the group
487 * @param counter the internal name of the counter
488 * @param amount amount by which counter is to be incremented
489 */
490 public void incrCounter(String group, String counter, long amount) {
491 findCounter(group, counter).increment(amount);
492 }
493
494 /**
495 * Returns current value of the specified counter, or 0 if the counter
496 * does not exist.
497 * @param key the counter enum to lookup
498 * @return the counter value or 0 if counter not found
499 */
500 public synchronized long getCounter(Enum<?> key) {
501 return findCounter(key).getValue();
502 }
503
504 /**
505 * Increments multiple counters by their amounts in another Counters
506 * instance.
507 * @param other the other Counters instance
508 */
509 public synchronized void incrAllCounters(Counters other) {
510 for (Group otherGroup: other) {
511 Group group = getGroup(otherGroup.getName());
512 group.setDisplayName(otherGroup.getDisplayName());
513 for (Counter otherCounter : otherGroup) {
514 Counter counter = group.getCounterForName(otherCounter.getName());
515 counter.setDisplayName(otherCounter.getDisplayName());
516 counter.increment(otherCounter.getValue());
517 }
518 }
519 }
520
521 /**
522 * @return the total number of counters
523 * @deprecated use {@link #countCounters()} instead
524 */
525 public int size() {
526 return countCounters();
527 }
528
529 /**
530 * Convenience method for computing the sum of two sets of counters.
531 * @param a the first counters
532 * @param b the second counters
533 * @return a new summed counters object
534 */
535 public static Counters sum(Counters a, Counters b) {
536 Counters counters = new Counters();
537 counters.incrAllCounters(a);
538 counters.incrAllCounters(b);
539 return counters;
540 }
541
542 /**
543 * Logs the current counter values.
544 * @param log The log to use.
545 */
546 public void log(Log log) {
547 log.info("Counters: " + size());
548 for(Group group: this) {
549 log.info(" " + group.getDisplayName());
550 for (Counter counter: group) {
551 log.info(" " + counter.getDisplayName() + "=" +
552 counter.getCounter());
553 }
554 }
555 }
556
557 /**
558 * Represent the counter in a textual format that can be converted back to
559 * its object form
560 * @return the string in the following format
561 * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*
562 */
563 public String makeEscapedCompactString() {
564 return toEscapedCompactString(this);
565 }
566
567 /**
568 * Convert a stringified (by {@link #makeEscapedCompactString()} counter
569 * representation into a counter object.
570 * @param compactString to parse
571 * @return a new counters object
572 * @throws ParseException
573 */
574 public static Counters fromEscapedCompactString(String compactString)
575 throws ParseException {
576 return parseEscapedCompactString(compactString, new Counters());
577 }
578 }