001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.mapred;
020
021 import static org.apache.hadoop.mapreduce.util.CountersStrings.parseEscapedCompactString;
022 import static org.apache.hadoop.mapreduce.util.CountersStrings.toEscapedCompactString;
023
024 import java.io.DataInput;
025 import java.io.DataOutput;
026 import java.io.IOException;
027 import java.text.ParseException;
028 import java.util.Collection;
029 import java.util.Iterator;
030
031 import org.apache.commons.collections.IteratorUtils;
032 import org.apache.commons.logging.Log;
033 import org.apache.hadoop.classification.InterfaceAudience;
034 import org.apache.hadoop.classification.InterfaceStability;
035 import org.apache.hadoop.mapreduce.FileSystemCounter;
036 import org.apache.hadoop.mapreduce.counters.AbstractCounterGroup;
037 import org.apache.hadoop.mapreduce.counters.AbstractCounters;
038 import org.apache.hadoop.mapreduce.counters.CounterGroupBase;
039 import org.apache.hadoop.mapreduce.counters.CounterGroupFactory;
040 import org.apache.hadoop.mapreduce.counters.FileSystemCounterGroup;
041 import org.apache.hadoop.mapreduce.counters.FrameworkCounterGroup;
042 import org.apache.hadoop.mapreduce.counters.GenericCounter;
043 import org.apache.hadoop.mapreduce.counters.Limits;
044 import org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter;
045 import org.apache.hadoop.mapreduce.util.CountersStrings;
046
047 import com.google.common.collect.Iterators;
048
049 /**
050 * A set of named counters.
051 *
052 * <p><code>Counters</code> represent global counters, defined either by the
053 * Map-Reduce framework or applications. Each <code>Counter</code> can be of
054 * any {@link Enum} type.</p>
055 *
056 * <p><code>Counters</code> are bunched into {@link Group}s, each comprising of
057 * counters from a particular <code>Enum</code> class.
058 */
059 @InterfaceAudience.Public
060 @InterfaceStability.Stable
061 public class Counters
062 extends AbstractCounters<Counters.Counter, Counters.Group> {
063
064 public static int MAX_COUNTER_LIMIT = Limits.COUNTERS_MAX;
065
066 public Counters() {
067 super(groupFactory);
068 }
069
070 public Counters(org.apache.hadoop.mapreduce.Counters newCounters) {
071 super(newCounters, groupFactory);
072 }
073
074 /**
075 * Downgrade new {@link org.apache.hadoop.mapreduce.Counters} to old Counters
076 * @param newCounters new Counters
077 * @return old Counters instance corresponding to newCounters
078 */
079 static Counters downgrade(org.apache.hadoop.mapreduce.Counters newCounters) {
080 return new Counters(newCounters);
081 }
082
083 public synchronized Group getGroup(String groupName) {
084 return super.getGroup(groupName);
085 }
086
087 @SuppressWarnings("unchecked")
088 public synchronized Collection<String> getGroupNames() {
089 return IteratorUtils.toList(super.getGroupNames().iterator());
090 }
091
092 public synchronized String makeCompactString() {
093 return CountersStrings.toEscapedCompactString(this);
094 }
095
096 /**
097 * A counter record, comprising its name and value.
098 */
099 public static class Counter implements org.apache.hadoop.mapreduce.Counter {
100 org.apache.hadoop.mapreduce.Counter realCounter;
101
102 Counter(org.apache.hadoop.mapreduce.Counter counter) {
103 this.realCounter = counter;
104 }
105
106 public Counter() {
107 this(new GenericCounter());
108 }
109
110 @SuppressWarnings("deprecation")
111 @Override
112 public void setDisplayName(String displayName) {
113 realCounter.setDisplayName(displayName);
114 }
115
116 @Override
117 public String getName() {
118 return realCounter.getName();
119 }
120
121 @Override
122 public String getDisplayName() {
123 return realCounter.getDisplayName();
124 }
125
126 @Override
127 public long getValue() {
128 return realCounter.getValue();
129 }
130
131 @Override
132 public void setValue(long value) {
133 realCounter.setValue(value);
134 }
135
136 @Override
137 public void increment(long incr) {
138 realCounter.increment(incr);
139 }
140
141 @Override
142 public void write(DataOutput out) throws IOException {
143 realCounter.write(out);
144 }
145
146 @Override
147 public void readFields(DataInput in) throws IOException {
148 realCounter.readFields(in);
149 }
150
151 /**
152 * Returns the compact stringified version of the counter in the format
153 * [(actual-name)(display-name)(value)]
154 * @return the stringified result
155 */
156 public String makeEscapedCompactString() {
157 return toEscapedCompactString(realCounter);
158 }
159
160 /**
161 * Checks for (content) equality of two (basic) counters
162 * @param counter to compare
163 * @return true if content equals
164 * @deprecated
165 */
166 @Deprecated
167 public boolean contentEquals(Counter counter) {
168 return realCounter.equals(counter.getUnderlyingCounter());
169 }
170
171 /**
172 * @return the value of the counter
173 */
174 public long getCounter() {
175 return realCounter.getValue();
176 }
177
178 @Override
179 public org.apache.hadoop.mapreduce.Counter getUnderlyingCounter() {
180 return realCounter;
181 }
182
183 @Override
184 public synchronized boolean equals(Object genericRight) {
185 if (genericRight instanceof Counter) {
186 synchronized (genericRight) {
187 Counter right = (Counter) genericRight;
188 return getName().equals(right.getName()) &&
189 getDisplayName().equals(right.getDisplayName()) &&
190 getValue() == right.getValue();
191 }
192 }
193 return false;
194 }
195
196 @Override
197 public int hashCode() {
198 return realCounter.hashCode();
199 }
200 }
201
202
203 /**
204 * <code>Group</code> of counters, comprising of counters from a particular
205 * counter {@link Enum} class.
206 *
207 * <p><code>Group</code>handles localization of the class name and the
208 * counter names.</p>
209 */
210 public static class Group implements CounterGroupBase<Counter> {
211 private CounterGroupBase<Counter> realGroup;
212
213 Group(GenericGroup group) {
214 this.realGroup = group;
215 }
216 Group(FSGroupImpl group) {
217 this.realGroup = group;
218 }
219
220 @SuppressWarnings({ "unchecked", "rawtypes" })
221 Group(FrameworkGroupImpl group) {
222 this.realGroup = group;
223 }
224
225 /**
226 * @param counterName the name of the counter
227 * @return the value of the specified counter, or 0 if the counter does
228 * not exist.
229 */
230 public long getCounter(String counterName) {
231 return getCounterValue(realGroup, counterName);
232 }
233
234 /**
235 * @return the compact stringified version of the group in the format
236 * {(actual-name)(display-name)(value)[][][]} where [] are compact strings
237 * for the counters within.
238 */
239 public String makeEscapedCompactString() {
240 return toEscapedCompactString(realGroup);
241 }
242
243 /**
244 * Get the counter for the given id and create it if it doesn't exist.
245 * @param id the numeric id of the counter within the group
246 * @param name the internal counter name
247 * @return the counter
248 * @deprecated use {@link #findCounter(String)} instead
249 */
250 @Deprecated
251 public Counter getCounter(int id, String name) {
252 return findCounter(name);
253 }
254
255 /**
256 * Get the counter for the given name and create it if it doesn't exist.
257 * @param name the internal counter name
258 * @return the counter
259 */
260 public Counter getCounterForName(String name) {
261 return findCounter(name);
262 }
263
264 @Override
265 public void write(DataOutput out) throws IOException {
266 realGroup.write(out);
267 }
268
269 @Override
270 public void readFields(DataInput in) throws IOException {
271 realGroup.readFields(in);
272 }
273
274 @Override
275 public Iterator<Counter> iterator() {
276 return realGroup.iterator();
277 }
278
279 @Override
280 public String getName() {
281 return realGroup.getName();
282 }
283
284 @Override
285 public String getDisplayName() {
286 return realGroup.getDisplayName();
287 }
288
289 @Override
290 public void setDisplayName(String displayName) {
291 realGroup.setDisplayName(displayName);
292 }
293
294 @Override
295 public void addCounter(Counter counter) {
296 realGroup.addCounter(counter);
297 }
298
299 @Override
300 public Counter addCounter(String name, String displayName, long value) {
301 return realGroup.addCounter(name, displayName, value);
302 }
303
304 @Override
305 public Counter findCounter(String counterName, String displayName) {
306 return realGroup.findCounter(counterName, displayName);
307 }
308
309 @Override
310 public Counter findCounter(String counterName, boolean create) {
311 return realGroup.findCounter(counterName, create);
312 }
313
314 @Override
315 public Counter findCounter(String counterName) {
316 return realGroup.findCounter(counterName);
317 }
318
319 @Override
320 public int size() {
321 return realGroup.size();
322 }
323
324 @Override
325 public void incrAllCounters(CounterGroupBase<Counter> rightGroup) {
326 realGroup.incrAllCounters(rightGroup);
327 }
328
329 @Override
330 public CounterGroupBase<Counter> getUnderlyingGroup() {
331 return realGroup;
332 }
333
334 @Override
335 public synchronized boolean equals(Object genericRight) {
336 if (genericRight instanceof CounterGroupBase<?>) {
337 @SuppressWarnings("unchecked")
338 CounterGroupBase<Counter> right = ((CounterGroupBase<Counter>)
339 genericRight).getUnderlyingGroup();
340 return Iterators.elementsEqual(iterator(), right.iterator());
341 }
342 return false;
343 }
344
345 @Override
346 public int hashCode() {
347 return realGroup.hashCode();
348 }
349 }
350
351 // All the group impls need this for legacy group interface
352 static long getCounterValue(CounterGroupBase<Counter> group, String counterName) {
353 Counter counter = group.findCounter(counterName, false);
354 if (counter != null) return counter.getValue();
355 return 0L;
356 }
357
358 // Mix the generic group implementation into the Group interface
359 private static class GenericGroup extends AbstractCounterGroup<Counter> {
360
361 GenericGroup(String name, String displayName, Limits limits) {
362 super(name, displayName, limits);
363 }
364
365 @Override
366 protected Counter newCounter(String counterName, String displayName,
367 long value) {
368 return new Counter(new GenericCounter(counterName, displayName, value));
369 }
370
371 @Override
372 protected Counter newCounter() {
373 return new Counter();
374 }
375
376 @Override
377 public CounterGroupBase<Counter> getUnderlyingGroup() {
378 return this;
379 }
380 }
381
382 // Mix the framework group implementation into the Group interface
383 private static class FrameworkGroupImpl<T extends Enum<T>>
384 extends FrameworkCounterGroup<T, Counter> {
385
386 // Mix the framework counter implementation into the Counter interface
387 class FrameworkCounterImpl extends FrameworkCounter {
388 FrameworkCounterImpl(T key) {
389 super(key);
390 }
391
392 }
393
394 FrameworkGroupImpl(Class<T> cls) {
395 super(cls);
396 }
397
398 @Override
399 protected Counter newCounter(T key) {
400 return new Counter(new FrameworkCounterImpl(key));
401 }
402
403 @Override
404 public CounterGroupBase<Counter> getUnderlyingGroup() {
405 return this;
406 }
407 }
408
409 // Mix the file system counter group implementation into the Group interface
410 private static class FSGroupImpl extends FileSystemCounterGroup<Counter> {
411
412 private class FSCounterImpl extends FSCounter {
413
414 FSCounterImpl(String scheme, FileSystemCounter key) {
415 super(scheme, key);
416 }
417
418 }
419
420 @Override
421 protected Counter newCounter(String scheme, FileSystemCounter key) {
422 return new Counter(new FSCounterImpl(scheme, key));
423 }
424
425 @Override
426 public CounterGroupBase<Counter> getUnderlyingGroup() {
427 return this;
428 }
429 }
430
431 public synchronized Counter findCounter(String group, String name) {
432 if (name.equals("MAP_INPUT_BYTES")) {
433 LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " +
434 "Use FileInputFormatCounters as group name and " +
435 " BYTES_READ as counter name instead");
436 return findCounter(FileInputFormatCounter.BYTES_READ);
437 }
438 return getGroup(group).getCounterForName(name);
439 }
440
441 /**
442 * Provide factory methods for counter group factory implementation.
443 * See also the GroupFactory in
444 * {@link org.apache.hadoop.mapreduce.Counters mapreduce.Counters}
445 */
446 static class GroupFactory extends CounterGroupFactory<Counter, Group> {
447
448 @Override
449 protected <T extends Enum<T>>
450 FrameworkGroupFactory<Group> newFrameworkGroupFactory(final Class<T> cls) {
451 return new FrameworkGroupFactory<Group>() {
452 @Override public Group newGroup(String name) {
453 return new Group(new FrameworkGroupImpl<T>(cls)); // impl in this package
454 }
455 };
456 }
457
458 @Override
459 protected Group newGenericGroup(String name, String displayName,
460 Limits limits) {
461 return new Group(new GenericGroup(name, displayName, limits));
462 }
463
464 @Override
465 protected Group newFileSystemGroup() {
466 return new Group(new FSGroupImpl());
467 }
468 }
469
470 private static final GroupFactory groupFactory = new GroupFactory();
471
472 /**
473 * Find a counter by using strings
474 * @param group the name of the group
475 * @param id the id of the counter within the group (0 to N-1)
476 * @param name the internal name of the counter
477 * @return the counter for that name
478 * @deprecated use {@link #findCounter(String, String)} instead
479 */
480 @Deprecated
481 public Counter findCounter(String group, int id, String name) {
482 return findCounter(group, name);
483 }
484
485 /**
486 * Increments the specified counter by the specified amount, creating it if
487 * it didn't already exist.
488 * @param key identifies a counter
489 * @param amount amount by which counter is to be incremented
490 */
491 public void incrCounter(Enum<?> key, long amount) {
492 findCounter(key).increment(amount);
493 }
494
495 /**
496 * Increments the specified counter by the specified amount, creating it if
497 * it didn't already exist.
498 * @param group the name of the group
499 * @param counter the internal name of the counter
500 * @param amount amount by which counter is to be incremented
501 */
502 public void incrCounter(String group, String counter, long amount) {
503 findCounter(group, counter).increment(amount);
504 }
505
506 /**
507 * Returns current value of the specified counter, or 0 if the counter
508 * does not exist.
509 * @param key the counter enum to lookup
510 * @return the counter value or 0 if counter not found
511 */
512 public synchronized long getCounter(Enum<?> key) {
513 return findCounter(key).getValue();
514 }
515
516 /**
517 * Increments multiple counters by their amounts in another Counters
518 * instance.
519 * @param other the other Counters instance
520 */
521 public synchronized void incrAllCounters(Counters other) {
522 for (Group otherGroup: other) {
523 Group group = getGroup(otherGroup.getName());
524 group.setDisplayName(otherGroup.getDisplayName());
525 for (Counter otherCounter : otherGroup) {
526 Counter counter = group.getCounterForName(otherCounter.getName());
527 counter.setDisplayName(otherCounter.getDisplayName());
528 counter.increment(otherCounter.getValue());
529 }
530 }
531 }
532
533 /**
534 * @return the total number of counters
535 * @deprecated use {@link #countCounters()} instead
536 */
537 public int size() {
538 return countCounters();
539 }
540
541 /**
542 * Convenience method for computing the sum of two sets of counters.
543 * @param a the first counters
544 * @param b the second counters
545 * @return a new summed counters object
546 */
547 public static Counters sum(Counters a, Counters b) {
548 Counters counters = new Counters();
549 counters.incrAllCounters(a);
550 counters.incrAllCounters(b);
551 return counters;
552 }
553
554 /**
555 * Logs the current counter values.
556 * @param log The log to use.
557 */
558 public void log(Log log) {
559 log.info("Counters: " + size());
560 for(Group group: this) {
561 log.info(" " + group.getDisplayName());
562 for (Counter counter: group) {
563 log.info(" " + counter.getDisplayName() + "=" +
564 counter.getCounter());
565 }
566 }
567 }
568
569 /**
570 * Represent the counter in a textual format that can be converted back to
571 * its object form
572 * @return the string in the following format
573 * {(groupName)(group-displayName)[(counterName)(displayName)(value)][]*}*
574 */
575 public String makeEscapedCompactString() {
576 return toEscapedCompactString(this);
577 }
578
579 /**
580 * Convert a stringified (by {@link #makeEscapedCompactString()} counter
581 * representation into a counter object.
582 * @param compactString to parse
583 * @return a new counters object
584 * @throws ParseException
585 */
586 public static Counters fromEscapedCompactString(String compactString)
587 throws ParseException {
588 return parseEscapedCompactString(compactString, new Counters());
589 }
590 }