001/**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements.  See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership.  The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License.  You may obtain a copy of the License at
009 *
010 *     http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018package org.apache.hadoop.hdfs.server.namenode.metrics;
019
020import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
021import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
022
023import org.apache.hadoop.conf.Configuration;
024import org.apache.hadoop.hdfs.DFSConfigKeys;
025import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
026import org.apache.hadoop.metrics2.MetricsSystem;
027import org.apache.hadoop.metrics2.annotation.Metric;
028import org.apache.hadoop.metrics2.annotation.Metrics;
029import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
030import org.apache.hadoop.metrics2.lib.MetricsRegistry;
031import org.apache.hadoop.metrics2.lib.MutableCounterLong;
032import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
033import org.apache.hadoop.metrics2.lib.MutableQuantiles;
034import org.apache.hadoop.metrics2.lib.MutableRate;
035import org.apache.hadoop.metrics2.source.JvmMetrics;
036
037/**
038 * This class is for maintaining  the various NameNode activity statistics
039 * and publishing them through the metrics interfaces.
040 */
041@Metrics(name="NameNodeActivity", about="NameNode metrics", context="dfs")
042public class NameNodeMetrics {
043  final MetricsRegistry registry = new MetricsRegistry("namenode");
044
045  @Metric MutableCounterLong createFileOps;
046  @Metric MutableCounterLong filesCreated;
047  @Metric MutableCounterLong filesAppended;
048  @Metric MutableCounterLong getBlockLocations;
049  @Metric MutableCounterLong filesRenamed;
050  @Metric MutableCounterLong filesTruncated;
051  @Metric MutableCounterLong getListingOps;
052  @Metric MutableCounterLong deleteFileOps;
053  @Metric("Number of files/dirs deleted by delete or rename operations")
054  MutableCounterLong filesDeleted;
055  @Metric MutableCounterLong fileInfoOps;
056  @Metric MutableCounterLong addBlockOps;
057  @Metric MutableCounterLong getAdditionalDatanodeOps;
058  @Metric MutableCounterLong createSymlinkOps;
059  @Metric MutableCounterLong getLinkTargetOps;
060  @Metric MutableCounterLong filesInGetListingOps;
061  @Metric("Number of allowSnapshot operations")
062  MutableCounterLong allowSnapshotOps;
063  @Metric("Number of disallowSnapshot operations")
064  MutableCounterLong disallowSnapshotOps;
065  @Metric("Number of createSnapshot operations")
066  MutableCounterLong createSnapshotOps;
067  @Metric("Number of deleteSnapshot operations")
068  MutableCounterLong deleteSnapshotOps;
069  @Metric("Number of renameSnapshot operations")
070  MutableCounterLong renameSnapshotOps;
071  @Metric("Number of listSnapshottableDirectory operations")
072  MutableCounterLong listSnapshottableDirOps;
073  @Metric("Number of snapshotDiffReport operations")
074  MutableCounterLong snapshotDiffReportOps;
075  @Metric("Number of blockReceivedAndDeleted calls")
076  MutableCounterLong blockReceivedAndDeletedOps;
077  @Metric("Number of blockReports from individual storages")
078  MutableCounterLong storageBlockReportOps;
079  @Metric("Number of blockReports and blockReceivedAndDeleted queued")
080  MutableGaugeInt blockOpsQueued;
081  @Metric("Number of blockReports and blockReceivedAndDeleted batch processed")
082  MutableCounterLong blockOpsBatched;
083
084  @Metric("Number of file system operations")
085  public long totalFileOps(){
086    return
087      getBlockLocations.value() +
088      createFileOps.value() +
089      filesAppended.value() +
090      addBlockOps.value() +
091      getAdditionalDatanodeOps.value() +
092      filesRenamed.value() +
093      filesTruncated.value() +
094      deleteFileOps.value() +
095      getListingOps.value() +
096      fileInfoOps.value() +
097      getLinkTargetOps.value() +
098      createSnapshotOps.value() +
099      deleteSnapshotOps.value() +
100      allowSnapshotOps.value() +
101      disallowSnapshotOps.value() +
102      renameSnapshotOps.value() +
103      listSnapshottableDirOps.value() +
104      createSymlinkOps.value() +
105      snapshotDiffReportOps.value();
106  }
107
108
109  @Metric("Journal transactions") MutableRate transactions;
110  @Metric("Journal syncs") MutableRate syncs;
111  final MutableQuantiles[] syncsQuantiles;
112  @Metric("Journal transactions batched in sync")
113  MutableCounterLong transactionsBatchedInSync;
114  @Metric("Block report") MutableRate blockReport;
115  final MutableQuantiles[] blockReportQuantiles;
116  @Metric("Cache report") MutableRate cacheReport;
117  final MutableQuantiles[] cacheReportQuantiles;
118
119  @Metric("Duration in SafeMode at startup in msec")
120  MutableGaugeInt safeModeTime;
121  @Metric("Time loading FS Image at startup in msec")
122  MutableGaugeInt fsImageLoadTime;
123
124  @Metric("GetImageServlet getEdit")
125  MutableRate getEdit;
126  @Metric("GetImageServlet getImage")
127  MutableRate getImage;
128  @Metric("GetImageServlet putImage")
129  MutableRate putImage;
130
131  JvmMetrics jvmMetrics = null;
132  
133  NameNodeMetrics(String processName, String sessionId, int[] intervals,
134      final JvmMetrics jvmMetrics) {
135    this.jvmMetrics = jvmMetrics;
136    registry.tag(ProcessName, processName).tag(SessionId, sessionId);
137    
138    final int len = intervals.length;
139    syncsQuantiles = new MutableQuantiles[len];
140    blockReportQuantiles = new MutableQuantiles[len];
141    cacheReportQuantiles = new MutableQuantiles[len];
142    
143    for (int i = 0; i < len; i++) {
144      int interval = intervals[i];
145      syncsQuantiles[i] = registry.newQuantiles(
146          "syncs" + interval + "s",
147          "Journal syncs", "ops", "latency", interval);
148      blockReportQuantiles[i] = registry.newQuantiles(
149          "blockReport" + interval + "s", 
150          "Block report", "ops", "latency", interval);
151      cacheReportQuantiles[i] = registry.newQuantiles(
152          "cacheReport" + interval + "s",
153          "Cache report", "ops", "latency", interval);
154    }
155  }
156
157  public static NameNodeMetrics create(Configuration conf, NamenodeRole r) {
158    String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
159    String processName = r.toString();
160    MetricsSystem ms = DefaultMetricsSystem.instance();
161    JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
162    
163    // Percentile measurement is off by default, by watching no intervals
164    int[] intervals = 
165        conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
166    return ms.register(new NameNodeMetrics(processName, sessionId,
167        intervals, jm));
168  }
169
170  public JvmMetrics getJvmMetrics() {
171    return jvmMetrics;
172  }
173  
174  public void shutdown() {
175    DefaultMetricsSystem.shutdown();
176  }
177
178  public void incrGetBlockLocations() {
179    getBlockLocations.incr();
180  }
181
182  public void incrFilesCreated() {
183    filesCreated.incr();
184  }
185
186  public void incrCreateFileOps() {
187    createFileOps.incr();
188  }
189
190  public void incrFilesAppended() {
191    filesAppended.incr();
192  }
193
194  public void incrAddBlockOps() {
195    addBlockOps.incr();
196  }
197  
198  public void incrGetAdditionalDatanodeOps() {
199    getAdditionalDatanodeOps.incr();
200  }
201
202  public void incrFilesRenamed() {
203    filesRenamed.incr();
204  }
205
206  public void incrFilesTruncated() {
207    filesTruncated.incr();
208  }
209
210  public void incrFilesDeleted(long delta) {
211    filesDeleted.incr(delta);
212  }
213
214  public void incrDeleteFileOps() {
215    deleteFileOps.incr();
216  }
217
218  public void incrGetListingOps() {
219    getListingOps.incr();
220  }
221
222  public void incrFilesInGetListingOps(int delta) {
223    filesInGetListingOps.incr(delta);
224  }
225
226  public void incrFileInfoOps() {
227    fileInfoOps.incr();
228  }
229
230  public void incrCreateSymlinkOps() {
231    createSymlinkOps.incr();
232  }
233
234  public void incrGetLinkTargetOps() {
235    getLinkTargetOps.incr();
236  }
237
238  public void incrAllowSnapshotOps() {
239    allowSnapshotOps.incr();
240  }
241  
242  public void incrDisAllowSnapshotOps() {
243    disallowSnapshotOps.incr();
244  }
245  
246  public void incrCreateSnapshotOps() {
247    createSnapshotOps.incr();
248  }
249  
250  public void incrDeleteSnapshotOps() {
251    deleteSnapshotOps.incr();
252  }
253  
254  public void incrRenameSnapshotOps() {
255    renameSnapshotOps.incr();
256  }
257  
258  public void incrListSnapshottableDirOps() {
259    listSnapshottableDirOps.incr();
260  }
261  
262  public void incrSnapshotDiffReportOps() {
263    snapshotDiffReportOps.incr();
264  }
265  
266  public void incrBlockReceivedAndDeletedOps() {
267    blockReceivedAndDeletedOps.incr();
268  }
269  
270  public void incrStorageBlockReportOps() {
271    storageBlockReportOps.incr();
272  }
273
274  public void setBlockOpsQueued(int size) {
275    blockOpsQueued.set(size);
276  }
277
278  public void addBlockOpsBatched(int count) {
279    blockOpsBatched.incr(count);
280  }
281
282  public void addTransaction(long latency) {
283    transactions.add(latency);
284  }
285
286  public void incrTransactionsBatchedInSync() {
287    transactionsBatchedInSync.incr();
288  }
289
290  public void addSync(long elapsed) {
291    syncs.add(elapsed);
292    for (MutableQuantiles q : syncsQuantiles) {
293      q.add(elapsed);
294    }
295  }
296
297  public void setFsImageLoadTime(long elapsed) {
298    fsImageLoadTime.set((int) elapsed);
299  }
300
301  public void addBlockReport(long latency) {
302    blockReport.add(latency);
303    for (MutableQuantiles q : blockReportQuantiles) {
304      q.add(latency);
305    }
306  }
307
308  public void addCacheBlockReport(long latency) {
309    cacheReport.add(latency);
310    for (MutableQuantiles q : cacheReportQuantiles) {
311      q.add(latency);
312    }
313  }
314
315  public void setSafeModeTime(long elapsed) {
316    safeModeTime.set((int) elapsed);
317  }
318
319  public void addGetEdit(long latency) {
320    getEdit.add(latency);
321  }
322
323  public void addGetImage(long latency) {
324    getImage.add(latency);
325  }
326
327  public void addPutImage(long latency) {
328    putImage.add(latency);
329  }
330}