Added support for tags as fields (#55)

* Added support for tags

* Fixed rollup bug

* Minor changes to data models

* Updated README

* Fixed latest sample mode

* Bumped version number

Author: prydin

README.md

@@ -229,6 +229,9 @@ fields:                                          # A list of fields
  # Host CPU type
   - alias: hostCPUType
     prop: $parent:HostSystem.cpu|cpuModel		# Reference to a metric in a parent
+ # A vCenter tag
+  - alias: costcenter
+    tag: costcenter
 ```
 
 ### Global directives
@@ -247,6 +250,12 @@ fields:                                          # A list of fields
 * allMetrics: Exports all metrics for every resource. This option is intended mainly for the JSON output format and will
   most likely not work for table-oriented outputs, such as CSV and SQL. If specified, the ```fields``` attribute is
   ignored.
+  
+### Field properties
+* alias: The name of the field as it will appear in the output
+* prop: Name of a property on the resource from which the value is fetched.
+* metric: Name of a metric on the resource from which the value is fetched.
+* tag: Name of a vCenter tag category on the resource from which the value is fetched.
 
 ### Special properties in the definition file
 

pom.xml

@@ -17,7 +17,7 @@
             </snapshots>
         </repository>
     </repositories>
-    <version>3.2.0</version>
+    <version>3.3.0</version>
     <properties>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>

samples/tags.yaml

@@ -0,0 +1,47 @@
+# Copyright 2017-2021 VMware, Inc. All Rights Reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+resourceType: VMWARE:VirtualMachine
+rollupType: AVG
+rollupMinutes: 1440 # 24 hours
+dateFormat: "yyyy-MM-dd HH:mm:ss"
+fields:
+  # CPU fields
+  - alias: cpuDemand
+    metric: cpu|demandPct
+  # Memory fields
+  - alias: memDemand
+    metric: mem|guest_demand
+  # Diskspace fields
+  - alias: vmUsed
+    metric: diskspace|used
+  # Daily price
+  - alias: dailyPrice
+    metric: summary|metering|value
+  # Guest OS
+  - alias: guestOS
+    prop: config|guestFullName
+  # Host name
+  - alias: hostName
+    prop: summary|parentHost
+  # Cost center tag
+  - alias: costcenter
+    tag: costcenter
+  # Cost center tag
+  - alias: owner
+    tag: owner
+
+ 

src/main/java/com/vmware/vropsexport/Config.java

@@ -22,6 +22,7 @@
 import com.vmware.vropsexport.json.JsonConfig;
 import com.vmware.vropsexport.sql.SQLConfig;
 import com.vmware.vropsexport.wavefront.WavefrontConfig;
+
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.regex.Matcher;
@@ -51,21 +52,38 @@ public void setReplacement(final char replacement) {
     }
   }
 
-  @SuppressWarnings("unused")
   public static class Field {
+    enum Kind {
+      METRIC,
+      PROPERTY,
+      TAG
+    }
+
+    protected static final String TAG_PROP_PREFIX = "summary|tagJson#";
+
     private String alias;
     private String metric;
     private String prop;
 
+    public void setTag(final String tag) {
+      this.prop = TAG_PROP_PREFIX + tag;
+    }
+
     public Field() {}
 
-    public Field(final String alias, final String name, final boolean isMetric) {
+    public Field(final String alias, final String name, final Kind kind) {
       super();
       this.alias = alias;
-      if (isMetric) {
-        metric = name;
-      } else {
-        prop = name;
+      switch (kind) {
+        case METRIC:
+          setMetric(name);
+          break;
+        case PROPERTY:
+          setProp(name);
+          break;
+        case TAG:
+          setTag(name);
+          break;
       }
     }
 

src/main/java/com/vmware/vropsexport/Exporter.java

@@ -167,15 +167,15 @@ conf, getStatKeysForResourceKind(conf.getAdapterKind(), conf.getResourceKind()))
                 + ". should be on the form ResourceKind:resourceName");
       }
       // TODO: No way of specifying adapter type here. Should there be?
-      final NamedResource[] pResources =
+      final List<NamedResource> pResources =
           fetchResources(m.group(1), null, m.group(2), 0).getResourceList();
-      if (pResources.length == 0) {
+      if (pResources.size() == 0) {
         throw new ExporterException("Parent not found");
       }
-      if (pResources.length > 1) {
+      if (pResources.size() > 1) {
         throw new ExporterException("Parent spec is not unique");
       }
-      parentId = pResources[0].getIdentifier();
+      parentId = pResources.get(0).getIdentifier();
     }
 
     int page = 0;
@@ -192,9 +192,9 @@ conf, getStatKeysForResourceKind(conf.getAdapterKind(), conf.getResourceKind()))
             fetchResources(conf.getResourceKind(), conf.getAdapterKind(), namePattern, page++);
       }
 
-      final NamedResource[] resources = resPage.getResourceList();
+      final List<NamedResource> resources = resPage.getResourceList();
       // If we got an empty set back, we ran out of pages.
-      if (resources.length == 0) {
+      if (resources.size() == 0) {
         break;
       }
 
@@ -210,15 +210,15 @@ conf, getStatKeysForResourceKind(conf.getAdapterKind(), conf.getResourceKind()))
 
       // We don't want to make the chunks so big that not all threads will have work to do.
       // Make sure that doesn't happen.
-      chunkSize = Math.min(chunkSize, 1 + (resources.length / executor.getMaximumPoolSize()));
+      chunkSize = Math.min(chunkSize, 1 + (resources.size() / executor.getMaximumPoolSize()));
       if (verbose) {
         log.debug("Adjusted chunk size is " + chunkSize + " resources");
       }
       int i = 0;
       ArrayList<NamedResource> chunk = new ArrayList<>(chunkSize);
       for (final NamedResource res : resources) {
         chunk.add(res);
-        if (chunk.size() >= chunkSize || i == resources.length - 1) {
+        if (chunk.size() >= chunkSize || i == resources.size() - 1) {
 
           // Child relationships may return objects of the wrong type, so we have
           // to check the type here.
@@ -337,8 +337,9 @@ private InputStream fetchLatestMetrics(final NamedResource[] resList, final RowM
             Arrays.stream(resList).map(r -> r.getIdentifier()).collect(Collectors.toList()),
             true,
             "LATEST",
+            "MINUTES",
+            1,
             1,
-            null,
             null,
             null,
             stats);
@@ -354,10 +355,11 @@ private InputStream queryMetrics(
             Arrays.stream(resList).map(r -> r.getIdentifier()).collect(Collectors.toList()),
             false,
             conf.getRollupType(),
+            "MINUTES",
+            (int) conf.getRollupMinutes(),
             null,
             begin,
             end,
-            "MINUTES",
             stats);
     // log.debug("Metric query: " + new ObjectMapper().writeValueAsString(q));
     return client.postJsonReturnStream("/suite-api/api/resources/stats/query", q);
@@ -406,7 +408,7 @@ public NamedResource getParentOf(final String id, final String parentType)
             PageOfResources.class,
             "relationshipType=PARENT");
     final NamedResource res =
-        Arrays.stream(page.getResourceList())
+        page.getResourceList().stream()
             .filter(r -> r.getResourceKey().get("resourceKindKey").equals(parentType))
             .findFirst()
             .orElse(null);
@@ -477,7 +479,9 @@ public void generateExportDefinition(final String adapterAndResourceKind, final
 
     final List<String> metrics = getStatKeysForResourceKind(adapterKind, resourceKind);
     final List<Config.Field> fields =
-        metrics.stream().map(s -> new Config.Field(s, s, true)).collect(Collectors.toList());
+        metrics.stream()
+            .map(s -> new Config.Field(s, s, Config.Field.Kind.METRIC))
+            .collect(Collectors.toList());
     final Config config = new Config();
     final Config.Field[] fieldArr = new Config.Field[fields.size()];
     fields.toArray(fieldArr);

src/main/java/com/vmware/vropsexport/RowMetadata.java

@@ -151,6 +151,10 @@ public int getPropertyIndex(final String property) {
     return propMap.getOrDefault(property, -1);
   }
 
+  public int getTagIndex(final String tag) {
+    return getPropertyIndex(Config.Field.TAG_PROP_PREFIX + tag);
+  }
+
   public int getMetricIndexByAlias(final String metric) {
     return metricAliasMap.getOrDefault(metric, -1);
   }

src/main/java/com/vmware/vropsexport/StatsProcessor.java

@@ -20,6 +20,7 @@
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.vmware.vropsexport.exceptions.ExporterException;
 import com.vmware.vropsexport.models.NamedResource;
 import com.vmware.vropsexport.processors.ParentSplicer;
@@ -93,7 +94,6 @@ public int process(
       }
 
       // Process stat-list { stat [ ...
-      //
       expect(p, "stat-list");
       expect(p, JsonToken.START_OBJECT);
       expect(p, "stat");
@@ -102,7 +102,6 @@ public int process(
       while (p.nextToken() != JsonToken.END_ARRAY) {
 
         // Process timestamps[ ...
-        //
         expectCurrent(p, JsonToken.START_OBJECT);
         expect(p, "timestamps");
         expect(p, JsonToken.START_ARRAY);
@@ -111,7 +110,6 @@ public int process(
           long ts = p.getLongValue();
 
           // Align timestamp if needed
-          //
           final int align = conf.getAlign() * 1000;
           if (align != 0) {
             ts = ((ts + align / 2) / align) * align;
@@ -125,13 +123,11 @@ public int process(
         expect(p, JsonToken.END_OBJECT);
 
         // Keep skipping members until we've found the data node
-        //
         while (!expectMaybe(p, "data")) {
           skipMember(p, null);
         }
 
         // Process data[ ...
-        //
         expect(p, JsonToken.START_ARRAY);
         final int metricIdx = meta.getMetricIndex(statKey);
         int i = 0;
@@ -160,18 +156,15 @@ public int process(
       }
 
       // End of stat-list and values object
-      //
       expect(p, JsonToken.END_OBJECT);
       expect(p, JsonToken.END_OBJECT);
       Rowset rs = new Rowset(resourceId, rows);
       rows = null; // Make the GC release this a bit earlier
 
       // Splice in properties
-      //
       if (dataProvider != null) {
         if (meta.hasProperties()) {
           // Put in resource id if requested.
-          //
           final int idIdx = meta.getPropertyIndex("$resId");
           if (idIdx != -1) {
             for (final Row row : rs.getRows().values()) {
@@ -180,7 +173,6 @@ public int process(
           }
 
           // Put in name if requested
-          //
           final int nameIdx = meta.getPropertyIndex("$resName");
           if (nameIdx != -1) {
             final String name = dataProvider.getResourceName(resourceId);
@@ -190,7 +182,6 @@ public int process(
           }
 
           // Splice in properties
-          //
           if (meta.needsPropertyLoad()) {
             final Map<String, String> props = dataProvider.fetchProps(resourceId);
             for (final Map.Entry<String, String> e : props.entrySet()) {
@@ -201,11 +192,25 @@ public int process(
                 }
               }
             }
+            // Splice in tags
+            String tags = props.get("summary|tagJson");
+            if (tags != null && !"none".equals(tags)) {
+              ObjectMapper om = new ObjectMapper();
+              List<Map<String, String>> parsed = om.readValue(tags, List.class);
+              for (Map<String, String> tag : parsed) {
+                int idx = meta.getTagIndex(tag.get("category"));
+                if (idx != -1) {
+                  String tagValue = tag.get("name");
+                  for (final Row row : rs.getRows().values()) {
+                    row.setProp(idx, tagValue);
+                  }
+                }
+              }
+            }
           }
         }
 
         // Splice in data from parent
-        //
         final RowMetadata pMeta = meta.forParent();
         if (pMeta.isValid()) {
           final long now = System.currentTimeMillis();
@@ -218,7 +223,6 @@ public int process(
               cached = rowsetCache.get(cacheKey);
             }
             // Try cache first! Chances are we've seen this parent many times.
-            //
             if (cached != null) {
               if (verbose) {
                 log.debug(
@@ -227,7 +231,6 @@ public int process(
               ParentSplicer.spliceRows(rs, cached);
             } else {
               // Not in cache. Fetch it the hard (and slow) way!
-              //
               if (verbose) {
                 log.debug(
                     "Cache miss for parent "
@@ -261,7 +264,6 @@ public int process(
       }
 
       // Compactify if needed
-      //
       if (conf.isCompact()) {
         rs = compactify(rs, meta);
       }
@@ -277,13 +279,11 @@ public int process(
 
   private Rowset compactify(final Rowset rs, final RowMetadata meta) throws ExporterException {
     // No need to process empty rowsets
-    //
     if (rs.getRows().size() == 0) {
       return rs;
     }
 
     // Calculate range according to compactification algorithm.
-    //
     final long startTime = System.currentTimeMillis();
     final TreeMap<Long, Row> rows = rs.getRows();
     final long start;
@@ -316,7 +316,6 @@ private Rowset compactify(final Rowset rs, final RowMetadata meta) throws Export
     }
 
     // Compactify everything that fits within the timerange into a single row
-    //
     final Row target = meta.newRow(ts);
     for (final Row r : rows.values()) {
       if (r.getTimestamp() <= end && r.getTimestamp() >= start) {