Version 2.1 includes data ingestion and query execution optimisations…

…, as well as new UDR and CDR envelope formats

collectors/cloudstack/ReleaseNotes.txt

@@ -0,0 +1,7 @@
+Version:   1.1.0
+Date:      16/September/2016
+Notes:     New output format for Usage Data
+
+Version:   1.0.0
+Date:      1/July/2016
+Notes:     Initial release of the RCB Cyclops CloudStack collector

collectors/cloudstack/config/cloudstack.conf

@@ -2,12 +2,12 @@
 ServerHTTPPort=4573
 
 # CloudStack Endpoint and its credentials
-CloudStackURL=https://yoururl
-CloudStackAPIKey=yourapi
-CloudStackSecretKey=yoursecretkey
+CloudStackURL=url
+CloudStackAPIKey=key
+CloudStackSecretKey=secret
 
 # If you want to start importing only from this day, specify YYYY-MM-DD, if not comment the line
-CloudStackFirstImport=2016-06-29
+CloudStackFirstImport=2016-09-16
 
 # Publisher (RabbitMQ) credentials
 PublisherHost=localhost

collectors/cloudstack/pom.xml

@@ -7,7 +7,7 @@
     <groupId>ch.icclab.cyclops.cloudstack</groupId>
     <artifactId>cyclops-cloudstack-collector</artifactId>
     <packaging>jar</packaging>
-    <version>0.0.1</version>
+    <version>1.1.0</version>
     <name>CloudStack collector</name>
 
     <properties>
@@ -43,6 +43,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-assembly-plugin</artifactId>
+                <version>2.6</version>
                 <configuration>
                     <descriptorRefs>
                         <descriptorRef>jar-with-dependencies</descriptorRef>

collectors/cloudstack/scripts/compile.sh

@@ -26,4 +26,4 @@ mvn dependency:tree
 mvn package assembly:single
 
 cd target
-mv cyclops-cloudstack-collector-0.0.1-jar-with-dependencies.jar ../bin/cloudstack.jar
+mv cyclops-cloudstack-collector-1.1.0-jar-with-dependencies.jar ../bin/cloudstack.jar

collectors/cloudstack/src/main/java/ch/icclab/cyclops/endpoint/RootEndpoint.java

@@ -36,6 +36,6 @@ public class RootEndpoint extends ServerResource {
     @Get
     public String root(){
         counter.increment(ENDPOINT);
-        return "RCB CloudStack collector micro service - version 1.0.1";
+        return "RCB CloudStack collector micro service - version 1.1.0";
     }
 }

collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/IPUsageData.java → collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/IP.java

@@ -24,7 +24,7 @@
  * Created on: 14-Oct-15
  * Description: POJO object for IP Usage Data (type 2)
  */
-public class IPUsageData extends UsageData {
+public class IP extends UsageData {
 
     // Whether source NAT is enabled for the IP address
     private boolean issourcenat;
@@ -50,4 +50,9 @@ public boolean issystem() {
     public void setIssystem(boolean issystem) {
         this.issystem = issystem;
     }
+
+    @Override
+    protected void additionalMetadata(Map map) {
+        // nothing to do
+    }
 }

collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/TemplateAndIsoUsageData.java → collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/ISO.java

@@ -16,15 +16,14 @@
  */
 package ch.icclab.cyclops.model;
 
-import java.util.HashMap;
 import java.util.Map;
 
 /**
  * Author: Martin Skoviera
  * Created on: 15-Oct-15
  * Description: POJO object for Template and ISO Usage Data, both type 7 (Template) and 8 (ISO)
  */
-public class TemplateAndIsoUsageData extends UsageData {
+public class ISO extends UsageData {
 
     // Size of the template, or ISO
     private Long size;
@@ -48,4 +47,12 @@ public Long getVirtualsize() {
     public void setVirtualsize(Long virtualsize) {
         this.virtualsize = virtualsize;
     }
+
+    @Override
+    protected void additionalMetadata(Map map) {
+        if (size != null)
+            map.put("size", size);
+        if (virtualsize != null)
+            map.put("virtualSize", virtualsize);
+    }
 }

collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/PolicyOrRuleUsageData.java → collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/LoadBalancer.java

@@ -24,5 +24,9 @@
  * Created on: 15-Oct-15
  * Description: POJO object for Load Balancer Policy (type 11) or Port Forwarding Rule Usage Data (type 12)
  */
-public class PolicyOrRuleUsageData extends UsageData {
+public class LoadBalancer extends UsageData {
+    @Override
+    protected void additionalMetadata(Map map) {
+        // nothing to do here
+    }
 }

collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkOfferingUsageData.java → collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkOffering.java

@@ -24,7 +24,7 @@
  * Created on: 15-Oct-15
  * Description: POJO object for Network Offering Usage Data (type 13)
  */
-public class NetworkOfferingUsageData extends UsageData {
+public class NetworkOffering extends UsageData {
 
     private boolean isdefault;
 
@@ -48,4 +48,11 @@ public String getVirtualmachineid() {
     public void setVirtualmachineid(String virtualmachineid) {
         this.virtualmachineid = virtualmachineid;
     }
+
+    @Override
+    protected void additionalMetadata(Map map) {
+        if (virtualmachineid != null) {
+            map.put("virtualmachineId", virtualmachineid);
+        }
+    }
 }

collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkReceived.java

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften
+ *  All Rights Reserved.
+ *
+ *     Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *     not use this file except in compliance with the License. You may obtain
+ *     a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *     WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *     License for the specific language governing permissions and limitations
+ *     under the License.
+ */
+package ch.icclab.cyclops.model;
+
+import java.util.Map;
+
+/**
+ * Author: Martin Skoviera
+ * Created on: 15-Oct-15
+ * Description: POJO object for Network Usage Data, both type 4 (Sent) and 5 (Received)
+ */
+public class NetworkReceived extends UsageData {
+
+    // Device type (domain router, external load balancer, etc.)
+    private String type;
+
+    // Its network ID
+    private String networkid;
+
+    /////////////////////////////
+    // Getters and Setters
+
+    public String getType() {
+        return type;
+    }
+
+    public void setType(String type) {
+        this.type = type;
+    }
+
+    public String getNetworkid() {
+        return networkid;
+    }
+
+    public void setNetworkid(String networkid) {
+        this.networkid = networkid;
+    }
+
+    @Override
+    protected void additionalMetadata(Map map) {
+        if (type != null)
+            map.put("type", type);
+        if (networkid != null)
+            map.put("networkId", networkid);
+    }
+}

collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkUsageData.java → collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/NetworkSent.java

@@ -16,12 +16,14 @@
  */
 package ch.icclab.cyclops.model;
 
+import java.util.Map;
+
 /**
  * Author: Martin Skoviera
  * Created on: 15-Oct-15
  * Description: POJO object for Network Usage Data, both type 4 (Sent) and 5 (Received)
  */
-public class NetworkUsageData extends UsageData {
+public class NetworkSent extends UsageData {
 
     // Device type (domain router, external load balancer, etc.)
     private String type;
@@ -47,4 +49,12 @@ public String getNetworkid() {
     public void setNetworkid(String networkid) {
         this.networkid = networkid;
     }
+
+    @Override
+    protected void additionalMetadata(Map map) {
+        if (type != null)
+            map.put("type", type);
+        if (networkid != null)
+            map.put("networkId", networkid);
+    }
 }

collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/PortForwarding.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften
+ *  All Rights Reserved.
+ *
+ *     Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *     not use this file except in compliance with the License. You may obtain
+ *     a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *     WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *     License for the specific language governing permissions and limitations
+ *     under the License.
+ */
+package ch.icclab.cyclops.model;
+
+import java.util.Map;
+
+/**
+ * Author: Martin Skoviera
+ * Created on: 15-Oct-15
+ * Description: POJO object for Load Balancer Policy (type 11) or Port Forwarding Rule Usage Data (type 12)
+ */
+public class PortForwarding extends UsageData {
+    @Override
+    protected void additionalMetadata(Map map) {
+        // nothing to do here
+    }
+}

collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/SnapshotUsageData.java → collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Snapshot.java

@@ -24,7 +24,7 @@
  * Created on: 15-Oct-15
  * Description: POJO object for Snapshot Usage Data (type 9)
  */
-public class SnapshotUsageData extends UsageData {
+public class Snapshot extends UsageData {
 
     // Size of the template, or ISO
     private Long size;
@@ -39,4 +39,10 @@ public Long getSize() {
     public void setSize(Long size) {
         this.size = size;
     }
+
+    @Override
+    protected void additionalMetadata(Map map) {
+        if (size != null)
+            map.put("size", size);
+    }
 }

collectors/cloudstack/src/main/java/ch/icclab/cyclops/model/Template.java

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2015. Zuercher Hochschule fuer Angewandte Wissenschaften
+ *  All Rights Reserved.
+ *
+ *     Licensed under the Apache License, Version 2.0 (the "License"); you may
+ *     not use this file except in compliance with the License. You may obtain
+ *     a copy of the License at
+ *
+ *          http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *     Unless required by applicable law or agreed to in writing, software
+ *     distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *     WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *     License for the specific language governing permissions and limitations
+ *     under the License.
+ */
+package ch.icclab.cyclops.model;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Author: Martin Skoviera
+ * Created on: 15-Oct-15
+ * Description: POJO object for Template and ISO Usage Data, both type 7 (Template) and 8 (ISO)
+ */
+public class Template extends UsageData {
+
+    // Size of the template, or ISO
+    private Long size;
+    private Long virtualsize;
+
+    /////////////////////////////
+    // Getters and Setters
+
+    public Long getSize() {
+        return size;
+    }
+
+    public void setSize(Long size) {
+        this.size = size;
+    }
+
+    public Long getVirtualsize() {
+        return virtualsize;
+    }
+
+    public void setVirtualsize(Long virtualsize) {
+        this.virtualsize = virtualsize;
+    }
+
+    @Override
+    protected void additionalMetadata(Map map) {
+        if (size != null)
+            map.put("size", size);
+        if (virtualsize != null)
+            map.put("virtualSize", virtualsize);
+    }
+}