Merge branch 'graphing' of Shokunin/neo-infra into master

config.ru

@@ -4,9 +4,9 @@
 require File.join(File.dirname(__FILE__), 'web', 'controllers')
 
 # serve up static assets using rack
-# map "/js" do
-#  run Rack::Directory.new("#{File.join(File.dirname(__FILE__), 'web', 'static', 'js')}")
-# end
+ map "/css" do
+  run Rack::Directory.new("#{File.join(File.dirname(__FILE__), 'web', 'static', 'css')}")
+ end
 
 map '/' do
   run Toppage
@@ -23,3 +23,7 @@ end
 map '/search' do
   run Search
 end
+
+map '/graph' do
+  run Graph
+end

lib/neoinfra.rb

@@ -17,4 +17,5 @@ module NeoInfra
   require 'neoinfra/vpcs'
   require 'neoinfra/nodes'
   require 'neoinfra/cloudwatch'
+  require 'neoinfra/graph'
 end

lib/neoinfra/aws.rb

@@ -5,6 +5,7 @@ require 'regions'
 require 'mime-types'
 require 'fog-aws'
 require 's3'
+require 'sqs'
 require 'date'
 require 'ipaddr'
 require 'neo4j'
@@ -101,6 +102,7 @@ module NeoInfra
     end
 
     def load_buckets
+      aws = NeoInfra::Aws.new
       cw = NeoInfra::Cloudwatch.new
       @cfg.accounts.each do |account|
         base_conf = {
@@ -108,29 +110,28 @@ module NeoInfra
           aws_access_key_id: account[:key],
           aws_secret_access_key: account[:secret]
         }
-        s = Fog::Storage.new(base_conf)
-        s.directories.each do |bucket|
-          next unless Bucket.where(name: bucket.key).empty?
-          begin
+        aws.regions.each do |region|
+          region_conf = { region: region }
+          s = Fog::Storage.new(region_conf.merge(base_conf))
+          s.directories.each do |bucket|
+            next unless bucket.location == region
+            next unless Bucket.where(name: bucket.key).empty?
             vers = bucket.versioning?.to_s
             crea = bucket.creation_date.to_s
-          rescue StandardError
-            vers = 'unknown'
-            crea = 'unknown'
+            b = Bucket.new(
+              name: bucket.key,
+              versioning: vers,
+              creation: crea,
+              size: cw.get_bucket_size(account[:key], account[:secret], bucket.location, bucket.key)
+            )
+            b.save
+            BucketRegion.create(from_node: b, to_node: Region.where(region: bucket.location).first)
+            BucketAccount.create(from_node: b, to_node: AwsAccount.where(name: account[:name]).first)
           end
-          b = Bucket.new(
-            name: bucket.key,
-            versioning: vers,
-            creation: crea,
-            size: cw.get_bucket_size(account[:key], account[:secret], bucket.location, bucket.key)
-          )
-          b.save
-          BucketRegion.create(from_node: b, to_node: Region.where(region: bucket.location).first)
-          BucketAccount.create(from_node: b, to_node: AwsAccount.where(name: account[:name]).first)
         end
       end
     end
-
+####
     def load_security_groups
       @cfg.accounts.each do |account|
         base_conf = {
@@ -339,6 +340,53 @@ module NeoInfra
       end
     end
 
+
+    def list_queues
+      queues = []
+      SQSQueue.all.order('n.name DESC').each do |d|
+        queues << {
+          'name'      => d.name,
+          'modified'  => d.modified,
+          'creation'  => d.creation,
+          'retention' => d.retention,
+          'maxsize'   => d.maxsize,
+          'region'    => d.region.region,
+          'owner'     => d.owner.name
+        }
+      end
+      queues
+    end
+
+    def load_queues
+      aws = NeoInfra::Aws.new
+      cw = NeoInfra::Cloudwatch.new
+      @cfg.accounts.each do |account|
+        base_conf = {
+          aws_access_key_id: account[:key],
+          aws_secret_access_key: account[:secret]
+        }
+        aws.regions.each do |region|
+          region_conf = { region: region }
+          q = Fog::AWS::SQS.new(region_conf.merge(base_conf))
+          q.list_queues.data[:body]['QueueUrls'].each do |x|
+            next unless SQSQueue.where(url: x).empty?
+            theAttrs = q.get_queue_attributes(x, "All").data[:body]['Attributes']
+            z = SQSQueue.new(
+              url: x,
+              name: x.split('/')[-1],
+              modified: theAttrs['LastModifiedTimestamp'],
+              creation: theAttrs['CreatedTimestamp'],
+              retention: theAttrs['MessageRetentionPeriod'],
+              maxsize: theAttrs['MaximumMessageSize'],
+            )
+            z.save
+            SQSQueueRegion.create(from_node: z, to_node: Region.where(region: region).first)
+            SQSQueueAccount.create(from_node: z, to_node: AwsAccount.where(name: account[:name]).first)
+          end
+        end
+      end
+    end   
+
     def load_rds
       @cfg.accounts.each do |account|
         base_conf = {

lib/neoinfra/graph.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'neoinfra'
+require 'vpc'
+require 'accounts'
+require 'fog-aws'
+require 'neo4j'
+require 'csv'
+require 'sinatra'
+require 's3'
+
+
+# NeoInfra Account information
+module NeoInfra
+  # Provide informations about the accounts available
+  class Graph
+    def initialize
+      @cfg = NeoInfra::Config.new
+      neo4j_url = "http://#{@cfg.neo4j[:host]}:#{@cfg.neo4j[:port]}"
+      Neo4j::Session.open(:server_db, neo4j_url)
+    end
+
+    def graph_vpcs
+      csv_string = CSV.generate(force_quotes: false ) do |csv|
+        csv << ['id,value']
+        csv << ['aws,']
+        @cfg = NeoInfra::Config.new
+        @cfg.accounts.each do |account|
+          csv << ["aws.#{account[:name]},"]
+
+          Vpc.where(default: "false").select{|x| x.owned.name == account[:name]}.collect{|y| y.region.region}.uniq.each do |region|
+            csv << ["aws.#{account[:name]}.#{region},"]
+          end
+
+          Vpc.where(default: "false").each do |vpc|
+            if vpc.owned.name == account[:name]
+              csv << ["aws.#{account[:name]}.#{vpc.region.region}.#{vpc.name},1"]
+            end
+          end
+
+        end
+      end
+      return csv_string.gsub('"', '')
+    end
+ 
+    def graph_buckets
+      csv_string = CSV.generate(force_quotes: false ) do |csv|
+        csv << ['id,value']
+        csv << ['aws,']
+        @cfg = NeoInfra::Config.new
+        @cfg.accounts.each do |account|
+          csv << ["aws.#{account[:name]},"]
+
+          Bucket.all.select{|x| x.owner.name == account[:name]}.collect{|y| y.region.region}.uniq.each do |region|
+            csv << ["aws.#{account[:name]}.#{region},"]
+          end
+
+          Bucket.all.each do |bucket|
+            if bucket.owner.name == account[:name]
+              csv << ["aws.#{account[:name]}.#{bucket.region.region}.#{bucket.name.gsub("\.", ':')},1"]
+            end
+          end
+
+        end
+      end
+      return csv_string.gsub('"', '')
+    end
+  
+    def graph_queues
+      csv_string = CSV.generate(force_quotes: false ) do |csv|
+        csv << ['id,value']
+        csv << ['aws,']
+        @cfg = NeoInfra::Config.new
+        @cfg.accounts.each do |account|
+          csv << ["aws.#{account[:name]},"]
+
+          SQSQueue.all.select{|x| x.owner.name == account[:name]}.collect{|y| y.region.region}.uniq.each do |region|
+            csv << ["aws.#{account[:name]}.#{region},"]
+          end
+
+          SQSQueue.all.each do |q|
+            if q.owner.name == account[:name]
+              csv << ["aws.#{account[:name]}.#{q.region.region}.#{q.name},1"]
+            end
+          end
+
+        end
+      end
+      return csv_string.gsub('"', '')
+    end 
+
+  end
+end

models/sqs.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'neo4j'
+
+# Information on SQSQueue
+class SQSQueue
+  include Neo4j::ActiveNode
+  property :url, constraint: :unique
+  property :name
+  property :modified
+  property :creation
+  property :retention
+  property :maxsize
+  has_one :out, :region, rel_class: :SQSQueueRegion
+  has_one :out, :owner, rel_class: :SQSQueueAccount
+end
+
+# Map SQSQueue to Region
+class SQSQueueRegion
+  include Neo4j::ActiveRel
+  from_class :SQSQueue
+  to_class :Region
+  type :region
+end
+
+# Map SQSQueue to Region
+class SQSQueueAccount
+  include Neo4j::ActiveRel
+  from_class :SQSQueue
+  to_class :AwsAccount
+  type :owner
+end

tasks/load_data.rake

@@ -64,6 +64,14 @@ namespace :load_data do
     j.load_lambda
   end
 
+  desc 'Load SQS'
+  task :sqs do
+    puts 'loading SQS'
+    j = NeoInfra::Aws.new
+    j.load_queues
+  end
+
+
   desc 'Load Everything'
-  task all: %i[accounts regions vpcs buckets security_groups nodes rds dynamo lambda]
+  task all: %i[accounts regions vpcs buckets security_groups nodes rds dynamo lambda sqs]
 end

web/controllers/graph.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+lib_dir = File.join(File.dirname(File.expand_path(__FILE__)), '..', '..', 'lib')
+$LOAD_PATH.unshift(lib_dir) unless
+  $LOAD_PATH.include?(lib_dir) || $LOAD_PATH.include?(lib_dir)
+
+require 'json'
+require 'neoinfra'
+require 'sinatra'
+require 'sinatra/base'
+require 'sinatra/contrib'
+
+# Handle loading data into the graph db
+class Graph < Sinatra::Base
+  register Sinatra::RespondWith
+  set :views, File.join(File.dirname(__FILE__), '..', '/views')
+
+  get '/vpcs' do
+    headers('Content-Type' => "text/csv")
+    g = NeoInfra::Graph.new
+    g.graph_vpcs
+  end
+
+  get '/buckets' do
+    headers('Content-Type' => "text/csv")
+    g = NeoInfra::Graph.new
+    g.graph_buckets
+  end
+
+  get '/queues' do
+    headers('Content-Type' => "text/csv")
+    g = NeoInfra::Graph.new
+    g.graph_queues
+  end
+
+end

web/controllers/views.rb

@@ -69,4 +69,25 @@ class Views < Sinatra::Base
       end
     end
   end
+
+  get '/queues' do
+    j = NeoInfra::Aws.new
+    respond_to do |wants|
+      wants.html do
+        erb :view_queues,
+            layout: :base_layout,
+            locals: { queues: j.list_queues }
+      end
+    end
+  end
+
+  get '/graph/:graph_type' do
+    respond_to do |wants|
+      wants.html do
+        erb :graphview,
+            layout: :base_layout,
+            locals: { graph_type: params['graph_type'] }
+      end
+    end
+  end
 end

web/static/css/dendogram.css

@@ -0,0 +1,33 @@
+<style>
+
+.node circle {
+  fill: #999;
+}
+
+.node text {
+  font: 12px sans-serif;
+}
+
+.node--leaf circle {
+  fill: #FF3333;
+  stroke-width: 1.5px;
+}
+
+.node--internal circle {
+  fill: steelblue;
+  stroke: steelblue;
+  stroke-width: 1.5px;
+}
+
+.node--internal text {
+  text-shadow: 0 1px 0 #fff, 0 -1px 0 #fff, 1px 0 0 #fff, -1px 0 0 #fff;
+}
+
+.link {
+  fill: none;
+  stroke: #555;
+  stroke-opacity: 0.4;
+  stroke-width: 1.5px;
+}
+
+</style>

web/views/base_layout.erb

@@ -7,6 +7,7 @@
     <meta name="author" content="">
     <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/font-awesome/4.4.0/css/font-awesome.min.css">
     <link rel="icon" href="../../favicon.ico">
+    <link rel="stylesheet" href="/css/dendogram.css">
     <script src="https://code.jquery.com/jquery-3.2.1.min.js" integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4=" crossorigin="anonymous"></script>
 
 
@@ -37,10 +38,19 @@
               <a class="dropdown-item" href="/view/rds">RDS</a>
               <a class="dropdown-item" href="/view/dynamos">Dyanmos</a>
               <a class="dropdown-item" href="/view/lambdas">Labmda Functions</a>
+              <a class="dropdown-item" href="/view/queues">SQS</a>
               <a class="dropdown-item" href="/audit/tags">Tag Audit</a>
               <a class="dropdown-item" href="/load/all">Load Data</a>
             </div>
           </li>
+          <li class="nav-item dropdown">
+            <a class="nav-link dropdown-toggle" id="dropdown02" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Graphs</a>
+            <div class="dropdown-menu" aria-labelledby="dropdown02">
+              <a class="dropdown-item" href="/view/graph/vpcs">VPCs</a>
+              <a class="dropdown-item" href="/view/graph/buckets">S3 Buckets</a>
+              <a class="dropdown-item" href="/view/graph/queues">SQS</a>
+            </div>
+          </li>
         </ul>
         <form class="form-inline my-2 my-lg-0" action="/search/all" method="POST">
           <input class="form-control mr-sm-2" type="text" name="search" placeholder="Search">

web/views/graphview.erb

@@ -0,0 +1,52 @@
+<center><h3> <%= graph_type %></h3>
+<svg width="1200" height="900"></svg></center>
+<script src="http://d3js.org/d3.v4.min.js"></script>
+<script>
+
+var svg = d3.select("svg"),
+    width = +svg.attr("width"),
+    height = +svg.attr("height"),
+    g = svg.append("g").attr("transform", "translate(40,0)");
+
+var tree = d3.cluster()
+    .size([height, width - 160]);
+
+var stratify = d3.stratify()
+    .parentId(function(d) { return d.id.substring(0, d.id.lastIndexOf(".")); });
+
+d3.csv("/graph/<%= graph_type %>", function(error, data) {
+  if (error) throw error;
+
+  var root = stratify(data)
+      .sort(function(a, b) { return (a.height - b.height) || a.id.localeCompare(b.id); });
+
+  tree(root);
+
+  var link = g.selectAll(".link")
+      .data(root.descendants().slice(1))
+    .enter().append("path")
+      .attr("class", "link")
+      .attr("d", function(d) {
+        return "M" + d.y + "," + d.x
+            + "C" + (d.parent.y + 100) + "," + d.x
+            + " " + (d.parent.y + 100) + "," + d.parent.x
+            + " " + d.parent.y + "," + d.parent.x;
+      });
+
+  var node = g.selectAll(".node")
+      .data(root.descendants())
+    .enter().append("g")
+      .attr("class", function(d) { return "node" + (d.children ? " node--internal" : " node--leaf"); })
+      .attr("transform", function(d) { return "translate(" + d.y + "," + d.x + ")"; })
+
+  node.append("circle")
+      .attr("r", 2.5);
+
+  node.append("text")
+      .attr("dy", 3)
+      .attr("x", function(d) { return d.children ? -8 : 8; })
+      .style("text-anchor", function(d) { return d.children ? "end" : "start"; })
+      .text(function(d) { return d.id.substring(d.id.lastIndexOf(".") + 1); });
+});
+
+</script>

web/views/view_buckets.erb

@@ -25,9 +25,9 @@
 <td><%= bucket['owner'] %></td>
 <td>
   <% if bucket['versioning'] == "true" %>
-    <i class="fa fa-check-circle" aria-hidden="true"></i>
+    <i class="fa fa-check-circle" style="color:#00db19;" aria-hidden="true"></i>
   <% elsif bucket['versioning'] == "false" %>
-    <i class="fa fa-times-circle" aria-hidden="true"></i>
+    <i class="fa fa-times-circle" style="color:#db001d;" aria-hidden="true"></i>
   <% else %>
     <i class="fa fa-question-circle" aria-hidden="true"></i>
   <% end %>

web/views/view_queues.erb

@@ -0,0 +1,33 @@
+<h2 class="title"><br><br><center>SQS Queues</center></h2>
+
+<div class="container">
+  <table class="table table-hover">
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Owner</th>
+        <th>Region</th>
+        <th>Creation</th>
+        <th>Modified</th>
+        <th>Max Size</th>
+      </tr>
+    </thead>
+    <tbody>
+<% queues.each do |queue| %>
+<tr>
+<% if queue['creation'] ==  queue['modified'] %>
+  <% @color = '00db19' %>
+<% else %>
+  <% @color = 'db001d' %>
+ <% end %>
+<td><%= queue['name'] %></td>
+<td><%= queue['owner'] %></td>
+<td><%= queue['region'] %></td>
+<td><%= queue['creation'] %></td>
+<td> <font color="#<%= @color %>"> <%= queue['modified'] %></font></td>
+<td><%= queue['maxsize'] %></td>
+</tr>
+<% end %>
+    </tbody>
+  </table>
+</div>