Merge pull request #3 from a-chacon/add_ip_model

feat: add ipgeo object related to visit with data of ip

Gemfile

@@ -13,4 +13,7 @@ gem "browser"
 group :test do
   gem 'byebug'
   gem 'faker', :git => 'https://github.com/faker-ruby/faker.git', :branch => 'master'
+  gem "webmock"
 end
+
+gem "http"

Gemfile.lock

@@ -9,9 +9,10 @@ GIT
 PATH
   remote: .
   specs:
-    rails_url_shortener (0.1.3)
+    rails_url_shortener (0.2.0)
       browser (>= 5.3.0)
       bundler (>= 1.15.0)
+      http (>= 5.0.4)
       rails (>= 7.0.2.3)
 
 GEM
@@ -82,17 +83,39 @@ GEM
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
     browser (5.3.1)
     builder (3.2.4)
     byebug (11.1.3)
     concurrent-ruby (1.1.10)
+    crack (0.4.5)
+      rexml
     crass (1.0.6)
     digest (3.1.0)
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
     erubi (1.10.0)
+    ffi (1.15.5)
+    ffi-compiler (1.0.1)
+      ffi (>= 1.0.0)
+      rake
     globalid (1.0.0)
       activesupport (>= 5.0)
+    hashdiff (1.0.1)
+    http (5.0.4)
+      addressable (~> 2.8)
+      http-cookie (~> 1.0)
+      http-form_data (~> 2.2)
+      llhttp-ffi (~> 0.4.0)
+    http-cookie (1.0.4)
+      domain_name (~> 0.5)
+    http-form_data (2.3.0)
     i18n (1.10.0)
       concurrent-ruby (~> 1.0)
+    llhttp-ffi (0.4.0)
+      ffi-compiler (~> 1.0)
+      rake (~> 13.0)
     loofah (2.16.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
@@ -119,6 +142,7 @@ GEM
     nio4r (2.5.8)
     nokogiri (1.13.3-x86_64-linux)
       racc (~> 1.4)
+    public_suffix (4.0.7)
     racc (1.6.0)
     rack (2.2.3)
     rack-test (1.1.0)
@@ -150,6 +174,7 @@ GEM
       thor (~> 1.0)
       zeitwerk (~> 2.5)
     rake (13.0.6)
+    rexml (3.2.5)
     sprockets (4.0.3)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -163,6 +188,13 @@ GEM
     timeout (0.2.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.8.1)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
     websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -175,9 +207,11 @@ DEPENDENCIES
   browser
   byebug
   faker!
+  http
   rails_url_shortener!
   sprockets-rails
   sqlite3
+  webmock
 
 BUNDLED WITH
    2.2.22

app/jobs/rails_url_shortener/application_job.rb

@@ -0,0 +1,9 @@
+module RailsUrlShortener
+  class ApplicationJob < ActiveJob::Base
+    # Automatically retry jobs that encountered a deadlock
+    # retry_on ActiveRecord::Deadlocked
+
+    # Most jobs are safe to ignore if the underlying records are no longer available
+    # discard_on ActiveJob::DeserializationError
+  end
+end

app/jobs/rails_url_shortener/ip_crawler_job.rb

@@ -0,0 +1,35 @@
+module RailsUrlShortener
+  class IpCrawlerJob < ApplicationJob
+    queue_as :default
+
+    require 'http'
+    require 'json'
+
+    ##
+    # this function get the ip related data
+    #
+    # create or update an existing record information for an ip
+
+    def perform(visit)
+      if Ipgeo.exists?(ip: visit.ip) && Ipgeo.find_by(ip: visit.ip).updated_at <= Time.now - 3.months
+        # Then update
+        ip = HTTP.get("http://ip-api.com/json/#{visit.ip}?fields=status,message,country,countryCode,region,regionName,city,zip,lat,lon,timezone,isp,org,as,mobile,proxy,hosting,query")
+        if ip.code == 200
+          ipgeo = Ipgeo.find_by(ip: visit.ip)
+          ipgeo.update(JSON.parse(ip.body).transform_keys { |key| key.to_s.underscore }.slice(*Ipgeo.column_names))
+        end
+      elsif !Ipgeo.exists?(ip: visit.ip)
+        # Then create a new record
+        ip = HTTP.get("http://ip-api.com/json/#{visit.ip}?fields=status,message,country,countryCode,region,regionName,city,zip,lat,lon,timezone,isp,org,as,mobile,proxy,hosting,query")
+        if ip.code == 200
+          ipgeo = Ipgeo.new(JSON.parse(ip.body).transform_keys { |key| key.to_s.underscore }.slice(*Ipgeo.column_names))
+          ipgeo.ip = JSON.parse(ip.body)['query']
+          ipgeo.save
+          visit.ipgeo = ipgeo
+        end
+      end
+    rescue Exception => e
+      print('Error' + e.to_s)
+    end
+  end
+end

app/models/rails_url_shortener/ipgeo.rb

@@ -0,0 +1,5 @@
+module RailsUrlShortener
+  class Ipgeo < ApplicationRecord
+    has_many :visits
+  end
+end

app/models/rails_url_shortener/visit.rb

@@ -2,6 +2,7 @@ module RailsUrlShortener
   require 'json'
   class Visit < ApplicationRecord
     belongs_to :url
+    belongs_to :ipgeo, optional: true
 
     ##
     # Parse a request information and save
@@ -15,7 +16,7 @@ def self.parse_and_save(url, request)
         false
       else
         # save
-        Visit.create(
+        visit = Visit.create(
           url: url,
           ip: request.ip,
           browser: browser.name,
@@ -25,6 +26,9 @@ def self.parse_and_save(url, request)
           bot: browser.bot?,
           user_agent: request.headers['User-Agent']
         )
+        # We enqueue a job for get more data later
+        IpCrawlerJob.perform_later(visit)
+        visit
       end
     end
   end

db/migrate/20220418184647_create_rails_url_shortener_ipgeos.rb

@@ -0,0 +1,28 @@
+class CreateRailsUrlShortenerIpgeos < ActiveRecord::Migration[7.0]
+  def up
+    create_table :rails_url_shortener_ipgeos do |t|
+      t.string  :ip
+      t.string  :country
+      t.string  :country_code
+      t.string  :region
+      t.string  :region_name
+      t.string  :city
+      t.string  :lat
+      t.string  :lon
+      t.string  :timezone
+      t.string  :isp
+      t.string  :org
+      t.string  :as
+      t.boolean :mobile
+      t.boolean :proxy
+      t.boolean :hosting
+      t.timestamps
+    end
+    add_column :rails_url_shortener_visits, :ipgeo_id, :integer
+    add_index :rails_url_shortener_visits, :ipgeo_id
+  end
+  def down
+    remove_column :rails_url_shortener_visits, :ipgeo_id
+    drop_table :rails_url_shortener_ipgeos
+  end
+end

lib/rails_url_shortener/engine.rb

@@ -2,5 +2,6 @@ module RailsUrlShortener
   class Engine < ::Rails::Engine
     isolate_namespace RailsUrlShortener
     require 'browser'
+    require 'http'
   end
 end

lib/rails_url_shortener/version.rb

@@ -1,3 +1,3 @@
 module RailsUrlShortener
-  VERSION = "0.1.4"
+  VERSION = "0.2.0"
 end

rails_url_shortener.gemspec

@@ -32,5 +32,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rails", ">= 7.0.2.3"
   spec.add_dependency "browser", ">= 5.3.0"
   spec.add_dependency "bundler", ">= 1.15.0"
+  spec.add_dependency "http", ">= 5.0.4"
 
 end

test/controllers/rails_url_shortener/urls_controller_test.rb

@@ -3,24 +3,29 @@
 module RailsUrlShortener
   class UrlsControllerTest < ActionDispatch::IntegrationTest
     include Engine.routes.url_helpers
+    include ActiveJob::TestHelper
 
     test 'show' do
       assert_difference 'Visit.count', 1 do
-        get "/shortener/#{rails_url_shortener_urls(:one).key}", headers: {
-          'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0'
-        }
-        assert_response :moved_permanently
-        assert_redirected_to rails_url_shortener_urls(:one).url
+        assert_enqueued_with(job: IpCrawlerJob) do
+          get "/shortener/#{rails_url_shortener_urls(:one).key}", headers: {
+            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0'
+          }
+          assert_response :moved_permanently
+          assert_redirected_to rails_url_shortener_urls(:one).url
+        end
       end
     end
 
     test 'show whith a not existing key' do
       assert_no_difference 'Visit.count', 1 do
-        get "/shortener/noexist", headers: {
-          'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0'
-        }
-        assert_response :moved_permanently
-        assert_redirected_to RailsUrlShortener.default_redirect
+        assert_no_enqueued_jobs do
+          get "/shortener/noexist", headers: {
+            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0'
+          }
+          assert_response :moved_permanently
+          assert_redirected_to RailsUrlShortener.default_redirect
+        end
       end
     end
   end

test/dummy/db/schema.rb

@@ -10,7 +10,27 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema[7.0].define(version: 2022_04_11_163405) do
+ActiveRecord::Schema[7.0].define(version: 2022_04_18_184647) do
+  create_table "rails_url_shortener_ipgeos", force: :cascade do |t|
+    t.string "ip"
+    t.string "country"
+    t.string "country_code"
+    t.string "region"
+    t.string "region_name"
+    t.string "city"
+    t.string "lat"
+    t.string "lon"
+    t.string "timezone"
+    t.string "isp"
+    t.string "org"
+    t.string "as"
+    t.boolean "mobile"
+    t.boolean "proxy"
+    t.boolean "hosting"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+  end
+
   create_table "rails_url_shortener_urls", force: :cascade do |t|
     t.string "owner_type"
     t.integer "owner_id"
@@ -35,6 +55,8 @@
     t.text "meta"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.integer "ipgeo_id"
+    t.index ["ipgeo_id"], name: "index_rails_url_shortener_visits_on_ipgeo_id"
     t.index ["url_id"], name: "index_rails_url_shortener_visits_on_url_id"
   end
 

test/fixtures/rails_url_shortener/ipgeos.yml

@@ -0,0 +1,13 @@
+# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
+
+# This model initially had no columns defined. If you add columns to the
+# model remove the "{}" from the fixture names and add the columns immediately
+# below each fixture, per the syntax in the comments below
+#
+one:
+  ip: 66.90.76.179
+  country_code: cl
+  country: chile
+  region:
+  region_name: Metropolitana
+  city: santiago

test/fixtures/rails_url_shortener/visits.yml

@@ -7,13 +7,14 @@
 <% (0..5).each do |i| %>
 <% user_agent = Faker::Internet.user_agent %>
 <%=numbers[u]%>_<%=numbers[i]%>:
-  url: <%numbers[u]%>
-  ip: <% Faker::Internet.public_ip_v4_address %>
-  browser: <% Browser.new(user_agent).name %>
-  browser_version: <% Browser.new(user_agent).full_version %>
-  platform: <% Browser.new(user_agent).platform.name %>
-  platform_version: <% Browser.new(user_agent).platform.version %>
-  user_agent: <% user_agent %>
+  url: <%= numbers[u]%>
+  ip: <%= Faker::Internet.public_ip_v4_address %>
+  browser: <%= Browser.new(user_agent).name %>
+  browser_version: <%= Browser.new(user_agent).full_version %>
+  platform: <%= Browser.new(user_agent).platform.name %>
+  platform_version: <%= Browser.new(user_agent).platform.version %>
+  user_agent: <%= user_agent %>
+  ipgeo: one
   # this variable is not important for now
   meta: "Mozilla/5.0 (Macintosh; Intel Mac OS X 11_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1 Safari/605.1.15"
 <% end %>