Merge pull request #39 from Arkoniak/upgrading_to_julia1

Upgrading to julia1
JuliaWeb · Mar 23, 2021 · ddf4126 · ddf4126 · Arkoniak · Mar 23, 2021
2 parents 1df1beb + 6f01c17
commit ddf4126
Show file tree

Hide file tree

Showing 11 changed files with 338 additions and 88 deletions.
diff --git a/Project.toml b/Project.toml
@@ -8,16 +8,15 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 GZip = "92fee26a-97fe-5a0c-ad85-20a5f3185b63"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-IPNets = "66763231-799b-5fff-8662-389acfc33a85"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
 
 [compat]
-GZip = "0.5"
-ZipFile = "0.9"
+CSV = "0.8"
 DataFrames = "0.22"
+GZip = "0.5"
 HTTP = "0.9"
-CSV = "0.8"
+ZipFile = "0.8, 0.9"
 julia = "1"
 
 [extras]

diff --git a/README.md b/README.md
@@ -1,24 +1,29 @@
 # GeoIP
 
-*IP Geolocalization using the [Geolite2](https://dev.maxmind.com/geoip/geoip2/geolite2/) Database*
+IP Geolocalization using the [Geolite2](https://dev.maxmind.com/geoip/geoip2/geolite2/) Database
 
 |                                                                                                **Documentation**                                                                                                |                                                                                                                                        **Build Status**                                                                                                                                        |
 |:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|
 |   [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://JuliaWeb.github.io/GeoIP.jl/stable)[![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://JuliaWeb.github.io/GeoIP.jl/dev)   |                       [![Build](https://github.com/JuliaWeb/GeoIP.jl/workflows/CI/badge.svg)](https://github.com/JuliaWeb/GeoIP.jl/actions)[![Coverage](https://codecov.io/gh/JuliaWeb/GeoIP.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/JuliaWeb/GeoIP.jl)                       |
 
 ## Usage
 
-`GeoIP.geolocate(IPv4)` will load data from the CSV if it's
+`GeoIP.geolocate(::IPv4)` will load data from the CSV if it's
 not already loaded.
 
+Manual loading can be forced with `load()` call. Data should be located either in `ENV["GEOIP_DATADIR"]` or in package "data" subdirectory.
+
 ## Example
 
 ```julia
 using GeoIP
+
 a = ip"1.2.3.4"
 geolocate(a)        # returns dictionary with all relevant information
 ```
 
 ## Acknowledgements
 This product uses, but not include, GeoLite2 data created by MaxMind, available from
 [http://www.maxmind.com](http://www.maxmind.com).
+
+Source code of [IPNets.jl](https://github.com/JuliaWeb/IPNets.jl) was integrated as a part of package.
diff --git a/docs/src/index.md b/docs/src/index.md
@@ -4,17 +4,20 @@ CurrentModule = GeoIP
 
 # GeoIP
 
-*IP Geolocalization using the [Geolite2](https://dev.maxmind.com/geoip/geoip2/geolite2/) Database*
+IP Geolocalization using the [Geolite2](https://dev.maxmind.com/geoip/geoip2/geolite2/) Database
 
 ## Usage
 
 `GeoIP.geolocate(IPv4)` will load data from the CSV if it's
 not already loaded.
 
+Manual loading can be forced with `load()` call. Data should be located either in `ENV["GEOIP_DATADIR"]` or in package "data" subdirectory.
+
 ## Example
 
 ```julia
 using GeoIP
+
 a = ip"1.2.3.4"
 geolocate(a)        # returns dictionary with all relevant information
 ```
@@ -23,6 +26,8 @@ geolocate(a)        # returns dictionary with all relevant information
 This product uses, but not include, GeoLite2 data created by MaxMind, available from
 [http://www.maxmind.com](http://www.maxmind.com).
 
+Source code of [IPNets.jl](https://github.com/JuliaWeb/IPNets.jl) was integrated as a part of package.
+
 ```@index
 ```
 

diff --git a/src/GeoIP.jl b/src/GeoIP.jl
@@ -1,18 +1,21 @@
 module GeoIP
 
-using IPNets
 using DataFrames
 using ZipFile
 using GZip
 using HTTP
 using CSV
 import Sockets: IPv4
 
+include("ipnets.jl")
+using .IPNets
+
 export
     # types
     Location,
     # methods
-    geolocate
+    geolocate,
+    load
 
 include("data.jl")
 include("geoip-module.jl")

diff --git a/src/data.jl b/src/data.jl
@@ -23,7 +23,7 @@ function readmd5()
             strip(readline(f))
         end
     else
-        info("Failed to find checksum file, updating data...")
+        @info "Failed to find checksum file, updating data..."
         update()
         readmd5()
     end
@@ -34,7 +34,7 @@ function getmd5()
         r = HTTP.get(CITYMD5URL)
         return string(r.data)
     catch
-        error("Failed to download checksum file from MaxMind, check network connectivity")
+        @error "Failed to download checksum file from MaxMind, check network connectivity"
     end
 end
 
@@ -44,7 +44,7 @@ function dldata(md5::String)
     r = try
         HTTP.get(CITYDLURL)
     catch
-        error("Failed to download file from MaxMind, check network connectivity")
+        @error "Failed to download file from MaxMind, check network connectivity"
     end
 
     archive = ZipFile.Reader(IOBuffer(r.data))
@@ -68,7 +68,7 @@ function dldata(md5::String)
             write(f, md5)
         end
     else
-        error("Problem with download: only $dlcount of 2 files downloaded")
+        @error "Problem with download: only $dlcount of 2 files downloaded"
     end
 end
 
@@ -77,34 +77,37 @@ function update()
     global dataloaded = false
 end
 
-function load()
-    blockfile = joinpath(DATADIR, BLOCKCSVGZ)
-    locfile = joinpath(DATADIR, CITYCSVGZ)
+function load(datadir = DATADIR)
+    blockfile = joinpath(datadir, BLOCKCSVGZ)
+    locfile = joinpath(datadir, CITYCSVGZ)
 
-    blocks = DataFrame()
-    locs = DataFrame()
+    local blocks
+    local locs
     try
         blocks = GZip.open(blockfile, "r") do stream
-            CSV.read(stream, nullable=true, types=[String, Int, Int, String, Int, Int, String, Float64, Float64, Int])
+            CSV.File(read(stream)) |> DataFrame
+            # CSV.File(stream, types=[String, Int, Int, String, Int, Int, String, Float64, Float64, Int]) |> DataFrame
         end
         locs = GZip.open(locfile, "r") do stream
-            CSV.read(stream, nullable=true, types=[Int, String, String, String, String, String, String, String, String, String, String, Int, String, Int])
+            # CSV.File(stream, types=[Int, String, String, String, String, String, String, String, String, String, String, Int, String, Int]) |> DataFrame
+            CSV.File(read(stream)) |> DataFrame
         end
     catch
-        error("Geolocation data cannot be read. Data directory may be corrupt...")
+        @error "Geolocation data cannot be read. Data directory may be corrupt..."
     end
 
     # Clean up unneeded columns and map others to appropriate data structures
-    delete!(blocks, [:represented_country_geoname_id, :is_anonymous_proxy, :is_satellite_provider])
+    select!(blocks, Not([:represented_country_geoname_id, :is_anonymous_proxy, :is_satellite_provider]))
 
-    blocks[:v4net] = map(x -> IPNets.IPv4Net(x), blocks[:network])
-    delete!(blocks, :network)
+    blocks[!, :v4net] = map(x -> IPNets.IPv4Net(x), blocks[!, :network])
+    select!(blocks, Not(:network))
 
-    blocks[:location] = map(Location, blocks[:longitude], blocks[:latitude])
-    delete!(blocks, [:longitude, :latitude])
+    blocks[!, :location] = map(Location, blocks[!, :longitude], blocks[!, :latitude])
+    select!(blocks, Not([:longitude, :latitude]))
+    blocks.geoname_id = map(x -> ismissing(x) ? -1 : Int(x), blocks.geoname_id)
 
-    alldata = join(blocks, locs, on=:geoname_id, kind=:inner)
+    alldata = leftjoin(blocks, locs, on = :geoname_id)
 
     global dataloaded = true
-    global geodata = sort(alldata, cols=[:v4net])
+    global geodata = sort(alldata, :v4net)
 end
diff --git a/src/geoip-module.jl b/src/geoip-module.jl
@@ -1,3 +1,6 @@
+########################################
+# Location structure
+########################################
 # It would be great to replace this with a real GIS package.
 abstract type Point end
 abstract type Point3D <: Point end
@@ -8,26 +11,32 @@ struct Location <: Point3D
     z::Float64
     datum::String
 
-    function Location(x,y,z=0, datum="WGS84")
+    function Location(x, y, z = 0, datum = "WGS84")
         if x === missing || y === missing
             return missing
         else
-            return new(x,y,z,datum)
+            return new(x, y, z, datum)
         end
     end
 end
 
-function geolocate(ip::IPv4; noupdate=true)
-    if updaterequired()
-        if !(noupdate)
+########################################
+# Geolocation functions
+########################################
+"""
+    geolocate(ip, noupdate = true)
+
+Returns geolocation and other information determined by `ip`. If `noupdate` is `true`, then no updates check is performed and current data is used for the location lookup.
+"""
+function geolocate(ip::IPv4; noupdate = true)
+    if !noupdate
+        if updaterequired()
             update()
-        else
-            warn("Geolocation data is out of date. Consider updating.")
         end
     end
 
     if !(dataloaded)
-        info("Geolocation data not in memory. Loading...")
+        @info "Geolocation data not in memory. Loading..."
         load()
     end
 
@@ -43,34 +52,15 @@ function geolocate(ip::IPv4; noupdate=true)
         end
     end
 
-    retdict = Dict{Symbol, Any}()
+    # TODO: sentinel value should be returned
+    retdict = Dict{String, Any}()
     if (found > 0) && ip in geodata[found, :v4net]
-        for (k,v) in eachcol(geodata[found, :])
-            retdict[k] = v[1]
-        end
+        # Placeholder, should be removed
+        row = geodata[found, :]
+        return Dict(collect(zip(names(row), row)))
     end
     return retdict
 end
 
-function geolocate(iparr::AbstractArray; noupdate=true)
-    masterdict = Dict{Symbol, Any}[]
-    for el in iparr
-        push!(masterdict, geolocate(el; noupdate=noupdate))
-    end
-    return masterdict
-end
-
-######################################
-# deprecations / convenience functions
-######################################
-@deprecate getcountrycode(ip)   geolocate(IPv4(ip))[:country_iso_code]
-@deprecate getcountryname(ip)   geolocate(IPv4(ip))[:country_name]
-@deprecate getregionname(ip)    geolocate(IPv4(ip))[:subdivision_1_name]
-@deprecate getcityname(ip)      geolocate(IPv4(ip))[:city_name]
-@deprecate getpostalcode(ip)    geolocate(IPv4(ip))[:postal_code]
-@deprecate getlongitude(ip)     geolocate(IPv4(ip))[:location].x
-@deprecate getlatitude(ip)      geolocate(IPv4(ip))[:location].y
-@deprecate getmetrocode(ip)     geolocate(IPv4(ip))[:metro_code]
-@deprecate getareacode(ip)      geolocate(IPv4(ip))[:metro_code]
-@deprecate geolocate(ipstr::AbstractString) geolocate(IPv4(ipstr))
-@deprecate geolocate(ipint::Integer) geolocate(IPv4(ipint))
+geolocate(ipstr::AbstractString; noupdate = true) = geolocate(IPv4(ipstr); noupdate = noupdate)
+geolocate(ipint::Integer; noupdate = true) = geolocate(IPv4(ipint); noupdate = noupdate)