Skip to content

Commit

Permalink
Merge branch 'feature/update-releasedate-format' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
@sandypadmanabhan
sandypadmanabhan committed Feb 26, 2021
2 parents 0831462 + 19991ff commit 1b1601e
Show file tree
Hide file tree
Showing 2 changed files with 117 additions and 19 deletions.
74 changes: 55 additions & 19 deletions NOMIS/nomis.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"net/http"
"net/url"
"os"
"regexp"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -171,10 +172,11 @@ func main() {
UsageNotes: &[]models.UsageNote{},
}
var metaTitleInfo [5]string
for indx1 := range res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation {
str1 := res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx1].Title
var annotations = res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation
for index1 := range annotations {
str1 := annotations[index1].Title
if strings.HasPrefix(str1, "MetadataTitle") {
title = res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx1].Text.(string)
title = annotations[index1].Text.(string)
splitTitle := strings.Split(str1, "MetadataTitle")
if splitTitle[1] == "" {
num = 0
Expand All @@ -186,51 +188,72 @@ func main() {
}
}

for indx := range res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation {
for index := range annotations {
var example string
var annotation = res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[index]

str := res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx].Title
str := annotation.Title

switch str {
case "MetadataText0":
mapData.Description = res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx].Text.(string)
mapData.Description = annotation.Text.(string)

case "Keywords":
keywrd := res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx].Text.(string)
keywrd := annotation.Text.(string)
var split = strings.Split(keywrd, ",")
mapData.Keywords = split

case "LastUpdated":
tt := res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx].Text.(string)
t, _ := time.Parse("2006-01-02 15:04:05", tt)
tt := annotation.Text.(string)
t, parseErr := time.Parse("2006-01-02 15:04:05", tt)
if parseErr != nil {
log.Event(ctx, "error parsing date", log.ERROR, log.Error(err))
os.Exit(1)
}
mapData.LastUpdated = t
generalModel.LastUpdated = mapData.LastUpdated

case "Units":
mapData.UnitOfMeasure = res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx].Text.(string)
mapData.UnitOfMeasure = annotation.Text.(string)

case "Mnemonic":
ref := res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx].Text.(string)
ref := annotation.Text.(string)
param := strings.Split(ref, "c2011")
if len(param)<2{
log.Event(nil, "error Mnemonic length invalid", log.ERROR)
os.Exit(1)
}
mapData.NomisReferenceURL = "https://www.nomisweb.co.uk/census/2011/" + param[1]

case "FirstReleased":
releaseDt := res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx].Text.(string)
rd, _ := time.Parse("2006-01-02 15:04:05", releaseDt)
censusInstances.ReleaseDate = rd.String()
releaseDt := annotation.Text.(string)
rd, err := time.Parse("2006-01-02 15:04:05", releaseDt)
if err != nil {
log.Event(ctx, "failed to parse date correctly", log.ERROR, log.Error(err))
os.Exit(1)
}
censusInstances.ReleaseDate = rd.Format("2006-01-02T15:04:05.000Z")
}

if strings.HasPrefix(str, "MetadataText") {
if str != "MetadataText0" {
example, err = CheckSubString(annotation.Text.(string))
if err != nil {
log.Event(nil, "failed to get metadatatext", log.ERROR, log.Error(err))
os.Exit(1)
}
}
splitMetaData := strings.Split(str, "MetadataText")
txtNumber, _ := strconv.Atoi(splitMetaData[1])
if splitMetaData[1] == "" && splitMetaData[1] != "0" {
*censusInstances.UsageNotes = append(*censusInstances.UsageNotes, models.UsageNote{
Note: res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx].Text.(string),
Note: example,
Title: metaTitleInfo[0],
})

} else if splitMetaData[1] != "0" {
*censusInstances.UsageNotes = append(*censusInstances.UsageNotes, models.UsageNote{
Note: res.Structure.Keyfamilies.Keyfamily[index0].Annotations.Annotation[indx].Text.(string),
Note: example,
Title: metaTitleInfo[txtNumber+1],
})
}
Expand Down Expand Up @@ -267,7 +290,7 @@ func downloadFile() {
// Build fileName from fullPath
fileURL, err := url.Parse(fullURLFile)
if err != nil {
fmt.Println(("error Parsing"))
fmt.Println("error Parsing")
os.Exit(1)
}
path := fileURL.Path
Expand All @@ -278,7 +301,7 @@ func downloadFile() {
// Create blank file
file, err := os.Create(newFileName)
if err != nil {
fmt.Println(("error creating the file"))
fmt.Println("error creating the file")
os.Exit(1)
}
client := http.Client{
Expand All @@ -292,7 +315,7 @@ func downloadFile() {
resp, err := client.Get(fullURLFile)

if err != nil {
fmt.Println(("error writing the file"))
fmt.Println("error writing the file")
os.Exit(1)
}
defer resp.Body.Close()
Expand All @@ -301,3 +324,16 @@ func downloadFile() {

fmt.Printf("Downloaded a file %s with size %d", fileName, size)
}

/*checkSubString checks if the string has substrings http and [Statistical Disclosure Control].
If both the substrings exists then it adds parenthesis where necessary and swaps the pattern (url)[text] to [text](url)
so it can be displayed correctly. If substrings does not exists then it returns the original string*/
func CheckSubString(existingStr string) (string, error) {

valueCheck, err := regexp.Compile(`(http[^\[]*)(\[[^\[]*\])`)
if err != nil {
return "", err
}

return valueCheck.ReplaceAllString(existingStr, `$2($1)`), nil
}
62 changes: 62 additions & 0 deletions NOMIS/nomis_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package main_test

import (
nomis "github.com/ONSdigital/dp-dataset-api/NOMIS"
. "github.com/smartystreets/goconvey/convey"
"testing"
)

func TestCheckSubString(t *testing.T) {
cases := []struct {
Description string
GivenString string
ExpectedResult string
}{
{
"Given a string with http and [Statistical Disclosure Control]",
"you can get the information from " +
"http://www.ons.gov.uk/statistical-disclosure-control/index.html[Statistical Disclosure Control] page on the ONS web site.",
"you can get the information from [Statistical Disclosure Control]" +
"(http://www.ons.gov.uk/statistical-disclosure-control/index.html) page on the ONS web site.",
},
{
"Given a string with multiple http and [Statistical Disclosure Control]",
"you can get the information from " +
"http://www.ons.gov.uk/statistical-disclosure-control/index.html[Statistical Disclosure Control] and " +
"http://www.nomis/indx.html[Statistical Disclosure Control] page on the ONS web site.",
"you can get the information from [Statistical Disclosure Control]" +
"(http://www.ons.gov.uk/statistical-disclosure-control/index.html) and [Statistical Disclosure Control](http://www.nomis/indx.html) page on the ONS web site.",
},
{
"Given a string without [Statistical Disclosure Control]",
"you can get the information from " +
"http://www.ons.gov.uk/statistical-disclosure-control/index.aspx page on the ONS web site.",
"you can get the information from " +
"http://www.ons.gov.uk/statistical-disclosure-control/index.aspx page on the ONS web site.",
},
{
"Given a string without http and [Statistical Disclosure Control]",
"you can get the information from " +
"http://www.ons.gov.uk/statistical-disclosure-control/index.aspx page on the ONS web site.",
"you can get the information from " +
"http://www.ons.gov.uk/statistical-disclosure-control/index.aspx page on the ONS web site.",
},
{
"Given a string no matches",
"you can get the information from " +
"hello world",
"you can get the information from " +
"hello world",
},
}

for _, test := range cases {
Convey(test.Description, t, func() {
Convey("Then the CheckSubString function should return the expected string", func() {
actualString, err := nomis.CheckSubString(test.GivenString)
So(err, ShouldBeNil)
So(actualString, ShouldResemble, test.ExpectedResult)
})
})
}
}

0 comments on commit 1b1601e

Please sign in to comment.